From 5eb0772bcff34eeab7c5b5dcf30b41c21b84f06f Mon Sep 17 00:00:00 2001 From: Benjamin Date: Sat, 22 Aug 2026 11:38:10 +0200 Subject: [PATCH 1/6] Add a generic message reader interface Adds a generic message reader interface for reading proto messages from storage. Many Buildbarn components depend on blobstore.BlobAccess only as a source of proto.Message and this interface allows us to sever that dependency in components which are otherwise not interested in the blobstore.BlobAccess interface. --- cmd/bb_storage/main.go | 20 ++++++-- internal/mock/BUILD.bazel | 12 +++++ pkg/blobstore/BUILD.bazel | 2 + pkg/blobstore/blob_access_message_reader.go | 32 +++++++++++++ pkg/blobstore/grpcservers/BUILD.bazel | 6 +++ .../grpcservers/action_cache_server.go | 20 +++----- .../grpcservers/action_cache_server_test.go | 46 ++++++++++++++++++ .../file_system_access_cache_server.go | 20 +++----- .../file_system_access_cache_server_test.go | 47 +++++++++++++++++++ ...rect_content_addressable_storage_server.go | 20 +++----- ...content_addressable_storage_server_test.go | 34 +++++++------- .../initial_size_class_cache_server.go | 20 +++----- .../initial_size_class_cache_server_test.go | 47 +++++++++++++++++++ pkg/storage/BUILD.bazel | 15 ++++++ pkg/storage/message_reader.go | 19 ++++++++ 15 files changed, 288 insertions(+), 72 deletions(-) create mode 100644 pkg/blobstore/blob_access_message_reader.go create mode 100644 pkg/blobstore/grpcservers/action_cache_server_test.go create mode 100644 pkg/blobstore/grpcservers/file_system_access_cache_server_test.go create mode 100644 pkg/blobstore/grpcservers/initial_size_class_cache_server_test.go create mode 100644 pkg/storage/BUILD.bazel create mode 100644 pkg/storage/message_reader.go diff --git a/cmd/bb_storage/main.go b/cmd/bb_storage/main.go index 192fcb5e4..3faab6b4f 100644 --- a/cmd/bb_storage/main.go +++ b/cmd/bb_storage/main.go @@ -214,7 +214,10 @@ func main() { s, grpcservers.NewActionCacheServer( actionCache, - int(configuration.MaximumMessageSizeBytes), + blobstore.NewBlobAccessMessageReader[*remoteexecution.ActionResult]( + actionCache, + int(configuration.MaximumMessageSizeBytes), + ), ), ) } @@ -223,7 +226,10 @@ func main() { s, grpcservers.NewIndirectContentAddressableStorageServer( indirectContentAddressableStorage, - int(configuration.MaximumMessageSizeBytes), + blobstore.NewBlobAccessMessageReader[*icas.Reference]( + indirectContentAddressableStorage, + int(configuration.MaximumMessageSizeBytes), + ), ), ) } @@ -232,7 +238,10 @@ func main() { s, grpcservers.NewInitialSizeClassCacheServer( initialSizeClassCache, - int(configuration.MaximumMessageSizeBytes), + blobstore.NewBlobAccessMessageReader[*iscc.PreviousExecutionStats]( + initialSizeClassCache, + int(configuration.MaximumMessageSizeBytes), + ), ), ) } @@ -241,7 +250,10 @@ func main() { s, grpcservers.NewFileSystemAccessCacheServer( fileSystemAccessCache, - int(configuration.MaximumMessageSizeBytes), + blobstore.NewBlobAccessMessageReader[*fsac.FileSystemAccessProfile]( + fileSystemAccessCache, + int(configuration.MaximumMessageSizeBytes), + ), ), ) } diff --git a/internal/mock/BUILD.bazel b/internal/mock/BUILD.bazel index 87d7fda65..94a9a7706 100644 --- a/internal/mock/BUILD.bazel +++ b/internal/mock/BUILD.bazel @@ -320,6 +320,16 @@ gomock( package = "mock", ) +gomock( + name = "storage", + out = "storage.go", + interfaces = ["MessageReader"], + library = "//pkg/storage", + mockgen_tool = "@org_uber_go_mock//mockgen", + package = "mock", + source = "//pkg/storage:message_reader.go", +) + gomock( name = "trace", out = "trace.go", @@ -380,6 +390,7 @@ go_library( "prometheus.go", "random.go", "remoteexecution.go", + "storage.go", "trace.go", "trace_wrap.go", "util.go", @@ -416,6 +427,7 @@ go_library( "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//metadata", "@org_golang_google_protobuf//encoding/protowire", + "@org_golang_google_protobuf//proto", "@org_uber_go_mock//gomock", ], ) diff --git a/pkg/blobstore/BUILD.bazel b/pkg/blobstore/BUILD.bazel index 45d054538..218c58a9c 100644 --- a/pkg/blobstore/BUILD.bazel +++ b/pkg/blobstore/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "action_result_timestamp_injecting_blob_access.go", "authorizing_blob_access.go", "blob_access.go", + "blob_access_message_reader.go", "cas_read_buffer_factory.go", "deadline_enforcing_blob_access.go", "demultiplexing_blob_access.go", @@ -42,6 +43,7 @@ go_library( "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", + "//pkg/storage", "//pkg/util", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/blob_access_message_reader.go b/pkg/blobstore/blob_access_message_reader.go new file mode 100644 index 000000000..6c697b1d1 --- /dev/null +++ b/pkg/blobstore/blob_access_message_reader.go @@ -0,0 +1,32 @@ +package blobstore + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/storage" + "google.golang.org/protobuf/proto" +) + +type blobAccessMessageReader[T proto.Message] struct { + blobAccess BlobAccess + maximumMessageSizeBytes int +} + +// NewBlobAccessMessageReader creates a storage.MessageReader that reads +// a message from a BlobAccess up to maximumMessageSizeBytes. +func NewBlobAccessMessageReader[T proto.Message](blobAccess BlobAccess, maximumMessageSizeBytes int) storage.MessageReader[T] { + return blobAccessMessageReader[T]{ + blobAccess: blobAccess, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (mr blobAccessMessageReader[T]) ReadMessage(ctx context.Context, d digest.Digest, msg T) (T, error) { + var zero T + ret, err := mr.blobAccess.Get(ctx, d).ToProto(msg, mr.maximumMessageSizeBytes) + if err != nil { + return zero, err + } + return ret.(T), nil +} diff --git a/pkg/blobstore/grpcservers/BUILD.bazel b/pkg/blobstore/grpcservers/BUILD.bazel index 556ad91e8..0069854c1 100644 --- a/pkg/blobstore/grpcservers/BUILD.bazel +++ b/pkg/blobstore/grpcservers/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", + "//pkg/storage", "//pkg/util", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", @@ -32,16 +33,21 @@ go_library( go_test( name = "grpcservers_test", srcs = [ + "action_cache_server_test.go", "byte_stream_server_test.go", "content_addressable_storage_server_test.go", + "file_system_access_cache_server_test.go", "indirect_content_addressable_storage_server_test.go", + "initial_size_class_cache_server_test.go", ], deps = [ ":grpcservers", "//internal/mock", "//pkg/blobstore/buffer", "//pkg/digest", + "//pkg/proto/fsac", "//pkg/proto/icas", + "//pkg/proto/iscc", "//pkg/testutil", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/grpcservers/action_cache_server.go b/pkg/blobstore/grpcservers/action_cache_server.go index e8d5d5942..2b92b7316 100644 --- a/pkg/blobstore/grpcservers/action_cache_server.go +++ b/pkg/blobstore/grpcservers/action_cache_server.go @@ -7,20 +7,21 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/storage" "github.com/buildbarn/bb-storage/pkg/util" ) type actionCacheServer struct { - blobAccess blobstore.BlobAccess - maximumMessageSizeBytes int + blobAccess blobstore.BlobAccess + resultReader storage.MessageReader[*remoteexecution.ActionResult] } // NewActionCacheServer creates a GRPC service for serving the contents // of a Bazel Action Cache (AC) to Bazel. -func NewActionCacheServer(blobAccess blobstore.BlobAccess, maximumMessageSizeBytes int) remoteexecution.ActionCacheServer { +func NewActionCacheServer(blobAccess blobstore.BlobAccess, resultReader storage.MessageReader[*remoteexecution.ActionResult]) remoteexecution.ActionCacheServer { return &actionCacheServer{ - blobAccess: blobAccess, - maximumMessageSizeBytes: maximumMessageSizeBytes, + blobAccess: blobAccess, + resultReader: resultReader, } } @@ -37,14 +38,7 @@ func (s *actionCacheServer) GetActionResult(ctx context.Context, in *remoteexecu if err != nil { return nil, err } - actionResult, err := s.blobAccess.Get(ctx, digest).ToProto( - &remoteexecution.ActionResult{}, - s.maximumMessageSizeBytes, - ) - if err != nil { - return nil, err - } - return actionResult.(*remoteexecution.ActionResult), nil + return s.resultReader.ReadMessage(ctx, digest, &remoteexecution.ActionResult{}) } func (s *actionCacheServer) UpdateActionResult(ctx context.Context, in *remoteexecution.UpdateActionResultRequest) (*remoteexecution.ActionResult, error) { diff --git a/pkg/blobstore/grpcservers/action_cache_server_test.go b/pkg/blobstore/grpcservers/action_cache_server_test.go new file mode 100644 index 000000000..5f3bcd60c --- /dev/null +++ b/pkg/blobstore/grpcservers/action_cache_server_test.go @@ -0,0 +1,46 @@ +package grpcservers_test + +import ( + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/internal/mock" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" + + "go.uber.org/mock/gomock" +) + +func TestActionCacheServer(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + blobAccess := mock.NewMockBlobAccess(ctrl) + resultReader := mock.NewMockMessageReader[*remoteexecution.ActionResult](ctrl) + + server := grpcservers.NewActionCacheServer(blobAccess, resultReader) + + t.Run("GetActionResultSuccess", func(t *testing.T) { + digest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 100) + result := &remoteexecution.ActionResult{} + + resultReader.EXPECT().ReadMessage( + gomock.Any(), + digest, + gomock.Any(), + ).Return(result, nil) + + resp, err := server.GetActionResult(ctx, &remoteexecution.GetActionResultRequest{ + InstanceName: "ubuntu1804", + DigestFunction: remoteexecution.DigestFunction_MD5, + ActionDigest: &remoteexecution.Digest{ + Hash: "09f7e02f1290be211da707a266f153b3", + SizeBytes: 100, + }, + }) + + require.NoError(t, err) + require.Equal(t, result, resp) + }) +} diff --git a/pkg/blobstore/grpcservers/file_system_access_cache_server.go b/pkg/blobstore/grpcservers/file_system_access_cache_server.go index 6d7bab695..90c2ed756 100644 --- a/pkg/blobstore/grpcservers/file_system_access_cache_server.go +++ b/pkg/blobstore/grpcservers/file_system_access_cache_server.go @@ -7,14 +7,15 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/fsac" + "github.com/buildbarn/bb-storage/pkg/storage" "github.com/buildbarn/bb-storage/pkg/util" "google.golang.org/protobuf/types/known/emptypb" ) type fileSystemAccessCacheServer struct { - blobAccess blobstore.BlobAccess - maximumMessageSizeBytes int + blobAccess blobstore.BlobAccess + profileReader storage.MessageReader[*fsac.FileSystemAccessProfile] } // NewFileSystemAccessCacheServer creates a gRPC service for serving the @@ -22,10 +23,10 @@ type fileSystemAccessCacheServer struct { // that is specific to Buildbarn, used to store profiles of file system // access patterns of build actions. These profiles can be used to // perform readahead of objects stored in the action's input root. -func NewFileSystemAccessCacheServer(blobAccess blobstore.BlobAccess, maximumMessageSizeBytes int) fsac.FileSystemAccessCacheServer { +func NewFileSystemAccessCacheServer(blobAccess blobstore.BlobAccess, profileReader storage.MessageReader[*fsac.FileSystemAccessProfile]) fsac.FileSystemAccessCacheServer { return &fileSystemAccessCacheServer{ - blobAccess: blobAccess, - maximumMessageSizeBytes: maximumMessageSizeBytes, + blobAccess: blobAccess, + profileReader: profileReader, } } @@ -43,14 +44,7 @@ func (s *fileSystemAccessCacheServer) GetFileSystemAccessProfile(ctx context.Con if err != nil { return nil, err } - previousExecutionStats, err := s.blobAccess.Get(ctx, digest).ToProto( - &fsac.FileSystemAccessProfile{}, - s.maximumMessageSizeBytes, - ) - if err != nil { - return nil, err - } - return previousExecutionStats.(*fsac.FileSystemAccessProfile), nil + return s.profileReader.ReadMessage(ctx, digest, &fsac.FileSystemAccessProfile{}) } func (s *fileSystemAccessCacheServer) UpdateFileSystemAccessProfile(ctx context.Context, in *fsac.UpdateFileSystemAccessProfileRequest) (*emptypb.Empty, error) { diff --git a/pkg/blobstore/grpcservers/file_system_access_cache_server_test.go b/pkg/blobstore/grpcservers/file_system_access_cache_server_test.go new file mode 100644 index 000000000..a302c5124 --- /dev/null +++ b/pkg/blobstore/grpcservers/file_system_access_cache_server_test.go @@ -0,0 +1,47 @@ +package grpcservers_test + +import ( + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/internal/mock" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/proto/fsac" + "github.com/stretchr/testify/require" + + "go.uber.org/mock/gomock" +) + +func TestFileSystemAccessCacheServer(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + blobAccess := mock.NewMockBlobAccess(ctrl) + profileReader := mock.NewMockMessageReader[*fsac.FileSystemAccessProfile](ctrl) + + server := grpcservers.NewFileSystemAccessCacheServer(blobAccess, profileReader) + + t.Run("GetFileSystemAccessProfileSuccess", func(t *testing.T) { + digest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 100) + profile := &fsac.FileSystemAccessProfile{} + + profileReader.EXPECT().ReadMessage( + gomock.Any(), + digest, + gomock.Any(), + ).Return(profile, nil) + + resp, err := server.GetFileSystemAccessProfile(ctx, &fsac.GetFileSystemAccessProfileRequest{ + InstanceName: "ubuntu1804", + DigestFunction: remoteexecution.DigestFunction_MD5, + ReducedActionDigest: &remoteexecution.Digest{ + Hash: "09f7e02f1290be211da707a266f153b3", + SizeBytes: 100, + }, + }) + + require.NoError(t, err) + require.Equal(t, profile, resp) + }) +} diff --git a/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server.go b/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server.go index b38557309..17bdd4c44 100644 --- a/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server.go +++ b/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server.go @@ -8,24 +8,25 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/icas" + "github.com/buildbarn/bb-storage/pkg/storage" "github.com/buildbarn/bb-storage/pkg/util" "google.golang.org/grpc/status" ) type indirectContentAddressableStorageServer struct { - blobAccess blobstore.BlobAccess - maximumMessageSizeBytes int + blobAccess blobstore.BlobAccess + referenceReader storage.MessageReader[*icas.Reference] } // NewIndirectContentAddressableStorageServer creates a gRPC service for // serving the contents of an Indirect Content Addressable Storage // (ICAS). The ICAS is a Buildbarn specific extension for integrating // external corpora into the CAS. -func NewIndirectContentAddressableStorageServer(blobAccess blobstore.BlobAccess, maximumMessageSizeBytes int) icas.IndirectContentAddressableStorageServer { +func NewIndirectContentAddressableStorageServer(blobAccess blobstore.BlobAccess, referenceReader storage.MessageReader[*icas.Reference]) icas.IndirectContentAddressableStorageServer { return &indirectContentAddressableStorageServer{ - blobAccess: blobAccess, - maximumMessageSizeBytes: maximumMessageSizeBytes, + blobAccess: blobAccess, + referenceReader: referenceReader, } } @@ -105,12 +106,5 @@ func (s *indirectContentAddressableStorageServer) GetReference(ctx context.Conte if err != nil { return nil, err } - actionResult, err := s.blobAccess.Get(ctx, digest).ToProto( - &icas.Reference{}, - s.maximumMessageSizeBytes, - ) - if err != nil { - return nil, err - } - return actionResult.(*icas.Reference), nil + return s.referenceReader.ReadMessage(ctx, digest, &icas.Reference{}) } diff --git a/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server_test.go b/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server_test.go index c4e2b9761..1acc80470 100644 --- a/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server_test.go +++ b/pkg/blobstore/grpcservers/indirect_content_addressable_storage_server_test.go @@ -23,7 +23,8 @@ func TestIndirectContentAddressableStorageServerFindMissingReferences(t *testing ctrl, ctx := gomock.WithContext(context.Background(), t) blobAccess := mock.NewMockBlobAccess(ctrl) - s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, 1000) + referenceReader := mock.NewMockMessageReader[*icas.Reference](ctrl) + s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, referenceReader) t.Run("BadDigest", func(t *testing.T) { // Malformed requests cannot be executed. @@ -107,7 +108,8 @@ func TestIndirectContentAddressableStorageServerBatchUpdateReferences(t *testing ctrl, ctx := gomock.WithContext(context.Background(), t) blobAccess := mock.NewMockBlobAccess(ctrl) - s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, 1000) + referenceReader := mock.NewMockMessageReader[*icas.Reference](ctrl) + s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, referenceReader) t.Run("Mixed", func(t *testing.T) { // Send a single batch update request containing three @@ -212,7 +214,8 @@ func TestIndirectContentAddressableStorageServerGetReference(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) blobAccess := mock.NewMockBlobAccess(ctrl) - s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, 1000) + referenceReader := mock.NewMockMessageReader[*icas.Reference](ctrl) + s := grpcservers.NewIndirectContentAddressableStorageServer(blobAccess, referenceReader) t.Run("BadDigest", func(t *testing.T) { // Malformed requests cannot be executed. @@ -228,11 +231,12 @@ func TestIndirectContentAddressableStorageServerGetReference(t *testing.T) { t.Run("BackendFailure", func(t *testing.T) { // Errors returned by the backend should be forwarded. - blobAccess.EXPECT().Get( + referenceReader.EXPECT().ReadMessage( ctx, digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), + gomock.Any(), ). - Return(buffer.NewBufferFromError(status.Error(codes.Internal, "Hardware failure"))) + Return(nil, status.Error(codes.Internal, "Hardware failure")) _, err := s.GetReference(ctx, &icas.GetReferenceRequest{ InstanceName: "example", @@ -246,20 +250,18 @@ func TestIndirectContentAddressableStorageServerGetReference(t *testing.T) { }) t.Run("Success", func(t *testing.T) { - dataIntegrityCallback := mock.NewMockDataIntegrityCallback(ctrl) - dataIntegrityCallback.EXPECT().Call(true) - blobAccess.EXPECT().Get( + referenceReader.EXPECT().ReadMessage( ctx, digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - ). - Return(buffer.NewProtoBufferFromProto( - &icas.Reference{ - Medium: &icas.Reference_HttpUrl{ - HttpUrl: "http://example.com/file3.txt", - }, + gomock.Any(), + ).Return( + &icas.Reference{ + Medium: &icas.Reference_HttpUrl{ + HttpUrl: "http://example.com/file3.txt", }, - buffer.BackendProvided(dataIntegrityCallback.Call), - )) + }, + nil, + ) resp, err := s.GetReference(ctx, &icas.GetReferenceRequest{ InstanceName: "example", diff --git a/pkg/blobstore/grpcservers/initial_size_class_cache_server.go b/pkg/blobstore/grpcservers/initial_size_class_cache_server.go index fbc4ae3ca..3bcab85f8 100644 --- a/pkg/blobstore/grpcservers/initial_size_class_cache_server.go +++ b/pkg/blobstore/grpcservers/initial_size_class_cache_server.go @@ -7,14 +7,15 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/iscc" + "github.com/buildbarn/bb-storage/pkg/storage" "github.com/buildbarn/bb-storage/pkg/util" "google.golang.org/protobuf/types/known/emptypb" ) type initialSizeClassCacheServer struct { - blobAccess blobstore.BlobAccess - maximumMessageSizeBytes int + blobAccess blobstore.BlobAccess + statsReader storage.MessageReader[*iscc.PreviousExecutionStats] } // NewInitialSizeClassCacheServer creates a gRPC service for serving the @@ -23,10 +24,10 @@ type initialSizeClassCacheServer struct { // execution times of actions, so that it can make better predictions // about which size class to pick during future invocations of similar // actions. -func NewInitialSizeClassCacheServer(blobAccess blobstore.BlobAccess, maximumMessageSizeBytes int) iscc.InitialSizeClassCacheServer { +func NewInitialSizeClassCacheServer(blobAccess blobstore.BlobAccess, statsReader storage.MessageReader[*iscc.PreviousExecutionStats]) iscc.InitialSizeClassCacheServer { return &initialSizeClassCacheServer{ - blobAccess: blobAccess, - maximumMessageSizeBytes: maximumMessageSizeBytes, + blobAccess: blobAccess, + statsReader: statsReader, } } @@ -44,14 +45,7 @@ func (s *initialSizeClassCacheServer) GetPreviousExecutionStats(ctx context.Cont if err != nil { return nil, err } - previousExecutionStats, err := s.blobAccess.Get(ctx, digest).ToProto( - &iscc.PreviousExecutionStats{}, - s.maximumMessageSizeBytes, - ) - if err != nil { - return nil, err - } - return previousExecutionStats.(*iscc.PreviousExecutionStats), nil + return s.statsReader.ReadMessage(ctx, digest, &iscc.PreviousExecutionStats{}) } func (s *initialSizeClassCacheServer) UpdatePreviousExecutionStats(ctx context.Context, in *iscc.UpdatePreviousExecutionStatsRequest) (*emptypb.Empty, error) { diff --git a/pkg/blobstore/grpcservers/initial_size_class_cache_server_test.go b/pkg/blobstore/grpcservers/initial_size_class_cache_server_test.go new file mode 100644 index 000000000..3d687683a --- /dev/null +++ b/pkg/blobstore/grpcservers/initial_size_class_cache_server_test.go @@ -0,0 +1,47 @@ +package grpcservers_test + +import ( + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/internal/mock" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/proto/iscc" + "github.com/stretchr/testify/require" + + "go.uber.org/mock/gomock" +) + +func TestInitialSizeClassCacheServer(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + blobAccess := mock.NewMockBlobAccess(ctrl) + statsReader := mock.NewMockMessageReader[*iscc.PreviousExecutionStats](ctrl) + + server := grpcservers.NewInitialSizeClassCacheServer(blobAccess, statsReader) + + t.Run("GetPreviousExecutionStatsSuccess", func(t *testing.T) { + digest := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 100) + stats := &iscc.PreviousExecutionStats{} + + statsReader.EXPECT().ReadMessage( + gomock.Any(), + digest, + gomock.Any(), + ).Return(stats, nil) + + resp, err := server.GetPreviousExecutionStats(ctx, &iscc.GetPreviousExecutionStatsRequest{ + InstanceName: "ubuntu1804", + DigestFunction: remoteexecution.DigestFunction_MD5, + ReducedActionDigest: &remoteexecution.Digest{ + Hash: "09f7e02f1290be211da707a266f153b3", + SizeBytes: 100, + }, + }) + + require.NoError(t, err) + require.Equal(t, stats, resp) + }) +} diff --git a/pkg/storage/BUILD.bazel b/pkg/storage/BUILD.bazel new file mode 100644 index 000000000..0708f6cc6 --- /dev/null +++ b/pkg/storage/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +# Exported for mock generation purposes. +exports_files(["message_reader.go"]) + +go_library( + name = "storage", + srcs = ["message_reader.go"], + importpath = "github.com/buildbarn/bb-storage/pkg/storage", + visibility = ["//visibility:public"], + deps = [ + "//pkg/digest", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/pkg/storage/message_reader.go b/pkg/storage/message_reader.go new file mode 100644 index 000000000..b5829da30 --- /dev/null +++ b/pkg/storage/message_reader.go @@ -0,0 +1,19 @@ +package storage + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/protobuf/proto" +) + +// MessageReader can be used to read a proto.Message from storage and +// return its contents. +type MessageReader[T proto.Message] interface { + // Return a parsed message from storage. + // + // Implementations may use the provided message to store the + // unmarshalled message. + ReadMessage(ctx context.Context, d digest.Digest, msg T) (T, error) +} From 9a21c6a750da33e4d0e071f55b757351e482ebf6 Mon Sep 17 00:00:00 2001 From: Benjamin Ingberg Date: Fri, 29 May 2026 09:49:44 +0200 Subject: [PATCH 2/6] Add support for SplitBlob and SpliceBlob methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds support for the SplitBlob and SpliceBlob methods from the Remote Execution v2 (REv2) api. SplitBlob and SpliceBlob can be used to facilitate uploads and downloads of large files but a naïve implementation like this has some major drawbacks as well. The blobs must exist in both their chunked and non chunked form, which may significantly increase storage requirements for large blobs. The protocol gives no guarantee that a large blob stored in the CAS exists in its chunked form which forces you to perform a fairly heavy Split call that loads the entire large blob in order to decomposition it into its chunks. This implementation mostly exists as a stepping stone for a different implementation where Buildbarn internally manages all blobs as chunked blobs. --- MODULE.bazel | 1 + cmd/bb_storage/main.go | 31 +- go.mod | 1 + go.sum | 2 + pkg/blobstore/BUILD.bazel | 1 + pkg/blobstore/chunklistvalidating/BUILD.bazel | 43 ++ .../chunk_concatenating_reader.go | 68 +++ .../chunk_list_validating_blob_access.go | 362 ++++++++++++++ .../chunk_list_validating_blob_access_test.go | 410 ++++++++++++++++ pkg/blobstore/chunklistvalidating/chunker.go | 16 + .../fake_blob_access_test.go | 88 ++++ .../integration/BUILD.bazel | 19 + .../chunk_list_validating_integration_test.go | 445 ++++++++++++++++++ .../chunklistvalidating/reader_chunker.go | 53 +++ .../reader_chunker_test.go | 107 +++++ pkg/blobstore/cls_read_buffer_factory.go | 27 ++ pkg/blobstore/configuration/BUILD.bazel | 2 + .../configuration/blob_access_creator.go | 2 +- .../configuration/cas_blob_access_creator.go | 4 +- .../configuration/cls_blob_access_creator.go | 86 ++++ .../configuration/new_blob_access.go | 19 +- .../proto_blob_access_creator.go | 3 +- pkg/blobstore/grpcclients/BUILD.bazel | 1 + pkg/blobstore/grpcclients/cls_blob_access.go | 101 ++++ .../content_addressable_storage_server.go | 93 +++- ...content_addressable_storage_server_test.go | 134 +++++- .../configuration/bb_storage/bb_storage.pb.go | 54 ++- .../configuration/bb_storage/bb_storage.proto | 4 + pkg/proto/configuration/blobstore/BUILD.bazel | 2 + .../configuration/blobstore/blobstore.pb.go | 273 +++++++---- .../configuration/blobstore/blobstore.proto | 41 ++ 31 files changed, 2358 insertions(+), 135 deletions(-) create mode 100644 pkg/blobstore/chunklistvalidating/BUILD.bazel create mode 100644 pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go create mode 100644 pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go create mode 100644 pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go create mode 100644 pkg/blobstore/chunklistvalidating/chunker.go create mode 100644 pkg/blobstore/chunklistvalidating/fake_blob_access_test.go create mode 100644 pkg/blobstore/chunklistvalidating/integration/BUILD.bazel create mode 100644 pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go create mode 100644 pkg/blobstore/chunklistvalidating/reader_chunker.go create mode 100644 pkg/blobstore/chunklistvalidating/reader_chunker_test.go create mode 100644 pkg/blobstore/cls_read_buffer_factory.go create mode 100644 pkg/blobstore/configuration/cls_blob_access_creator.go create mode 100644 pkg/blobstore/grpcclients/cls_blob_access.go diff --git a/MODULE.bazel b/MODULE.bazel index 3dc48bb5d..8e1d6cfc5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -49,6 +49,7 @@ use_repo( "com_github_aws_aws_sdk_go_v2_service_sts", "com_github_bazelbuild_buildtools", "com_github_bazelbuild_remote_apis", + "com_github_buildbarn_go_cdc", "com_github_buildbarn_go_sha256tree", "com_github_fxtlabs_primes", "com_github_go_jose_go_jose_v3", diff --git a/cmd/bb_storage/main.go b/cmd/bb_storage/main.go index 3faab6b4f..92e3650f0 100644 --- a/cmd/bb_storage/main.go +++ b/cmd/bb_storage/main.go @@ -84,6 +84,27 @@ func main() { contentAddressableStorage = authorizedBackend } + // Chunk List Storage (CLS). + var chunkListStorage blobstore.BlobAccess + if configuration.ChunkListStorage != nil { + info, authorizedBackend, allAuthorizers, err := newScannableBlobAccess( + dependenciesGroup, + configuration.ChunkListStorage, + blobstore_configuration.NewCLSBlobAccessCreator( + contentAddressableStorageInfo, + grpcClientFactory, + int(configuration.MaximumMessageSizeBytes), + ), + grpcClientFactory, + ) + if err != nil { + return util.StatusWrap(err, "Failed to create Chunk List Storage") + } + cacheCapabilitiesProviders = append(cacheCapabilitiesProviders, info.BlobAccess) + cacheCapabilitiesAuthorizers = append(cacheCapabilitiesAuthorizers, allAuthorizers...) + chunkListStorage = authorizedBackend + } + // Action Cache (AC). var actionCache blobstore.BlobAccess if configuration.ActionCache != nil { @@ -193,12 +214,14 @@ func main() { configuration.GrpcServers, func(s grpc.ServiceRegistrar) { if contentAddressableStorage != nil { + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer( + contentAddressableStorage, + chunkListStorage, + configuration.MaximumMessageSizeBytes, + ) remoteexecution.RegisterContentAddressableStorageServer( s, - grpcservers.NewContentAddressableStorageServer( - contentAddressableStorage, - configuration.MaximumMessageSizeBytes, - ), + contentAddressableStorageServer, ) bytestream.RegisterByteStreamServer( s, diff --git a/go.mod b/go.mod index 082763b1a..4fd9fa898 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/bazelbuild/buildtools v0.0.0-20260527135131-3b47c424ecf5 github.com/bazelbuild/remote-apis v0.0.0-20260331222004-becdd8f9ff81 github.com/bazelbuild/rules_go v0.62.0 + github.com/buildbarn/go-cdc v0.0.9 github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b github.com/fxtlabs/primes v0.0.0-20150821004651-dad82d10a449 github.com/go-jose/go-jose/v3 v3.0.5 diff --git a/go.sum b/go.sum index d097ebee7..545e90b57 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,8 @@ github.com/bazelbuild/rules_go v0.62.0/go.mod h1:6YghDRf6l3FSiAwncK+Ww9jE1naoQzN github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/buildbarn/go-cdc v0.0.9 h1:bWfgn92ed8Oo2zZKJdMAfB0APGz7Q8zvnqUn3hPuihM= +github.com/buildbarn/go-cdc v0.0.9/go.mod h1:KUMqSMvoRlby3uak9aKIvgz3KgNqwm2CMUoVX1EDr8k= github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b h1:IKUxixGBm9UxobU7c248z0BF0ojG19uoSLz8MFZM/KA= github.com/buildbarn/go-sha256tree v0.0.0-20250310211320-0f70f20e855b/go.mod h1:e7g3/yWApcg+PpDqd4eQEEV8pexQmfCgK3frP+1Wuvk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/pkg/blobstore/BUILD.bazel b/pkg/blobstore/BUILD.bazel index 218c58a9c..2cd9368ef 100644 --- a/pkg/blobstore/BUILD.bazel +++ b/pkg/blobstore/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "blob_access.go", "blob_access_message_reader.go", "cas_read_buffer_factory.go", + "cls_read_buffer_factory.go", "deadline_enforcing_blob_access.go", "demultiplexing_blob_access.go", "empty_blob_injecting_blob_access.go", diff --git a/pkg/blobstore/chunklistvalidating/BUILD.bazel b/pkg/blobstore/chunklistvalidating/BUILD.bazel new file mode 100644 index 000000000..1bb49c7f0 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "chunklistvalidating", + srcs = [ + "chunk_concatenating_reader.go", + "chunk_list_validating_blob_access.go", + "chunker.go", + "reader_chunker.go", + ], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating", + visibility = ["//visibility:public"], + deps = [ + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/digest", + "//pkg/util", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_buildbarn_go_cdc//:go-cdc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + "@org_golang_x_sync//errgroup", + ], +) + +go_test( + name = "chunklistvalidating_test", + srcs = [ + "chunk_list_validating_blob_access_test.go", + "fake_blob_access_test.go", + "reader_chunker_test.go", + ], + deps = [ + ":chunklistvalidating", + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/digest", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) diff --git a/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go b/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go new file mode 100644 index 000000000..643af130e --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go @@ -0,0 +1,68 @@ +package chunklistvalidating + +import ( + "context" + "io" + + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// chunkConcatenatingReader is a helper utility that implements the +// io.ReadCloser api over a series of digest.Digest objectes fetched +// sequentially from the CAS. +type chunkConcatenatingReader struct { + ctx context.Context + contentAddressableStorage blobstore.BlobAccess + chunkDigests []digest.Digest + currentIndex int + currentReader io.ReadCloser + closed bool +} + +func (r *chunkConcatenatingReader) Read(p []byte) (int, error) { + if r.closed { + return 0, status.Error(codes.Internal, "Reader is already closed") + } + for { + if r.currentReader == nil { + if r.currentIndex >= len(r.chunkDigests) { + return 0, io.EOF + } + chunkDigest := r.chunkDigests[r.currentIndex] + b := r.contentAddressableStorage.Get(r.ctx, chunkDigest) + r.currentReader = b.ToReader() + r.currentIndex++ + } + + n, err := r.currentReader.Read(p) + if n > 0 { + return n, nil + } + if err == io.EOF { + err = r.currentReader.Close() + r.currentReader = nil + if err != nil { + return 0, err + } + continue + } + if err != nil { + _ = r.currentReader.Close() + r.currentReader = nil + return 0, util.StatusWrap(err, "Failed to read chunk") + } + } +} + +func (r *chunkConcatenatingReader) Close() (err error) { + r.closed = true + if r.currentReader != nil { + err = r.currentReader.Close() + r.currentReader = nil + } + return err +} diff --git a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go new file mode 100644 index 000000000..351ea3a0f --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go @@ -0,0 +1,362 @@ +package chunklistvalidating + +import ( + "context" + "io" + "slices" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type chunkListValidatingBlobAccess struct { + blobstore.BlobAccess + contentAddressableStorage blobstore.BlobAccess + maximumMessageSizeBytes int +} + +// NewChunkListValidatingBlobAccess creates a wrapper around a Chunk +// List Storage (CLS) that ensures only valid chunk lists are stored in +// the CLS. A valid chunk list is a chunk list which follows the +// chunking parameters, has all the chunks present in the Content +// Addressable Storage (CAS) and where the chunks concatenate into the +// appropriate digest. +// +// This validation is fairly expensive and validation should only be +// done at a single layer as close as possible to the CAS where the full +// view of the CAS is available. +func NewChunkListValidatingBlobAccess(chunkListStorage, contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int) blobstore.BlobAccess { + return &chunkListValidatingBlobAccess{ + BlobAccess: chunkListStorage, + contentAddressableStorage: contentAddressableStorage, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +// Fetch the chunking parameters from the GetCapabilities +// implementation. +func (ba *chunkListValidatingBlobAccess) getValidChunkingParameters(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.RepMaxCdcParams, error) { + capabilities, err := ba.BlobAccess.GetCapabilities(ctx, instanceName) + if err != nil { + return nil, util.StatusWrap(err, "Unable to GetCapabilities to determine chunking parameters") + } + + params := capabilities.CacheCapabilities.GetRepMaxCdcParams() + if params == nil { + return nil, status.Error(codes.Unimplemented, "This backend only supports upstream servers with RepMaxCDC support") + } + if params.MinChunkSizeBytes < 64 { + return nil, status.Errorf(codes.Internal, "RepMaxCDC minimum chunk size was %d bytes but a minimum of 64 bytes is required", params.MinChunkSizeBytes) + } + + return params, nil +} + +// Check the downstream blob access if this particular blob has already +// been split. If that's the case and all the chunks are still there we +// can return early. In case of errors we will return nil and continue +// with the regular code path. +func (ba *chunkListValidatingBlobAccess) checkSplitResult(ctx context.Context, d digest.Digest) buffer.Buffer { + b1, b2 := ba.BlobAccess.Get(ctx, d).CloneCopy(ba.maximumMessageSizeBytes) + responseMsg, err := b1.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + b2.Discard() + return nil + } + + splitBlobResponse := responseMsg.(*remoteexecution.SplitBlobResponse) + digestFunction := d.GetDigestFunction() + digestSetBuilder := digest.NewSetBuilder(len(splitBlobResponse.ChunkDigests)) + digestSetBuilder.Add(d) + + for _, chunkDigestProto := range splitBlobResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) + if err != nil { + b2.Discard() + return nil + } + digestSetBuilder.Add(chunkDigest) + } + + missing, err := ba.contentAddressableStorage.FindMissing(ctx, digestSetBuilder.Build()) + if err == nil && missing.Empty() { + return b2 + } + b2.Discard() + return nil +} + +// Get returns a valid SplitResult for the given digest chunking the +// blob and storing the chunk list if needed. +func (ba *chunkListValidatingBlobAccess) Get(ctx context.Context, d digest.Digest) buffer.Buffer { + params, err := ba.getValidChunkingParameters(ctx, d.GetInstanceName()) + if err != nil { + return buffer.NewBufferFromError(err) + } + + // Check for the trivial case, the blob is small enough that it will + // always decompose to a single chunk of the same size as the + // original blob. We verify the existence of the blob in CAS and + // break out early. + blobSize := d.GetSizeBytes() + if uint64(blobSize) < 2*params.MinChunkSizeBytes { + missing, err := ba.contentAddressableStorage.FindMissing(ctx, d.ToSingletonSet()) + if err != nil { + return buffer.NewBufferFromError(util.StatusWrap(err, "Failed to verify blob existence")) + } + if !missing.Empty() { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found in CAS")) + } + + response := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{d.GetProto()}, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + + return buffer.NewProtoBufferFromProto(response, buffer.UserProvided) + } + + // Check if we have already computed the result for this blob. + if result := ba.checkSplitResult(ctx, d); result != nil { + return result + } + + // Fallthrough case, compute the chunk list, upload the chunks and + // store the chunk list. + blobReader := ba.contentAddressableStorage.Get(ctx, d).ToReader() + defer blobReader.Close() + chunker := NewReaderChunker(d.GetDigestFunction(), blobReader, int64(params.MinChunkSizeBytes), int64(params.HorizonSizeBytes)) + + chunkDigests := make([]*remoteexecution.Digest, 0, uint64(blobSize)/params.MinChunkSizeBytes+1) + + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + if err != nil { + return buffer.NewBufferFromError(err) + } + + missing, err := ba.contentAddressableStorage.FindMissing(ctx, chunk.Digest.ToSingletonSet()) + if err != nil { + return buffer.NewBufferFromError(err) + } + if !missing.Empty() { + if err := ba.contentAddressableStorage.Put(ctx, chunk.Digest, buffer.NewValidatedBufferFromByteSlice(chunk.Data)); err != nil { + return buffer.NewBufferFromError(err) + } + } + + chunkDigests = append(chunkDigests, chunk.Digest.GetProto()) + } + + response := &remoteexecution.SplitBlobResponse{ + ChunkDigests: chunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + + b1, b2 := buffer.NewProtoBufferFromProto(response, buffer.UserProvided).CloneCopy(ba.maximumMessageSizeBytes) + + if err := ba.BlobAccess.Put(ctx, d, b1); err != nil { + b2.Discard() + return buffer.NewBufferFromError(util.StatusWrap(err, "Failed to store the split blob response")) + } + + return b2 +} + +func (ba *chunkListValidatingBlobAccess) matchesStoredChunkList(ctx context.Context, d digest.Digest, userResponse *remoteexecution.SplitBlobResponse) bool { + existingMsg, err := ba.BlobAccess.Get(ctx, d).ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + return false + } + + cachedResponse := existingMsg.(*remoteexecution.SplitBlobResponse) + return slices.EqualFunc(cachedResponse.ChunkDigests, userResponse.ChunkDigests, + func(c, u *remoteexecution.Digest) bool { + return c.Hash == u.Hash && c.SizeBytes == u.SizeBytes + }) +} + +func (ba *chunkListValidatingBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { + // Parse the buffer as a SplitBlobResponse + msg, err := b.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + return util.StatusWrap(err, "Failed to parse input as SplitBlobResponse") + } + userResponse := msg.(*remoteexecution.SplitBlobResponse) + + digestFunction := d.GetDigestFunction() + var userChunks []digest.Digest + digestSetBuilder := digest.NewSetBuilder(len(userResponse.ChunkDigests)) + for _, chunkDigestProto := range userResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) + if err != nil { + return status.Errorf(codes.InvalidArgument, "Invalid chunk digest: %v", err) + } + digestSetBuilder.Add(chunkDigest) + userChunks = append(userChunks, chunkDigest) + } + + // Check that all referenced chunks are present in storage. + missing, err := ba.contentAddressableStorage.FindMissing(ctx, digestSetBuilder.Build()) + if err != nil { + return util.StatusWrap(err, "Failed to check existence of chunks") + } + if !missing.Empty() { + return status.Error(codes.NotFound, "At least one chunk in the chunk list was not found") + } + + // Check the trivial cases without hitting the downstream blob + // stores. + + // No chunks given, blob must be the empty blob. + if len(userChunks) == 0 { + if d.GetSizeBytes() != 0 { + return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") + } + if d.GetDigestFunction().NewGenerator(0).Sum() != d { + return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") + } + return nil + } + // Single chunk given, the blob must be equal to the chunk. At this + // point we have already verified the presence of the chunk so we do + // not have to verify the presence of the blob. + if len(userChunks) == 1 { + if d != userChunks[0] { + return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") + } + return nil + } + + chunksMatchesStoredLists := ba.matchesStoredChunkList(ctx, d, userResponse) + missing, err = ba.contentAddressableStorage.FindMissing(ctx, d.ToSingletonSet()) + if err != nil { + return util.StatusWrap(err, "Failed to check existence of blob") + } + blobExistsInCAS := missing.Empty() + + // The request is identical to an already existing chunk list with + // content we have verified exists in CAS. + if blobExistsInCAS && chunksMatchesStoredLists { + return nil + } + + // No more shortcuts available go through the heavy path of + // concatenating/verifying and chunking the blobs. + params, err := ba.getValidChunkingParameters(ctx, d.GetInstanceName()) + if err != nil { + return err + } + + reader := &chunkConcatenatingReader{ + ctx: ctx, + contentAddressableStorage: ba.contentAddressableStorage, + chunkDigests: userChunks, + } + + blobBuffer := buffer.NewCASBufferFromReader(d, reader, buffer.UserProvided) + b1, b2 := blobBuffer.CloneStream() + + // Stream 1: Uploads the blob to CAS. + group, gCtx := errgroup.WithContext(ctx) + group.Go(func() error { + if blobExistsInCAS { + // Upload unnecessary, blob already exists in CAS. + b1.Discard() + return nil + } + return ba.contentAddressableStorage.Put(gCtx, d, b1) + }) + + // Stream 2: Chunk the stream to compute the digest and cache the + // canonical chunks. + var canonicalChunkDigests []*remoteexecution.Digest + group.Go(func() error { + b2Reader := b2.ToReader() + defer b2Reader.Close() + chunker := NewReaderChunker(d.GetDigestFunction(), b2Reader, int64(params.MinChunkSizeBytes), int64(params.HorizonSizeBytes)) + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + missing, err := ba.contentAddressableStorage.FindMissing(gCtx, chunk.Digest.ToSingletonSet()) + if err != nil { + return err + } + if !missing.Empty() { + if err := ba.contentAddressableStorage.Put(gCtx, chunk.Digest, buffer.NewValidatedBufferFromByteSlice(chunk.Data)); err != nil { + return util.StatusWrap(err, "Failed to save chunk") + } + } + canonicalChunkDigests = append(canonicalChunkDigests, chunk.Digest.GetProto()) + } + }) + + // Wait for the full blob validation and upload to complete. + if err := group.Wait(); err != nil { + return util.StatusWrap(err, "Failed to splice the blob") + } + + // Store the canonical response. + canonicalResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: canonicalChunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + canonicalBuffer := buffer.NewProtoBufferFromProto(canonicalResponse, buffer.UserProvided) + if err := ba.BlobAccess.Put(ctx, d, canonicalBuffer); err != nil { + return util.StatusWrap(err, "Failed to save canonical chunk list") + } + return nil +} + +func (ba *chunkListValidatingBlobAccess) findMissingChunks(ctx context.Context, d digest.Digest) (digest.Set, error) { + splitBlobResponseProto, err := ba.BlobAccess.Get(ctx, d).ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + return digest.EmptySet, err + } + splitBlobResponse := splitBlobResponseProto.(*remoteexecution.SplitBlobResponse) + digestFunction := d.GetDigestFunction() + builder := digest.NewSetBuilder(len(splitBlobResponse.ChunkDigests)) + for _, chunkDigestProto := range splitBlobResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) + if err != nil { + return digest.EmptySet, util.StatusWrapf(err, "Invalid chunk digest %#v", chunkDigestProto) + } + builder.Add(chunkDigest) + } + return ba.contentAddressableStorage.FindMissing(ctx, builder.Build()) +} + +func (ba *chunkListValidatingBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + missingBlobs, err := ba.BlobAccess.FindMissing(ctx, digests) + if err != nil { + return digest.EmptySet, err + } + nonMissingBlobs, _, _ := digest.GetDifferenceAndIntersection(digests, missingBlobs) + missings := make([]digest.Set, 1, 1+nonMissingBlobs.Length()) + missings[0] = missingBlobs + for _, d := range nonMissingBlobs.Items() { + missingChunks, err := ba.findMissingChunks(ctx, d) + if err != nil { + return digest.EmptySet, err + } + if !missingChunks.Empty() { + missings = append(missings, d.ToSingletonSet()) + } + } + return digest.GetUnion(missings), nil +} diff --git a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go new file mode 100644 index 000000000..3830e720b --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go @@ -0,0 +1,410 @@ +package chunklistvalidating_test + +import ( + "bytes" + "context" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// mustComputeDigest is a test helper to easily generate digests from +// byte slices. +func mustComputeDigest(t *testing.T, digestFunction digest.Function, data []byte) digest.Digest { + t.Helper() + generator := digestFunction.NewGenerator(int64(len(data))) + _, err := generator.Write(data) + require.NoError(t, err) + return generator.Sum() +} + +var testCDCParams = &remoteexecution.RepMaxCdcParams{ + MinChunkSizeBytes: 1024, + HorizonSizeBytes: 8 * 1024, +} +var maximumMessageSizeBytes = 1024 * 1024 + +func TestChunkListValidatingBlobAccessGetTrivialSmallBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + blobData := []byte("Small trivial blob") + blobDigest := mustComputeDigest(t, digestFunction, blobData) + + require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) + + fakeCAS.ResetTouches() + msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + + splitResponse := msg.(*remoteexecution.SplitBlobResponse) + require.Len(t, splitResponse.ChunkDigests, 1) + require.Equal(t, blobDigest.GetProto().Hash, splitResponse.ChunkDigests[0].Hash) + require.Greater(t, fakeCAS.GetTouches(blobDigest), 0, "Blob did not have its lifetime renewed.") +} + +func TestChunkListValidatingBlobAccessGetLargeBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + blobData := bytes.Repeat([]byte("test_data_pattern_"), 6000) + blobDigest := mustComputeDigest(t, digestFunction, blobData) + + require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) + + msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + + splitResponse := msg.(*remoteexecution.SplitBlobResponse) + require.Greater(t, len(splitResponse.ChunkDigests), 1, "Blob should have been divided into multiple chunks.") + + for _, chunkProto := range splitResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkProto) + require.NoError(t, err) + + require.Greater(t, fakeCAS.GetTouches(chunkDigest), 0, "Chunk generated by CDC did not have its lifetime renewed.") + missing, err := fakeCAS.FindMissing(ctx, chunkDigest.ToSingletonSet()) + require.NoError(t, err) + require.True(t, missing.Empty(), "Chunk generated by CDC was not saved to the CAS.") + } + + cachedMsg, err := fakeCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + require.Equal(t, len(splitResponse.ChunkDigests), len(cachedMsg.(*remoteexecution.SplitBlobResponse).ChunkDigests)) +} + +func TestChunkListValidatingBlobAccessGetExtendsLifetimes(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + blobData := bytes.Repeat([]byte("test_data_pattern_"), 6000) // ~108KB + blobDigest := mustComputeDigest(t, digestFunction, blobData) + require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) + + // Split the blob to populate the CAS and CLS for this blob. + msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + splitResponse := msg.(*remoteexecution.SplitBlobResponse) + + // Reset touches. + fakeCAS.ResetTouches() + + // Perform a cached split. + msgCached, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + cachedResponse := msgCached.(*remoteexecution.SplitBlobResponse) + require.Equal(t, len(splitResponse.ChunkDigests), len(cachedResponse.ChunkDigests)) + + // The original blob MUST have had its lifetime extended + require.Greater(t, fakeCAS.GetTouches(blobDigest), 0, "Original blob's lifetime was not extended during call to SplitBlob") + require.Greater(t, fakeCLS.GetTouches(blobDigest), 0, "Original blob's chunk list lifetime was not extended during call to SplitBlob") + + // Every chunk MUST have been touched in the CAS + for _, chunkProto := range cachedResponse.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkProto) + require.NoError(t, err) + require.Greater(t, fakeCAS.GetTouches(chunkDigest), 0, "Chunk's lifetime was not extended during call to SplitBlob") + } +} + +func TestChunkListValidatingBlobAccessPutManualSplice(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunk1Data := []byte("Hello, ") + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Data := []byte("World!") + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + expectedFullData := []byte("Hello, World!") + fullBlobDigest := mustComputeDigest(t, digestFunction, expectedFullData) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + chunk1Digest.GetProto(), + chunk2Digest.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) + require.NoError(t, err) + + composedData, err := fakeCAS.Get(ctx, fullBlobDigest).ToByteSlice(len(expectedFullData)) + require.NoError(t, err) + require.Equal(t, expectedFullData, composedData) +} + +func TestChunkListValidatingBlobAccessPutCanonicalization(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + blobData := bytes.Repeat([]byte("canonicalization_test_data"), 4000) // ~104KB + chunk1Data := blobData[:10] + chunk2Data := blobData[10:] + + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + fullBlobDigest := mustComputeDigest(t, digestFunction, blobData) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + chunk1Digest.GetProto(), + chunk2Digest.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) + require.NoError(t, err) + + composedData, err := fakeCAS.Get(ctx, fullBlobDigest).ToByteSlice(200000) + require.NoError(t, err) + require.Equal(t, blobData, composedData) + + canonicalBuffer := fakeCLS.Get(ctx, fullBlobDigest) + canonicalProto, err := canonicalBuffer.ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + require.NoError(t, err) + + canonicalResp := canonicalProto.(*remoteexecution.SplitBlobResponse) + require.Greater(t, len(canonicalResp.ChunkDigests), 0) + require.NotEqual(t, chunk1Digest.GetProto().Hash, canonicalResp.ChunkDigests[0].Hash, "Server should not have echoed back the non-standard chunks") +} + +func TestChunkListValidatingBlobAccessPutMissingChunk(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + chunkDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{chunkDigest.GetProto()}, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, chunkDigest, reqBuffer) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) +} + +func TestChunkListValidatingBlobAccessPutDigestMismatch(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkData := []byte("Valid chunk data") + chunkDigest := mustComputeDigest(t, digestFunction, chunkData) + require.NoError(t, fakeCAS.Put(ctx, chunkDigest, buffer.NewValidatedBufferFromByteSlice(chunkData))) + + wrongBlobDigest := mustComputeDigest(t, digestFunction, []byte("Different data")) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{chunkDigest.GetProto()}, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, wrongBlobDigest, reqBuffer) + require.Error(t, err) + require.Contains(t, err.Error(), "does not compose to blob") +} + +func TestChunkListValidatingBlobAccessPutEmptyBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + emptyDigest := mustComputeDigest(t, digestFunction, nil) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{}, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, emptyDigest, reqBuffer) + require.NoError(t, err) +} + +func TestChunkListValidatingBlobAccessPutRepeatedChunks(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkA := []byte("A") + digestA := mustComputeDigest(t, digestFunction, chunkA) + require.NoError(t, fakeCAS.Put(ctx, digestA, buffer.NewValidatedBufferFromByteSlice(chunkA))) + + chunkB := []byte("B") + digestB := mustComputeDigest(t, digestFunction, chunkB) + require.NoError(t, fakeCAS.Put(ctx, digestB, buffer.NewValidatedBufferFromByteSlice(chunkB))) + + expectedData := []byte("AABA") + expectedDigest := mustComputeDigest(t, digestFunction, expectedData) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + digestA.GetProto(), + digestA.GetProto(), + digestB.GetProto(), + digestA.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, expectedDigest, reqBuffer) + require.NoError(t, err) + + composedData, err := fakeCAS.Get(ctx, expectedDigest).ToByteSlice(100) + require.NoError(t, err) + require.Equal(t, expectedData, composedData) +} + +func TestChunkListValidatingBlobAccessPutInlineEmptyChunk(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunkData := []byte("Valid") + chunkDigest := mustComputeDigest(t, digestFunction, chunkData) + require.NoError(t, fakeCAS.Put(ctx, chunkDigest, buffer.NewValidatedBufferFromByteSlice(chunkData))) + + emptyDigest := mustComputeDigest(t, digestFunction, nil) + require.NoError(t, fakeCAS.Put(ctx, emptyDigest, buffer.NewValidatedBufferFromByteSlice(nil))) + + expectedDigest := mustComputeDigest(t, digestFunction, chunkData) + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + chunkDigest.GetProto(), + emptyDigest.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, expectedDigest, reqBuffer) + require.NoError(t, err) +} + +func TestChunkListValidatingBlobAccessPutExtendsLifetimes(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + + chunk1Data := []byte("Hello, ") + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + + chunk2Data := []byte("World!") + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + + expectedFullData := []byte("Hello, World!") + fullBlobDigest := mustComputeDigest(t, digestFunction, expectedFullData) + + fakeCAS.ResetTouches() + + splitResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + chunk1Digest.GetProto(), + chunk2Digest.GetProto(), + }, + } + reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) + + err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) + + // From the REAPI, the server may either process the splice and + // return OK, OR it may return ALREADY_EXISTS if the blob is already + // composed and the server chooses not to extend the lifetime of the + // user's specific chunks. + if status.Code(err) == codes.AlreadyExists { + // The server is free not to touch the user's chunks. However, + // it MUST still have verified/touched the original blob. + require.Greater(t, fakeCAS.GetTouches(fullBlobDigest), 0, "Composed blob lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCLS.GetTouches(fullBlobDigest), 0, "Composed blob chunk list lifetime was not extended during SpliceBlob") + } else { + // Because the server accepted the Splice request, it is + // strictly obligated to extend the lifetimes of BOTH the + // provided chunks and the composed blob. + require.NoError(t, err) + + require.Greater(t, fakeCAS.GetTouches(chunk1Digest), 0, "Chunk 1 lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCAS.GetTouches(chunk2Digest), 0, "Chunk 2 lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCAS.GetTouches(fullBlobDigest), 0, "Composed blob lifetime was not extended during SpliceBlob") + require.Greater(t, fakeCLS.GetTouches(fullBlobDigest), 0, "Composed blob chunk list lifetime was not extended during SpliceBlob") + } +} + +func TestChunkListValidatingBlobAccessGetMissingBlob(t *testing.T) { + ctx := context.Background() + + fakeCAS := newFakeBlobAccess(nil) + fakeCLS := newFakeBlobAccess(testCDCParams) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, 1024*1024) + + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + ghostDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) + + _, err := validatingCLS.Get(ctx, ghostDigest).ToProto(&remoteexecution.SplitBlobResponse{}, 1024*1024) + + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) +} diff --git a/pkg/blobstore/chunklistvalidating/chunker.go b/pkg/blobstore/chunklistvalidating/chunker.go new file mode 100644 index 000000000..8badbc7e9 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/chunker.go @@ -0,0 +1,16 @@ +package chunklistvalidating + +import ( + "github.com/buildbarn/bb-storage/pkg/digest" +) + +// Chunk is a struct of raw binary data with its digest. +type Chunk struct { + Digest digest.Digest + Data []byte +} + +// Chunker is an interface that provides a sequence of chunks. +type Chunker interface { + NextChunk() (Chunk, error) +} diff --git a/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go b/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go new file mode 100644 index 000000000..ffd7922c1 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/fake_blob_access_test.go @@ -0,0 +1,88 @@ +package chunklistvalidating_test + +import ( + "context" + "sync" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// fakeBlobAccess provides a thread-safe, in-memory BlobAccess for +// testing. +type fakeBlobAccess struct { + blobstore.BlobAccess + lock sync.Mutex + blobs map[digest.Digest][]byte + touches map[digest.Digest]int // Tracks lifetime extensions + chunkingParameters *remoteexecution.RepMaxCdcParams +} + +func newFakeBlobAccess(chunkingParameters *remoteexecution.RepMaxCdcParams) *fakeBlobAccess { + return &fakeBlobAccess{ + blobs: make(map[digest.Digest][]byte), + touches: make(map[digest.Digest]int), + chunkingParameters: chunkingParameters, + } +} + +func (f *fakeBlobAccess) Get(ctx context.Context, d digest.Digest) buffer.Buffer { + f.lock.Lock() + defer f.lock.Unlock() + data, ok := f.blobs[d] + if !ok { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found")) + } + return buffer.NewValidatedBufferFromByteSlice(data) +} + +func (f *fakeBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { + data, err := b.ToByteSlice(100 * 1024 * 1024) + if err != nil { + return err + } + f.lock.Lock() + defer f.lock.Unlock() + f.touches[d]++ + f.blobs[d] = data + return nil +} + +func (f *fakeBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + f.lock.Lock() + defer f.lock.Unlock() + missing := digest.NewSetBuilder(digests.Length()) + for _, d := range digests.Items() { + if _, ok := f.blobs[d]; !ok { + missing.Add(d) + } else { + f.touches[d]++ + } + } + return missing.Build(), nil +} + +func (f *fakeBlobAccess) GetCapabilities(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.ServerCapabilities, error) { + return &remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + RepMaxCdcParams: f.chunkingParameters, + }, + }, nil +} + +func (f *fakeBlobAccess) GetTouches(d digest.Digest) int { + f.lock.Lock() + defer f.lock.Unlock() + return f.touches[d] +} + +func (f *fakeBlobAccess) ResetTouches() { + f.lock.Lock() + defer f.lock.Unlock() + f.touches = make(map[digest.Digest]int) +} diff --git a/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel b/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel new file mode 100644 index 000000000..ca423091e --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "integration_test", + srcs = ["chunk_list_validating_integration_test.go"], + data = ["//cmd/bb_storage"], + env = { + "BB_STORAGE_RUNFILE_PATH": "$(rlocationpath //cmd/bb_storage:bb_storage)", + }, + deps = [ + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//credentials/insecure", + "@org_golang_google_grpc//status", + "@rules_go//go/runfiles", + ], +) diff --git a/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go b/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go new file mode 100644 index 000000000..033ec595c --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go @@ -0,0 +1,445 @@ +package integration + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "math/rand" + "os" + "os/exec" + "strings" + "testing" + "time" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/stretchr/testify/require" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +type serverParams struct { + disableCLS bool + socketPath string + upstreamSocketPath string +} + +func escapeJSON(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, "\"", "\\\"") + s = strings.ReplaceAll(s, "'", "\\'") + s = strings.ReplaceAll(s, "\n", "\\n") + return s +} + +func storageConfig(params serverParams) string { + return fmt.Sprintf(` +local cls = %t; +local listenPath = '%s'; +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { allow: {} }, + }], + maximumMessageSizeBytes: 4 * 1024 * 1024, + contentAddressableStorage: { + backend: { + 'local': { + keyLocationMapInMemory: { entries: 1024 }, + keyLocationMapMaximumGetAttempts: 32, + keyLocationMapMaximumPutAttempts: 64, + oldBlocks: 1, + currentBlocks: 1, + newBlocks: 1, + blocksInMemory: { blockSizeBytes: 1024 * 1024 }, + }, + }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, + chunkListStorage: if !cls then null else { + backend: { + 'local': { + keyLocationMapInMemory: { entries: 1024 }, + keyLocationMapMaximumGetAttempts: 32, + keyLocationMapMaximumPutAttempts: 64, + oldBlocks: 1, + currentBlocks: 1, + newBlocks: 1, + blocksInMemory: { blockSizeBytes: 1024 * 1024 }, + chunkingParameters: { + minChunkSizeBytes: 256, + horizonSizeBytes: 8*256, + } + }, + }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, +} +`, !params.disableCLS, escapeJSON(params.socketPath)) +} + +func frontendConfig(params serverParams) string { + return fmt.Sprintf(` +local cls = %t; +local listenPath = '%s'; +// unix:// doesn't work under Windows. +// https://github.com/grpc/grpc-go/issues/8675 +local upstreamAddress = 'unix:%s'; +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { allow: {} }, + }], + maximumMessageSizeBytes: 4 * 1024 * 1024, + contentAddressableStorage: { + backend: { grpc: { client: { address: upstreamAddress } } }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, + chunkListStorage: if !cls then null else { + backend: { chunkListValidating: { backend: { grpc: { client: { address: upstreamAddress } } } } }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, +} +`, !params.disableCLS, escapeJSON(params.socketPath), escapeJSON(params.upstreamSocketPath)) +} + +func writeConfigFile(name, content string) (file *os.File, err error) { + if file, err = os.CreateTemp("", name); err != nil { + return nil, err + } + if _, err = file.WriteString(content); err != nil { + return nil, err + } + if err = file.Close(); err != nil { + return nil, err + } + return file, nil +} + +func setupServer(t *testing.T, name, config string) func() { + rf, err := runfiles.New() + if err != nil { + t.Fatalf("Failed to initialize runfiles: %v", err) + } + runfilePath := os.Getenv("BB_STORAGE_RUNFILE_PATH") + require.NotEmpty(t, runfilePath, "BB_STORAGE_RUNFILE_PATH environment variable is not set") + + bbStoragePath, err := rf.Rlocation(runfilePath) + require.NoError(t, err) + + configFile, err := writeConfigFile(name, config) + require.NoError(t, err) + + cmd := exec.Command(bbStoragePath, configFile.Name()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Start() + require.NoError(t, err) + + return func() { + cmd.Process.Kill() + cmd.Wait() + os.Remove(configFile.Name()) + } +} + +func createSocketPath(t *testing.T) string { + t.Helper() + socketFile, err := os.CreateTemp("", "bb_*.sock") + require.NoError(t, err) + socketPath := socketFile.Name() + socketFile.Close() + err = os.Remove(socketPath) + require.NoError(t, err) + return socketPath +} + +func setupServers(t *testing.T, storageParams, frontendParams serverParams) (func(), remoteexecution.CapabilitiesClient, remoteexecution.ContentAddressableStorageClient) { + storageParams.socketPath = createSocketPath(t) + closeStorage := setupServer(t, "storage", storageConfig(storageParams)) + require.Eventually(t, func() bool { + _, err := os.Stat(storageParams.socketPath) + return err == nil + }, 1*time.Second, 10*time.Millisecond, "Storage server did not start.") + + frontendParams.socketPath = createSocketPath(t) + frontendParams.upstreamSocketPath = storageParams.socketPath + closeFrontend := setupServer(t, "frontend", frontendConfig(frontendParams)) + require.Eventually(t, func() bool { + _, err := os.Stat(frontendParams.socketPath) + return err == nil + }, 1*time.Second, 10*time.Millisecond, "Frontend server did not start.") + + conn, err := grpc.NewClient(fmt.Sprintf("unix:%s", frontendParams.socketPath), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + + return func() { + closeStorage() + closeFrontend() + conn.Close() + os.Remove(storageParams.socketPath) + os.Remove(frontendParams.socketPath) + }, remoteexecution.NewCapabilitiesClient(conn), remoteexecution.NewContentAddressableStorageClient(conn) +} + +func TestChunkListValidatingCapabilities(t *testing.T) { + tests := []struct { + name string + storageParams serverParams + frontendParams serverParams + expectSupport bool + }{ + {"Enabled In Both", serverParams{}, serverParams{}, true}, + {"Disabled in Storage", serverParams{disableCLS: true}, serverParams{}, false}, + {"Disabled in Frontend", serverParams{}, serverParams{disableCLS: true}, false}, + {"Disabled in Both", serverParams{disableCLS: true}, serverParams{disableCLS: true}, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + closer, capabilitiesClient, _ := setupServers(t, tc.storageParams, tc.frontendParams) + defer closer() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + capabilities, err := capabilitiesClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ + InstanceName: "", + }) + require.NoError(t, err) + + cacheCaps := capabilities.CacheCapabilities + require.NotNil(t, cacheCaps) + + if tc.expectSupport { + require.True(t, cacheCaps.SpliceBlobSupport) + require.True(t, cacheCaps.SplitBlobSupport) + + chunkingParameters := cacheCaps.GetRepMaxCdcParams() + require.NotNil(t, chunkingParameters) + require.Equal(t, uint64(256), chunkingParameters.GetMinChunkSizeBytes()) + require.Equal(t, uint64(2048), chunkingParameters.GetHorizonSizeBytes()) + } else { + require.False(t, cacheCaps.SpliceBlobSupport) + require.False(t, cacheCaps.SplitBlobSupport) + require.Nil(t, cacheCaps.GetRepMaxCdcParams()) + } + }) + } +} + +func computeDigest(data []byte) *remoteexecution.Digest { + hash := sha256.Sum256(data) + return &remoteexecution.Digest{ + Hash: hex.EncodeToString(hash[:]), + SizeBytes: int64(len(data)), + } +} + +func makeRandomData(t *testing.T, size int, seed int64) []byte { + t.Helper() + data := make([]byte, size) + r := rand.New(rand.NewSource(seed)) + _, err := r.Read(data) + require.NoError(t, err) + return data +} + +func uploadBlob(ctx context.Context, t *testing.T, cas remoteexecution.ContentAddressableStorageClient, data []byte) *remoteexecution.Digest { + t.Helper() + digest := computeDigest(data) + req := &remoteexecution.BatchUpdateBlobsRequest{ + Requests: []*remoteexecution.BatchUpdateBlobsRequest_Request{ + {Digest: digest, Data: data}, + }, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + res, err := cas.BatchUpdateBlobs(ctx, req) + require.NoError(t, err) + require.NotEmpty(t, res.Responses, "server returned empty responses array") + status := res.Responses[0].GetStatus() + require.Equal(t, int32(0), status.GetCode(), status.GetMessage()) + return digest +} + +func findMissingBlobs(ctx context.Context, t *testing.T, cas remoteexecution.ContentAddressableStorageClient, digests []*remoteexecution.Digest) []*remoteexecution.Digest { + t.Helper() + req := &remoteexecution.FindMissingBlobsRequest{ + BlobDigests: digests, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + res, err := cas.FindMissingBlobs(ctx, req) + require.NoError(t, err) + return res.MissingBlobDigests +} + +func TestRepMaxCDCSplitAndSpliceBehaviors(t *testing.T) { + minChunkSize := int64(256) + + t.Run("RoundTripSplitThenSplice", func(t *testing.T) { + closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) + defer closer() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + dataSize := (minChunkSize * 4) + 128 + data := makeRandomData(t, int(dataSize), 0) + blobDigest := uploadBlob(ctx, t, casClient, data) + + splitReq := &remoteexecution.SplitBlobRequest{ + BlobDigest: blobDigest, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + splitRes, err := casClient.SplitBlob(ctx, splitReq) + require.NoError(t, err) + + spliceReq := &remoteexecution.SpliceBlobRequest{ + BlobDigest: blobDigest, + ChunkDigests: splitRes.ChunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) + require.NoError(t, err) + require.Equal(t, blobDigest.Hash, spliceRes.BlobDigest.Hash) + }) + + t.Run("SpliceNonStandardChunkingThenSplit", func(t *testing.T) { + closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) + defer closer() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := makeRandomData(t, int(minChunkSize*2), 0) + + chunk1 := blobData[:1] + chunk2 := blobData[1:] + + digest1 := uploadBlob(ctx, t, casClient, chunk1) + digest2 := uploadBlob(ctx, t, casClient, chunk2) + expectedDigest := computeDigest(blobData) + + spliceReq := &remoteexecution.SpliceBlobRequest{ + BlobDigest: expectedDigest, + ChunkDigests: []*remoteexecution.Digest{digest1, digest2}, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) + require.NoError(t, err) + require.Equal(t, expectedDigest.Hash, spliceRes.BlobDigest.Hash) + + splitReq := &remoteexecution.SplitBlobRequest{ + BlobDigest: expectedDigest, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + splitRes, err := casClient.SplitBlob(ctx, splitReq) + require.NoError(t, err) + + require.NotEmpty(t, splitRes.ChunkDigests) + + // Check that it didn't just echo our chunks back + isEcho := len(splitRes.ChunkDigests) == 2 && + splitRes.ChunkDigests[0].Hash == digest1.Hash && + splitRes.ChunkDigests[1].Hash == digest2.Hash + require.False(t, isEcho, "Server echoed non-standard chunks") + + var totalSize int64 + for _, c := range splitRes.ChunkDigests { + totalSize += c.SizeBytes + } + require.Equal(t, expectedDigest.SizeBytes, totalSize) + }) + + t.Run("SpliceAlreadyExistsOrNoop", func(t *testing.T) { + closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) + defer closer() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := append([]byte("This blob will be fully uploaded before we try to splice it."), makeRandomData(t, 16, 0)...) + expectedDigest := uploadBlob(ctx, t, casClient, blobData) + + chunk1 := blobData[:10] + chunk2 := blobData[10:] + digest1 := uploadBlob(ctx, t, casClient, chunk1) + digest2 := uploadBlob(ctx, t, casClient, chunk2) + + spliceReq := &remoteexecution.SpliceBlobRequest{ + BlobDigest: expectedDigest, + ChunkDigests: []*remoteexecution.Digest{digest1, digest2}, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) + + if err != nil { + require.Equal(t, codes.AlreadyExists, status.Code(err), "Expected OK or ALREADY_EXISTS") + } else { + require.Equal(t, expectedDigest.Hash, spliceRes.BlobDigest.Hash) + } + }) + + t.Run("ValidationSpliceBlobRejections", func(t *testing.T) { + closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) + defer closer() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + validData := makeRandomData(t, 512, 0) + validDigest := uploadBlob(ctx, t, casClient, validData) + ghostDigest := computeDigest([]byte("I do not exist")) + + tests := []struct { + name string + req *remoteexecution.SpliceBlobRequest + expectError codes.Code + }{ + { + name: "Missing Chunk", + req: &remoteexecution.SpliceBlobRequest{ + BlobDigest: ghostDigest, + ChunkDigests: []*remoteexecution.Digest{ghostDigest}, + }, + expectError: codes.NotFound, + }, + { + name: "Digest Mismatch", + req: &remoteexecution.SpliceBlobRequest{ + BlobDigest: computeDigest([]byte("Fake target")), + ChunkDigests: []*remoteexecution.Digest{validDigest}, + }, + expectError: codes.InvalidArgument, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.req.ChunkingFunction = remoteexecution.ChunkingFunction_REP_MAX_CDC + tc.req.DigestFunction = remoteexecution.DigestFunction_SHA256 + + _, err := casClient.SpliceBlob(ctx, tc.req) + require.Error(t, err) + require.Equal(t, tc.expectError, status.Code(err)) + }) + } + }) +} diff --git a/pkg/blobstore/chunklistvalidating/reader_chunker.go b/pkg/blobstore/chunklistvalidating/reader_chunker.go new file mode 100644 index 000000000..0d250dabd --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/reader_chunker.go @@ -0,0 +1,53 @@ +package chunklistvalidating + +import ( + "bufio" + "io" + + "github.com/buildbarn/bb-storage/pkg/digest" + cdc "github.com/buildbarn/go-cdc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type readerChunker struct { + cdcChunker cdc.ContentDefinedChunker + reader io.Reader + digestFunction digest.Function +} + +func (c *readerChunker) NextChunk() (Chunk, error) { + chunk, err := c.cdcChunker.ReadNextChunk() + if err != nil { + return Chunk{}, err + } + + digestGenerator := c.digestFunction.NewGenerator(int64(len(chunk))) + if _, err := digestGenerator.Write(chunk); err != nil { + return Chunk{}, status.Error(codes.Internal, "Could not compute digest of chunk") + } + chunkDigest := digestGenerator.Sum() + + return Chunk{ + Data: chunk, + Digest: chunkDigest, + }, nil +} + +// NewReaderChunker creates a chunker that reads from an io.Reader +func NewReaderChunker(digestFunction digest.Function, reader io.Reader, minChunkSizeBytes, horizonSizeBytes int64) Chunker { + // The internal RepMaxContentDefinedChunker may peek up to this many + // bytes. We therefore make sure that the underlying buffer is big + // enough to prevent bufio.ErrBufferFull errors. + bufferSizeBytes := 2*minChunkSizeBytes + horizonSizeBytes + return &readerChunker{ + cdc.NewRepMaxContentDefinedChunker( + bufio.NewReaderSize(reader, int(bufferSizeBytes)), + &cdc.FastContentDefinedChunkerGearTable, + int(minChunkSizeBytes), + int(horizonSizeBytes), + ), + reader, + digestFunction, + } +} diff --git a/pkg/blobstore/chunklistvalidating/reader_chunker_test.go b/pkg/blobstore/chunklistvalidating/reader_chunker_test.go new file mode 100644 index 000000000..c5630d427 --- /dev/null +++ b/pkg/blobstore/chunklistvalidating/reader_chunker_test.go @@ -0,0 +1,107 @@ +package chunklistvalidating_test + +import ( + "io" + "math/rand" + "testing" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" +) + +const ( + minChunkSize = 256 << 10 // 256 KiB + maxChunkSize = 2*minChunkSize - 1 + horizonLookaheadBytes = 8 * minChunkSize +) + +func FuzzReaderChunker(f *testing.F) { + for i := range 20 { + // Fuzz test i+1 MB of data with seed i. + f.Add((i+1)<<20, int64(i)) + } + f.Fuzz(func(t *testing.T, dataSizeBytes int, seed int64) { + require := require.New(t) + rng := rand.New(rand.NewSource(seed)) + originalData := make([]byte, dataSizeBytes) + rng.Read(originalData) + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + composedData := make([]byte, 0, dataSizeBytes) + var numberOfChunks int + for numberOfChunks = 0; ; numberOfChunks++ { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + require.NoError(err, "Failed to generate chunk %d.", numberOfChunks) + + chunkSize := int64(len(chunk.Data)) + chunkHasher := chunk.Digest.NewHasher(chunkSize) + chunkHasher.Write(chunk.Data) + + require.Equal(chunk.Digest.GetHashBytes(), chunkHasher.Sum(nil), "Digest mismatch for %d.", numberOfChunks) + composedData = append(composedData, chunk.Data...) + } + + require.Equal(originalData, composedData) + + originalDigestGen := digestFunc.NewGenerator(int64(dataSizeBytes)) + originalDigestGen.Write(originalData) + + composedDigestGen := digestFunc.NewGenerator(int64(dataSizeBytes)) + composedDigestGen.Write(composedData) + + require.Equal(originalDigestGen.Sum(), composedDigestGen.Sum(), "The digest of the composed data does not match the digest of the original data.") + + minNumberOfChunks := dataSizeBytes / maxChunkSize + require.GreaterOrEqual(numberOfChunks, minNumberOfChunks, "Produced fewer chunks than should be possible.") + + maxNumberOfChunks := dataSizeBytes / minChunkSize + require.LessOrEqual(numberOfChunks, maxNumberOfChunks, "Produced more chunks than should be possible.") + }) +} + +func TestReaderChunkerSmallBlob(t *testing.T) { + // Test with a small blob that should produce a single chunk + originalData := []byte("Hello, World!") + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + chunks := make([][]byte, 0, 1) + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + require.NoError(t, err) + + chunks = append(chunks, chunk.Data) + } + require.Len(t, chunks, 1) + require.Equal(t, originalData, chunks[0]) +} + +func TestReaderChunkerEmptyBlob(t *testing.T) { + // Test with empty blob + originalData := []byte{} + reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() + defer reader.Close() + + digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) + chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + + chunk, err := chunker.NextChunk() + require.ErrorIs(t, io.EOF, err) + require.Empty(t, chunk.Data) +} diff --git a/pkg/blobstore/cls_read_buffer_factory.go b/pkg/blobstore/cls_read_buffer_factory.go new file mode 100644 index 000000000..7735f064b --- /dev/null +++ b/pkg/blobstore/cls_read_buffer_factory.go @@ -0,0 +1,27 @@ +package blobstore + +import ( + "io" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/digest" +) + +type clsReadBufferFactory struct{} + +func (clsReadBufferFactory) NewBufferFromByteSlice(digest digest.Digest, data []byte, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return buffer.NewProtoBufferFromByteSlice(&remoteexecution.SplitBlobResponse{}, data, buffer.BackendProvided(dataIntegrityCallback)) +} + +func (clsReadBufferFactory) NewBufferFromReader(digest digest.Digest, r io.ReadCloser, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return buffer.NewProtoBufferFromReader(&remoteexecution.SplitBlobResponse{}, r, buffer.BackendProvided(dataIntegrityCallback)) +} + +func (f clsReadBufferFactory) NewBufferFromReaderAt(digest digest.Digest, r buffer.ReadAtCloser, sizeBytes int64, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { + return f.NewBufferFromReader(digest, newReaderFromReaderAt(r), dataIntegrityCallback) +} + +// CLSReadBufferFactory is capable of creating identifiers and buffers +// for objects stored in the Chunk List Storage (CLS). +var CLSReadBufferFactory ReadBufferFactory = clsReadBufferFactory{} diff --git a/pkg/blobstore/configuration/BUILD.bazel b/pkg/blobstore/configuration/BUILD.bazel index 33ebe095b..8882b2b2e 100644 --- a/pkg/blobstore/configuration/BUILD.bazel +++ b/pkg/blobstore/configuration/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "blob_replicator_creator.go", "cas_blob_access_creator.go", "cas_blob_replicator_creator.go", + "cls_blob_access_creator.go", "fsac_blob_access_creator.go", "icas_blob_access_creator.go", "icas_blob_replicator_creator.go", @@ -21,6 +22,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/blobstore", + "//pkg/blobstore/chunklistvalidating", "//pkg/blobstore/completenesschecking", "//pkg/blobstore/grpcclients", "//pkg/blobstore/local", diff --git a/pkg/blobstore/configuration/blob_access_creator.go b/pkg/blobstore/configuration/blob_access_creator.go index 6b44c1171..50d6032ce 100644 --- a/pkg/blobstore/configuration/blob_access_creator.go +++ b/pkg/blobstore/configuration/blob_access_creator.go @@ -48,7 +48,7 @@ type BlobAccessCreator interface { // NewHierarchicalInstanceNamesLocalBlobAccess() creates a // BlobAccess suitable for storing data on the local system that // uses hierarchical instance names. - NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) + NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) // NewCustomBlobAccess() can be used as a fallback to create // BlobAccess instances that only apply to this storage type. // For example, CompletenessCheckingBlobAccess is only diff --git a/pkg/blobstore/configuration/cas_blob_access_creator.go b/pkg/blobstore/configuration/cas_blob_access_creator.go index 2677982a9..ac215dd68 100644 --- a/pkg/blobstore/configuration/cas_blob_access_creator.go +++ b/pkg/blobstore/configuration/cas_blob_access_creator.go @@ -72,8 +72,8 @@ func (casBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks in return local.NewImmutableBlockListGrowthPolicy(currentBlocks, newBlocks), nil } -func (casBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) { - return local.NewHierarchicalCASBlobAccess(keyLocationMap, locationBlobMap, globalLock, casCapabilitiesProvider), nil +func (casBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) { + return local.NewHierarchicalCASBlobAccess(keyLocationMap, locationBlobMap, globalLock, capabilitiesProvider), nil } func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { diff --git a/pkg/blobstore/configuration/cls_blob_access_creator.go b/pkg/blobstore/configuration/cls_blob_access_creator.go new file mode 100644 index 000000000..af2e695c9 --- /dev/null +++ b/pkg/blobstore/configuration/cls_blob_access_creator.go @@ -0,0 +1,86 @@ +package configuration + +import ( + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" + "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/grpc" + "github.com/buildbarn/bb-storage/pkg/program" + pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type clsBlobAccessCreator struct { + protoBlobAccessCreator + protoBlobReplicatorCreator + + contentAddressableStorage *BlobAccessInfo + grpcClientFactory grpc.ClientFactory + maximumMessageSizeBytes int +} + +// NewCLSBlobAccessCreator creates a BlobAccessCreator that can be +// provided to NewBlobAccessFromConfiguration() to construct a +// BlobAccess that is suitable for querying for chunk list. +func NewCLSBlobAccessCreator(contentAddressableStorage *BlobAccessInfo, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { + return &clsBlobAccessCreator{ + contentAddressableStorage: contentAddressableStorage, + grpcClientFactory: grpcClientFactory, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (clsBlobAccessCreator) GetReadBufferFactory() blobstore.ReadBufferFactory { + return blobstore.CLSReadBufferFactory +} + +func (clsBlobAccessCreator) GetStorageTypeName() string { + return "cls" +} + +func (clsBlobAccessCreator) GetDefaultCapabilitiesProvider() capabilities.Provider { + return capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{}) +} + +func (bac *clsBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { + switch backend := configuration.Backend.(type) { + case *pb.BlobAccessConfiguration_ChunkListValidating: + if bac.contentAddressableStorage == nil { + return BlobAccessInfo{}, "", status.Error(codes.InvalidArgument, "Chunk list validation can only be enabled if a Content Addressable Storage is configured") + } + + base, err := nestedCreator.NewNestedBlobAccess(backend.ChunkListValidating.Backend, bac) + if err != nil { + return BlobAccessInfo{}, "", err + } + return BlobAccessInfo{ + BlobAccess: chunklistvalidating.NewChunkListValidatingBlobAccess( + base.BlobAccess, + bac.contentAddressableStorage.BlobAccess, + bac.maximumMessageSizeBytes, + ), + DigestKeyFormat: base.DigestKeyFormat.Combine(bac.contentAddressableStorage.DigestKeyFormat), + }, "chunk_list_validating", nil + + case *pb.BlobAccessConfiguration_Grpc: + client, err := bac.grpcClientFactory.NewClientFromConfiguration(backend.Grpc.Client, terminationGroup) + if err != nil { + return BlobAccessInfo{}, "", err + } + return BlobAccessInfo{ + BlobAccess: grpcclients.NewCLSBlobAccess(client, bac.maximumMessageSizeBytes), + DigestKeyFormat: digest.KeyWithInstance, + }, "grpc", nil + + default: + return newProtoCustomBlobAccess(configuration, nestedCreator, bac) + } +} + +func (clsBlobAccessCreator) WrapTopLevelBlobAccess(blobAccess blobstore.BlobAccess) blobstore.BlobAccess { + return blobAccess +} diff --git a/pkg/blobstore/configuration/new_blob_access.go b/pkg/blobstore/configuration/new_blob_access.go index 941494d66..07e52041c 100644 --- a/pkg/blobstore/configuration/new_blob_access.go +++ b/pkg/blobstore/configuration/new_blob_access.go @@ -7,6 +7,7 @@ import ( "sync" "time" + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/local" "github.com/buildbarn/bb-storage/pkg/blobstore/mirrored" @@ -14,6 +15,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/readfallback" "github.com/buildbarn/bb-storage/pkg/blobstore/sharding" "github.com/buildbarn/bb-storage/pkg/blockdevice" + "github.com/buildbarn/bb-storage/pkg/capabilities" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/eviction" @@ -352,11 +354,26 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration * ) var localBlobAccess blobstore.BlobAccess + capabilitiesProvider := creator.GetDefaultCapabilitiesProvider() + chunkingParameters := backend.Local.GetChunkingParameters() + if chunkingParameters != nil { + capabilitiesProvider = capabilities.NewMergingProvider([]capabilities.Provider{ + capabilitiesProvider, + capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + SplitBlobSupport: true, + SpliceBlobSupport: true, + RepMaxCdcParams: chunkingParameters, + }, + }), + }) + } if backend.Local.HierarchicalInstanceNames { localBlobAccess, err = creator.NewHierarchicalInstanceNamesLocalBlobAccess( keyLocationMap, locationBlobMap, &globalLock, + capabilitiesProvider, ) if err != nil { return BlobAccessInfo{}, "", err @@ -368,7 +385,7 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration * digestKeyFormat, &globalLock, storageTypeName, - creator.GetDefaultCapabilitiesProvider(), + capabilitiesProvider, ) } return BlobAccessInfo{ diff --git a/pkg/blobstore/configuration/proto_blob_access_creator.go b/pkg/blobstore/configuration/proto_blob_access_creator.go index aa7daf857..d7ff593ba 100644 --- a/pkg/blobstore/configuration/proto_blob_access_creator.go +++ b/pkg/blobstore/configuration/proto_blob_access_creator.go @@ -5,6 +5,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/local" + "github.com/buildbarn/bb-storage/pkg/capabilities" "github.com/buildbarn/bb-storage/pkg/digest" pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" @@ -25,7 +26,7 @@ func (protoBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks return local.NewMutableBlockListGrowthPolicy(currentBlocks), nil } -func (protoBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex) (blobstore.BlobAccess, error) { +func (protoBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) { return nil, status.Error(codes.InvalidArgument, "The hierarchical instance names option can only be used for the Content Addressable Storage") } diff --git a/pkg/blobstore/grpcclients/BUILD.bazel b/pkg/blobstore/grpcclients/BUILD.bazel index 257d8fd04..bb518ecf6 100644 --- a/pkg/blobstore/grpcclients/BUILD.bazel +++ b/pkg/blobstore/grpcclients/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "ac_blob_access.go", "cas_blob_access.go", + "cls_blob_access.go", "fsac_blob_access.go", "icas_blob_access.go", "iscc_blob_access.go", diff --git a/pkg/blobstore/grpcclients/cls_blob_access.go b/pkg/blobstore/grpcclients/cls_blob_access.go new file mode 100644 index 000000000..a27e5ea84 --- /dev/null +++ b/pkg/blobstore/grpcclients/cls_blob_access.go @@ -0,0 +1,101 @@ +package grpcclients + +import ( + "context" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type clsBlobAccess struct { + contentAddressableStorageClient remoteexecution.ContentAddressableStorageClient + capabilitiesClient remoteexecution.CapabilitiesClient + maximumMessageSizeBytes int +} + +// NewCLSBlobAccess creates a BlobAccess that relays any requests to a +// gRPC server that implements the split and splice api calls of a +// remoteexecution.ContentAddressableStorage service. +func NewCLSBlobAccess(client grpc.ClientConnInterface, maximumMessageSizeBytes int) blobstore.BlobAccess { + return &clsBlobAccess{ + contentAddressableStorageClient: remoteexecution.NewContentAddressableStorageClient(client), + capabilitiesClient: remoteexecution.NewCapabilitiesClient(client), + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (ba *clsBlobAccess) Get(ctx context.Context, digest digest.Digest) buffer.Buffer { + digestFunction := digest.GetDigestFunction() + splitBlobsResponse, err := ba.contentAddressableStorageClient.SplitBlob(ctx, &remoteexecution.SplitBlobRequest{ + InstanceName: digestFunction.GetInstanceName().String(), + BlobDigest: digest.GetProto(), + DigestFunction: digestFunction.GetEnumValue(), + }) + if err != nil { + return buffer.NewBufferFromError(err) + } + return buffer.NewProtoBufferFromProto(splitBlobsResponse, buffer.BackendProvided(buffer.Irreparable(digest))) +} + +func (ba *clsBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { + b, _ := slicer.Slice(ba.Get(ctx, parentDigest), childDigest) + return b +} + +func (ba *clsBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { + splitBlobResponseProto, err := b.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + if err != nil { + return err + } + splitBlobResponse := splitBlobResponseProto.(*remoteexecution.SplitBlobResponse) + digestFunction := digest.GetDigestFunction() + _, err = ba.contentAddressableStorageClient.SpliceBlob(ctx, &remoteexecution.SpliceBlobRequest{ + InstanceName: digestFunction.GetInstanceName().String(), + DigestFunction: digestFunction.GetEnumValue(), + ChunkDigests: splitBlobResponse.GetChunkDigests(), + ChunkingFunction: splitBlobResponse.GetChunkingFunction(), + BlobDigest: digest.GetProto(), + }) + return err +} + +func (ba *clsBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + missing := digest.NewSetBuilder(digests.Length()) + for _, d := range digests.Items() { + _, err := ba.contentAddressableStorageClient.SplitBlob(ctx, &remoteexecution.SplitBlobRequest{ + InstanceName: d.GetInstanceName().String(), + BlobDigest: d.GetProto(), + DigestFunction: d.GetDigestFunction().GetEnumValue(), + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + }) + if status.Code(err) == codes.NotFound { + missing.Add(d) + } else if err != nil { + return digest.EmptySet, err + } + } + return missing.Build(), nil +} + +func (ba *clsBlobAccess) GetCapabilities(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.ServerCapabilities, error) { + serverCapabilities, err := getServerCapabilitiesWithCacheCapabilities(ctx, ba.capabilitiesClient, instanceName) + if err != nil { + return nil, err + } + cacheCapabilities := serverCapabilities.CacheCapabilities + // Only return fields that pertain to Chunk List Storage. + return &remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + SplitBlobSupport: cacheCapabilities.SplitBlobSupport, + SpliceBlobSupport: cacheCapabilities.SpliceBlobSupport, + RepMaxCdcParams: cacheCapabilities.RepMaxCdcParams, + }, + }, nil +} diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server.go b/pkg/blobstore/grpcservers/content_addressable_storage_server.go index 18802b959..bd76a5e4c 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server.go @@ -15,14 +15,16 @@ import ( type contentAddressableStorageServer struct { contentAddressableStorage blobstore.BlobAccess + chunkListStorage blobstore.BlobAccess maximumMessageSizeBytes int64 } // NewContentAddressableStorageServer creates a GRPC service for serving // the contents of a Bazel Content Addressable Storage (CAS) to Bazel. -func NewContentAddressableStorageServer(contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { +func NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage blobstore.BlobAccess, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { return &contentAddressableStorageServer{ contentAddressableStorage: contentAddressableStorage, + chunkListStorage: chunkListStorage, maximumMessageSizeBytes: maximumMessageSizeBytes, } } @@ -56,6 +58,36 @@ func (s *contentAddressableStorageServer) FindMissingBlobs(ctx context.Context, for _, outDigest := range outDigests.Items() { partialDigests = append(partialDigests, outDigest.GetProto()) } + + // Server is configured with Chunk List Storage (CLS) so we must + // verify the CLS as well. Note that in this version of bb-storage a + // missing chunk list for a blob does not imply that the blob is + // missing. It is merely required to manage the life time of chunk + // lists. In a future version of bb-storage FMB calls will go to + // either the chunk storage or the chunk list storage. + if s.chunkListStorage != nil { + capabilities, err := s.chunkListStorage.GetCapabilities(ctx, instanceName) + if err != nil { + return nil, err + } + if capabilities.CacheCapabilities.GetRepMaxCdcParams() == nil { + return nil, status.Error(codes.Internal, "This server implementation is only compatible with RepMaxCDC") + } + minChunkSize := capabilities.CacheCapabilities.RepMaxCdcParams.MinChunkSizeBytes + maxChunkSize := 2*minChunkSize - 1 + bigBlobDigests := digest.NewSetBuilder(0) + for _, partialDigest := range in.BlobDigests { + if partialDigest.GetSizeBytes() > int64(maxChunkSize) { + digest, err := digestFunction.NewDigestFromProto(partialDigest) + if err != nil { + return nil, err + } + bigBlobDigests.Add(digest) + } + } + _, _ = s.chunkListStorage.FindMissing(ctx, bigBlobDigests.Build()) + } + return &remoteexecution.FindMissingBlobsResponse{ MissingBlobDigests: partialDigests, }, nil @@ -150,10 +182,61 @@ func (contentAddressableStorageServer) GetTree(in *remoteexecution.GetTreeReques return status.Error(codes.Unimplemented, "This service does not support downloading directory trees") } -func (contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *remoteexecution.SpliceBlobRequest) (*remoteexecution.SpliceBlobResponse, error) { - return nil, status.Error(codes.Unimplemented, "This service does not support splicing blobs") +func (s *contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *remoteexecution.SpliceBlobRequest) (*remoteexecution.SpliceBlobResponse, error) { + // TODO: Require that s.chunkListStorage is non-null once we require chunking. + if s.chunkListStorage == nil { + return nil, status.Error(codes.Unimplemented, "This service does not support SpliceBlob") + } + + instanceName, err := digest.NewInstanceName(in.InstanceName) + if err != nil { + return nil, util.StatusWrapf(err, "Invalid instance name %#v", in.InstanceName) + } + digestFunction, err := instanceName.GetDigestFunction(in.DigestFunction, len(in.BlobDigest.GetHash())) + if err != nil { + return nil, err + } + blobDigest, err := digestFunction.NewDigestFromProto(in.BlobDigest) + if err != nil { + return nil, err + } + + splitBlobResponse := &remoteexecution.SplitBlobResponse{ + ChunkDigests: in.ChunkDigests, + ChunkingFunction: in.ChunkingFunction, + } + b := buffer.NewProtoBufferFromProto(splitBlobResponse, buffer.UserProvided) + + if err := s.chunkListStorage.Put(ctx, blobDigest, b); err != nil { + return nil, err + } + + return &remoteexecution.SpliceBlobResponse{ + BlobDigest: in.BlobDigest, + }, nil } -func (contentAddressableStorageServer) SplitBlob(ctx context.Context, in *remoteexecution.SplitBlobRequest) (*remoteexecution.SplitBlobResponse, error) { - return nil, status.Error(codes.Unimplemented, "This service does not support splitting blobs") +func (s *contentAddressableStorageServer) SplitBlob(ctx context.Context, in *remoteexecution.SplitBlobRequest) (*remoteexecution.SplitBlobResponse, error) { + // TODO: Require that s.chunkListStorage is non-null once we require chunking. + if s.chunkListStorage == nil { + return nil, status.Error(codes.Unimplemented, "This service does not support SplitBlob") + } + + instanceName, err := digest.NewInstanceName(in.InstanceName) + if err != nil { + return nil, util.StatusWrapf(err, "Invalid instance name %#v", in.InstanceName) + } + digestFunction, err := instanceName.GetDigestFunction(in.DigestFunction, len(in.BlobDigest.GetHash())) + if err != nil { + return nil, err + } + blobDigest, err := digestFunction.NewDigestFromProto(in.BlobDigest) + if err != nil { + return nil, err + } + splitBlobResponse, err := s.chunkListStorage.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, int(s.maximumMessageSizeBytes)) + if err != nil { + return nil, err + } + return splitBlobResponse.(*remoteexecution.SplitBlobResponse), nil } diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go b/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go index 89a278297..7c5ed83b4 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go @@ -55,7 +55,7 @@ func TestContentAddressableStorageServerBatchReadBlobsSuccess(t *testing.T) { buf3 := buffer.NewBufferFromError(status.Error(codes.NotFound, "The object you requested could not be found")) contentAddressableStorage.EXPECT().Get(ctx, digest3).Return(buf3) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, 1<<16) + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, nil, 4<<20) response, err := contentAddressableStorageServer.BatchReadBlobs(ctx, request) require.NoError(t, err) @@ -108,8 +108,138 @@ func TestContentAddressableStorageServerBatchReadBlobsFailure(t *testing.T) { contentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, 200) + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, nil, 200) _, err := contentAddressableStorageServer.BatchReadBlobs(ctx, request) testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Attempted to read a total of at least 357 bytes, while a maximum of 200 bytes is permitted"), err) } + +func TestContentAddressableStorageServerFindMissingBlobs(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + digest1 := digest.MustNewDigest("my_instance_name", remoteexecution.DigestFunction_SHA256, "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", 16) + digest2 := digest.MustNewDigest("my_instance_name", remoteexecution.DigestFunction_SHA256, "0479688f99e8cbc70291ce272876ff8e0db71a0889daf2752884b0996056b4a0", 256) + + request := &remoteexecution.FindMissingBlobsRequest{ + InstanceName: "my_instance_name", + BlobDigests: []*remoteexecution.Digest{ + {Hash: digest1.GetHashString(), SizeBytes: digest1.GetSizeBytes()}, + {Hash: digest2.GetHashString(), SizeBytes: digest2.GetSizeBytes()}, + }, + } + + contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + chunkListStorage := mock.NewMockBlobAccess(ctrl) + setBuilder := digest.NewSetBuilder(2) + digestSet := setBuilder.Add(digest1).Add(digest2).Build() + + // Missing chunk lists is not an error, nor does it imply a missing + // blob at this stage. + contentAddressableStorage.EXPECT().FindMissing(ctx, digestSet).Return(digest.EmptySet, nil) + chunkListStorage.EXPECT().GetCapabilities(ctx, digest1.GetInstanceName()).Return( + &remoteexecution.ServerCapabilities{ + CacheCapabilities: &remoteexecution.CacheCapabilities{ + SplitBlobSupport: true, + SpliceBlobSupport: true, + RepMaxCdcParams: &remoteexecution.RepMaxCdcParams{ + MinChunkSizeBytes: 64, + HorizonSizeBytes: 128, + }, + }, + }, + nil, + ) + chunkListStorage.EXPECT().FindMissing(ctx, digest2.ToSingletonSet()).Return(digest2.ToSingletonSet(), nil) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) + + response, err := contentAddressableStorageServer.FindMissingBlobs(ctx, request) + require.NoError(t, err) + require.Empty(t, response.GetMissingBlobDigests()) +} + +func TestContentAddressableStorageServerSplitBlob(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SplitBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + InstanceName: "my_instance_name", + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + chunkListStorage := mock.NewMockBlobAccess(ctrl) + + instanceName, err := digest.NewInstanceName(request.InstanceName) + require.NoError(t, err) + digestFunction, err := instanceName.GetDigestFunction(request.DigestFunction, len(request.BlobDigest.Hash)) + require.NoError(t, err) + blobDigest, err := digestFunction.NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + + chunkListStorage.EXPECT().Get(ctx, blobDigest).Return( + buffer.NewProtoBufferFromProto( + &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{ + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + }, + }, + buffer.UserProvided, + ), + ) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) + _, err = contentAddressableStorageServer.SplitBlob(ctx, request) + require.NoError(t, err) +} + +func TestContentAddressableStorageServerSpliceBlob(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SpliceBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + ChunkDigests: []*remoteexecution.Digest{ + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + { + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }, + }, + InstanceName: "my_instance_name", + } + + contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + chunkListStorage := mock.NewMockBlobAccess(ctrl) + + instanceName, err := digest.NewInstanceName(request.InstanceName) + require.NoError(t, err) + digestFunction, err := instanceName.GetDigestFunction(request.DigestFunction, len(request.BlobDigest.Hash)) + require.NoError(t, err) + blobDigest, err := digestFunction.NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + + chunkListStorage.EXPECT().Put(ctx, blobDigest, buffer.NewProtoBufferFromProto(&remoteexecution.SplitBlobResponse{ + ChunkDigests: request.ChunkDigests, + }, buffer.UserProvided)).Return(nil) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) + response, err := contentAddressableStorageServer.SpliceBlob(ctx, request) + require.NoError(t, err) + require.Equal(t, request.BlobDigest, response.BlobDigest) +} diff --git a/pkg/proto/configuration/bb_storage/bb_storage.pb.go b/pkg/proto/configuration/bb_storage/bb_storage.pb.go index fa4fb021e..fb4668e07 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.pb.go +++ b/pkg/proto/configuration/bb_storage/bb_storage.pb.go @@ -35,6 +35,7 @@ type ApplicationConfiguration struct { MaximumMessageSizeBytes int64 `protobuf:"varint,8,opt,name=maximum_message_size_bytes,json=maximumMessageSizeBytes,proto3" json:"maximum_message_size_bytes,omitempty"` Global *global.Configuration `protobuf:"bytes,9,opt,name=global,proto3" json:"global,omitempty"` ContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,17,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` + ChunkListStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,22,opt,name=chunk_list_storage,json=chunkListStorage,proto3" json:"chunk_list_storage,omitempty"` ActionCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,18,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` IndirectContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,10,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` InitialSizeClassCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,11,opt,name=initial_size_class_cache,json=initialSizeClassCache,proto3" json:"initial_size_class_cache,omitempty"` @@ -111,6 +112,13 @@ func (x *ApplicationConfiguration) GetContentAddressableStorage() *ScannableBlob return nil } +func (x *ApplicationConfiguration) GetChunkListStorage() *ScannableBlobAccessConfiguration { + if x != nil { + return x.ChunkListStorage + } + return nil +} + func (x *ApplicationConfiguration) GetActionCache() *NonScannableBlobAccessConfiguration { if x != nil { return x.ActionCache @@ -292,7 +300,7 @@ var File_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_s const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDesc = "" + "\n" + - "Sgithub.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage/bb_storage.proto\x12\"buildbarn.configuration.bb_storage\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto\x1aQgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x1aMgithub.com/buildbarn/bb-storage/pkg/proto/configuration/builder/builder.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/zstd/zstd.proto\"\xbd\v\n" + + "Sgithub.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage/bb_storage.proto\x12\"buildbarn.configuration.bb_storage\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto\x1aQgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x1aMgithub.com/buildbarn/bb-storage/pkg/proto/configuration/builder/builder.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/zstd/zstd.proto\"\xb1\f\n" + "\x18ApplicationConfiguration\x12T\n" + "\fgrpc_servers\x18\x04 \x03(\v21.buildbarn.configuration.grpc.ServerConfigurationR\vgrpcServers\x12l\n" + "\n" + @@ -300,7 +308,8 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb "schedulers\x12;\n" + "\x1amaximum_message_size_bytes\x18\b \x01(\x03R\x17maximumMessageSizeBytes\x12E\n" + "\x06global\x18\t \x01(\v2-.buildbarn.configuration.global.ConfigurationR\x06global\x12\x84\x01\n" + - "\x1bcontent_addressable_storage\x18\x11 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x19contentAddressableStorage\x12j\n" + + "\x1bcontent_addressable_storage\x18\x11 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x19contentAddressableStorage\x12r\n" + + "\x12chunk_list_storage\x18\x16 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x10chunkListStorage\x12j\n" + "\faction_cache\x18\x12 \x01(\v2G.buildbarn.configuration.bb_storage.NonScannableBlobAccessConfigurationR\vactionCache\x12\x95\x01\n" + "$indirect_content_addressable_storage\x18\n" + " \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR!indirectContentAddressableStorage\x12\x80\x01\n" + @@ -353,26 +362,27 @@ var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_s 3, // 1: buildbarn.configuration.bb_storage.ApplicationConfiguration.schedulers:type_name -> buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry 5, // 2: buildbarn.configuration.bb_storage.ApplicationConfiguration.global:type_name -> buildbarn.configuration.global.Configuration 2, // 3: buildbarn.configuration.bb_storage.ApplicationConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 1, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.action_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 2, // 5: buildbarn.configuration.bb_storage.ApplicationConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 1, // 6: buildbarn.configuration.bb_storage.ApplicationConfiguration.initial_size_class_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 1, // 7: buildbarn.configuration.bb_storage.ApplicationConfiguration.file_system_access_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 6, // 8: buildbarn.configuration.bb_storage.ApplicationConfiguration.execute_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 7, // 9: buildbarn.configuration.bb_storage.ApplicationConfiguration.supported_compressors:type_name -> build.bazel.remote.execution.v2.Compressor.Value - 8, // 10: buildbarn.configuration.bb_storage.ApplicationConfiguration.zstd_pool:type_name -> buildbarn.configuration.zstd.PoolConfiguration - 9, // 11: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 6, // 12: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 13: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 9, // 14: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 6, // 15: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 16: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 17: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 10, // 18: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry.value:type_name -> buildbarn.configuration.builder.SchedulerConfiguration - 19, // [19:19] is the sub-list for method output_type - 19, // [19:19] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 2, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.chunk_list_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration + 1, // 5: buildbarn.configuration.bb_storage.ApplicationConfiguration.action_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 2, // 6: buildbarn.configuration.bb_storage.ApplicationConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration + 1, // 7: buildbarn.configuration.bb_storage.ApplicationConfiguration.initial_size_class_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 1, // 8: buildbarn.configuration.bb_storage.ApplicationConfiguration.file_system_access_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 6, // 9: buildbarn.configuration.bb_storage.ApplicationConfiguration.execute_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 10: buildbarn.configuration.bb_storage.ApplicationConfiguration.supported_compressors:type_name -> build.bazel.remote.execution.v2.Compressor.Value + 8, // 11: buildbarn.configuration.bb_storage.ApplicationConfiguration.zstd_pool:type_name -> buildbarn.configuration.zstd.PoolConfiguration + 9, // 12: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 6, // 13: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 6, // 14: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 9, // 15: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 6, // 16: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 6, // 17: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 6, // 18: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 10, // 19: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry.value:type_name -> buildbarn.configuration.builder.SchedulerConfiguration + 20, // [20:20] is the sub-list for method output_type + 20, // [20:20] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { diff --git a/pkg/proto/configuration/bb_storage/bb_storage.proto b/pkg/proto/configuration/bb_storage/bb_storage.proto index 365c75527..55f431b9b 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.proto +++ b/pkg/proto/configuration/bb_storage/bb_storage.proto @@ -61,6 +61,10 @@ message ApplicationConfiguration { // Storage (CAS). ScannableBlobAccessConfiguration content_addressable_storage = 17; + // Optional: Blobstore configuration for the Content List Storage + // (CLS). + ScannableBlobAccessConfiguration chunk_list_storage = 22; + // Optional: Blobstore configuration for the Action Cache (AC). NonScannableBlobAccessConfiguration action_cache = 18; diff --git a/pkg/proto/configuration/blobstore/BUILD.bazel b/pkg/proto/configuration/blobstore/BUILD.bazel index a64c36145..83d0b07cc 100644 --- a/pkg/proto/configuration/blobstore/BUILD.bazel +++ b/pkg/proto/configuration/blobstore/BUILD.bazel @@ -14,6 +14,7 @@ proto_library( "//pkg/proto/configuration/digest:digest_proto", "//pkg/proto/configuration/grpc:grpc_proto", "//pkg/proto/configuration/http/client:client_proto", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto", "@googleapis//google/rpc:status_proto", "@protobuf//:duration_proto", "@protobuf//:empty_proto", @@ -33,6 +34,7 @@ go_proto_library( "//pkg/proto/configuration/digest", "//pkg/proto/configuration/grpc", "//pkg/proto/configuration/http/client", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", "@org_golang_google_genproto_googleapis_rpc//status", ], ) diff --git a/pkg/proto/configuration/blobstore/blobstore.pb.go b/pkg/proto/configuration/blobstore/blobstore.pb.go index dc3f8a4e0..237805e13 100644 --- a/pkg/proto/configuration/blobstore/blobstore.pb.go +++ b/pkg/proto/configuration/blobstore/blobstore.pb.go @@ -7,6 +7,7 @@ package blobstore import ( + v2 "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" blockdevice "github.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice" aws "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws" gcp "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp" @@ -106,6 +107,7 @@ type BlobAccessConfiguration struct { // *BlobAccessConfiguration_WithLabels // *BlobAccessConfiguration_Label // *BlobAccessConfiguration_DeadlineEnforcing + // *BlobAccessConfiguration_ChunkListValidating Backend isBlobAccessConfiguration_Backend `protobuf_oneof:"backend"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -319,6 +321,15 @@ func (x *BlobAccessConfiguration) GetDeadlineEnforcing() *DeadlineEnforcingBlobA return nil } +func (x *BlobAccessConfiguration) GetChunkListValidating() *ChunkListValidatingBlobAccessConfiguration { + if x != nil { + if x, ok := x.Backend.(*BlobAccessConfiguration_ChunkListValidating); ok { + return x.ChunkListValidating + } + } + return nil +} + type isBlobAccessConfiguration_Backend interface { isBlobAccessConfiguration_Backend() } @@ -399,6 +410,10 @@ type BlobAccessConfiguration_DeadlineEnforcing struct { DeadlineEnforcing *DeadlineEnforcingBlobAccess `protobuf:"bytes,28,opt,name=deadline_enforcing,json=deadlineEnforcing,proto3,oneof"` } +type BlobAccessConfiguration_ChunkListValidating struct { + ChunkListValidating *ChunkListValidatingBlobAccessConfiguration `protobuf:"bytes,29,opt,name=chunk_list_validating,json=chunkListValidating,proto3,oneof"` +} + func (*BlobAccessConfiguration_ReadCaching) isBlobAccessConfiguration_Backend() {} func (*BlobAccessConfiguration_Grpc) isBlobAccessConfiguration_Backend() {} @@ -437,6 +452,8 @@ func (*BlobAccessConfiguration_Label) isBlobAccessConfiguration_Backend() {} func (*BlobAccessConfiguration_DeadlineEnforcing) isBlobAccessConfiguration_Backend() {} +func (*BlobAccessConfiguration_ChunkListValidating) isBlobAccessConfiguration_Backend() {} + type ReadCachingBlobAccessConfiguration struct { state protoimpl.MessageState `protogen:"open.v1"` Slow *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=slow,proto3" json:"slow,omitempty"` @@ -628,6 +645,7 @@ type LocalBlobAccessConfiguration struct { BlocksBackend isLocalBlobAccessConfiguration_BlocksBackend `protobuf_oneof:"blocks_backend"` Persistent *LocalBlobAccessConfiguration_Persistent `protobuf:"bytes,13,opt,name=persistent,proto3" json:"persistent,omitempty"` HierarchicalInstanceNames bool `protobuf:"varint,14,opt,name=hierarchical_instance_names,json=hierarchicalInstanceNames,proto3" json:"hierarchical_instance_names,omitempty"` + ChunkingParameters *v2.RepMaxCdcParams `protobuf:"bytes,15,opt,name=chunking_parameters,json=chunkingParameters,proto3" json:"chunking_parameters,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -761,6 +779,13 @@ func (x *LocalBlobAccessConfiguration) GetHierarchicalInstanceNames() bool { return false } +func (x *LocalBlobAccessConfiguration) GetChunkingParameters() *v2.RepMaxCdcParams { + if x != nil { + return x.ChunkingParameters + } + return nil +} + type isLocalBlobAccessConfiguration_KeyLocationMapBackend interface { isLocalBlobAccessConfiguration_KeyLocationMapBackend() } @@ -1726,6 +1751,50 @@ func (x *GrpcBlobAccessConfiguration) GetEnableCompression() bool { return false } +type ChunkListValidatingBlobAccessConfiguration struct { + state protoimpl.MessageState `protogen:"open.v1"` + Backend *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=backend,proto3" json:"backend,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChunkListValidatingBlobAccessConfiguration) Reset() { + *x = ChunkListValidatingBlobAccessConfiguration{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChunkListValidatingBlobAccessConfiguration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkListValidatingBlobAccessConfiguration) ProtoMessage() {} + +func (x *ChunkListValidatingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkListValidatingBlobAccessConfiguration.ProtoReflect.Descriptor instead. +func (*ChunkListValidatingBlobAccessConfiguration) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{21} +} + +func (x *ChunkListValidatingBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { + if x != nil { + return x.Backend + } + return nil +} + type ShardingBlobAccessConfiguration_Shard struct { state protoimpl.MessageState `protogen:"open.v1"` Backend *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=backend,proto3" json:"backend,omitempty"` @@ -1736,7 +1805,7 @@ type ShardingBlobAccessConfiguration_Shard struct { func (x *ShardingBlobAccessConfiguration_Shard) Reset() { *x = ShardingBlobAccessConfiguration_Shard{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1748,7 +1817,7 @@ func (x *ShardingBlobAccessConfiguration_Shard) String() string { func (*ShardingBlobAccessConfiguration_Shard) ProtoMessage() {} func (x *ShardingBlobAccessConfiguration_Shard) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1787,7 +1856,7 @@ type LocalBlobAccessConfiguration_KeyLocationMapInMemory struct { func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) Reset() { *x = LocalBlobAccessConfiguration_KeyLocationMapInMemory{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1799,7 +1868,7 @@ func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) String() string { func (*LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1831,7 +1900,7 @@ type LocalBlobAccessConfiguration_BlocksInMemory struct { func (x *LocalBlobAccessConfiguration_BlocksInMemory) Reset() { *x = LocalBlobAccessConfiguration_BlocksInMemory{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1843,7 +1912,7 @@ func (x *LocalBlobAccessConfiguration_BlocksInMemory) String() string { func (*LocalBlobAccessConfiguration_BlocksInMemory) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_BlocksInMemory) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1877,7 +1946,7 @@ type LocalBlobAccessConfiguration_BlocksOnBlockDevice struct { func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) Reset() { *x = LocalBlobAccessConfiguration_BlocksOnBlockDevice{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1889,7 +1958,7 @@ func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) String() string { func (*LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1936,7 +2005,7 @@ type LocalBlobAccessConfiguration_Persistent struct { func (x *LocalBlobAccessConfiguration_Persistent) Reset() { *x = LocalBlobAccessConfiguration_Persistent{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1948,7 +2017,7 @@ func (x *LocalBlobAccessConfiguration_Persistent) String() string { func (*LocalBlobAccessConfiguration_Persistent) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_Persistent) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1982,10 +2051,10 @@ var File_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobs const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc = "" + "\n" + - "Qgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x12!buildbarn.configuration.blobstore\x1aUgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/digest/digest.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aPgithub.com/buildbarn/bb-storage/pkg/proto/configuration/http/client/client.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"\xf3\x01\n" + + "Qgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x12!buildbarn.configuration.blobstore\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aUgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/digest/digest.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aPgithub.com/buildbarn/bb-storage/pkg/proto/configuration/http/client/client.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"\xf3\x01\n" + "\x16BlobstoreConfiguration\x12z\n" + "\x1bcontent_addressable_storage\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\x19contentAddressableStorage\x12]\n" + - "\faction_cache\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\vactionCache\"\xe3\x0f\n" + + "\faction_cache\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\vactionCache\"\xe9\x10\n" + "\x17BlobAccessConfiguration\x12j\n" + "\fread_caching\x18\x04 \x01(\v2E.buildbarn.configuration.blobstore.ReadCachingBlobAccessConfigurationH\x00R\vreadCaching\x12T\n" + "\x04grpc\x18\a \x01(\v2>.buildbarn.configuration.blobstore.GrpcBlobAccessConfigurationH\x00R\x04grpc\x12*\n" + @@ -2008,7 +2077,8 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\vwith_labels\x18\x1a \x01(\v2D.buildbarn.configuration.blobstore.WithLabelsBlobAccessConfigurationH\x00R\n" + "withLabels\x12\x16\n" + "\x05label\x18\x1b \x01(\tH\x00R\x05label\x12o\n" + - "\x12deadline_enforcing\x18\x1c \x01(\v2>.buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccessH\x00R\x11deadlineEnforcingB\t\n" + + "\x12deadline_enforcing\x18\x1c \x01(\v2>.buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccessH\x00R\x11deadlineEnforcing\x12\x83\x01\n" + + "\x15chunk_list_validating\x18\x1d \x01(\v2M.buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfigurationH\x00R\x13chunkListValidatingB\t\n" + "\abackendJ\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x05\x10\x06J\x04\b\x06\x10\aJ\x04\b\n" + "\x10\v\"\xa4\x02\n" + "\"ReadCachingBlobAccessConfiguration\x12N\n" + @@ -2029,7 +2099,7 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\tbackend_a\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\bbackendA\x12W\n" + "\tbackend_b\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\bbackendB\x12i\n" + "\x11replicator_a_to_b\x18\x03 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorAToB\x12i\n" + - "\x11replicator_b_to_a\x18\x04 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorBToA\"\xb6\f\n" + + "\x11replicator_b_to_a\x18\x04 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\x0ereplicatorBToA\"\x99\r\n" + "\x1cLocalBlobAccessConfiguration\x12\x94\x01\n" + "\x1akey_location_map_in_memory\x18\v \x01(\v2V.buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemoryH\x00R\x16keyLocationMapInMemory\x12{\n" + " key_location_map_on_block_device\x18\f \x01(\v22.buildbarn.configuration.blockdevice.ConfigurationH\x00R\x1bkeyLocationMapOnBlockDevice\x12O\n" + @@ -2046,7 +2116,8 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\n" + "persistent\x18\r \x01(\v2J.buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.PersistentR\n" + "persistent\x12>\n" + - "\x1bhierarchical_instance_names\x18\x0e \x01(\bR\x19hierarchicalInstanceNames\x1a2\n" + + "\x1bhierarchical_instance_names\x18\x0e \x01(\bR\x19hierarchicalInstanceNames\x12a\n" + + "\x13chunking_parameters\x18\x0f \x01(\v20.build.bazel.remote.execution.v2.RepMaxCdcParamsR\x12chunkingParameters\x1a2\n" + "\x16KeyLocationMapInMemory\x12\x18\n" + "\aentries\x18\x01 \x01(\x03R\aentries\x1a:\n" + "\x0eBlocksInMemory\x12(\n" + @@ -2127,7 +2198,9 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\abackend\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\abackend\"\x97\x01\n" + "\x1bGrpcBlobAccessConfiguration\x12I\n" + "\x06client\x18\x01 \x01(\v21.buildbarn.configuration.grpc.ClientConfigurationR\x06client\x12-\n" + - "\x12enable_compression\x18\x02 \x01(\bR\x11enableCompressionBCZAgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstoreb\x06proto3" + "\x12enable_compression\x18\x02 \x01(\bR\x11enableCompression\"\x82\x01\n" + + "*ChunkListValidatingBlobAccessConfiguration\x12T\n" + + "\abackend\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\abackendBCZAgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstoreb\x06proto3" var ( file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescOnce sync.Once @@ -2141,7 +2214,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescData } -var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 29) +var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 30) var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_goTypes = []any{ (*BlobstoreConfiguration)(nil), // 0: buildbarn.configuration.blobstore.BlobstoreConfiguration (*BlobAccessConfiguration)(nil), // 1: buildbarn.configuration.blobstore.BlobAccessConfiguration @@ -2164,31 +2237,33 @@ var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobs (*WithLabelsBlobAccessConfiguration)(nil), // 18: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration (*DeadlineEnforcingBlobAccess)(nil), // 19: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess (*GrpcBlobAccessConfiguration)(nil), // 20: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration - (*ShardingBlobAccessConfiguration_Shard)(nil), // 21: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard - nil, // 22: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry - (*LocalBlobAccessConfiguration_KeyLocationMapInMemory)(nil), // 23: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory - (*LocalBlobAccessConfiguration_BlocksInMemory)(nil), // 24: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory - (*LocalBlobAccessConfiguration_BlocksOnBlockDevice)(nil), // 25: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice - (*LocalBlobAccessConfiguration_Persistent)(nil), // 26: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent - nil, // 27: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry - nil, // 28: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry - (*status.Status)(nil), // 29: google.rpc.Status - (*blockdevice.Configuration)(nil), // 30: buildbarn.configuration.blockdevice.Configuration - (*digest.ExistenceCacheConfiguration)(nil), // 31: buildbarn.configuration.digest.ExistenceCacheConfiguration - (*aws.SessionConfiguration)(nil), // 32: buildbarn.configuration.cloud.aws.SessionConfiguration - (*client.Configuration)(nil), // 33: buildbarn.configuration.http.client.Configuration - (*gcp.ClientOptionsConfiguration)(nil), // 34: buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration - (*emptypb.Empty)(nil), // 35: google.protobuf.Empty - (*grpc.ClientConfiguration)(nil), // 36: buildbarn.configuration.grpc.ClientConfiguration - (*durationpb.Duration)(nil), // 37: google.protobuf.Duration - (*timestamppb.Timestamp)(nil), // 38: google.protobuf.Timestamp + (*ChunkListValidatingBlobAccessConfiguration)(nil), // 21: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration + (*ShardingBlobAccessConfiguration_Shard)(nil), // 22: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard + nil, // 23: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry + (*LocalBlobAccessConfiguration_KeyLocationMapInMemory)(nil), // 24: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory + (*LocalBlobAccessConfiguration_BlocksInMemory)(nil), // 25: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory + (*LocalBlobAccessConfiguration_BlocksOnBlockDevice)(nil), // 26: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice + (*LocalBlobAccessConfiguration_Persistent)(nil), // 27: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent + nil, // 28: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry + nil, // 29: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry + (*status.Status)(nil), // 30: google.rpc.Status + (*blockdevice.Configuration)(nil), // 31: buildbarn.configuration.blockdevice.Configuration + (*v2.RepMaxCdcParams)(nil), // 32: build.bazel.remote.execution.v2.RepMaxCdcParams + (*digest.ExistenceCacheConfiguration)(nil), // 33: buildbarn.configuration.digest.ExistenceCacheConfiguration + (*aws.SessionConfiguration)(nil), // 34: buildbarn.configuration.cloud.aws.SessionConfiguration + (*client.Configuration)(nil), // 35: buildbarn.configuration.http.client.Configuration + (*gcp.ClientOptionsConfiguration)(nil), // 36: buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration + (*emptypb.Empty)(nil), // 37: google.protobuf.Empty + (*grpc.ClientConfiguration)(nil), // 38: buildbarn.configuration.grpc.ClientConfiguration + (*durationpb.Duration)(nil), // 39: google.protobuf.Duration + (*timestamppb.Timestamp)(nil), // 40: google.protobuf.Timestamp } var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_depIdxs = []int32{ 1, // 0: buildbarn.configuration.blobstore.BlobstoreConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration 1, // 1: buildbarn.configuration.blobstore.BlobstoreConfiguration.action_cache:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration 2, // 2: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_caching:type_name -> buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration 20, // 3: buildbarn.configuration.blobstore.BlobAccessConfiguration.grpc:type_name -> buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration - 29, // 4: buildbarn.configuration.blobstore.BlobAccessConfiguration.error:type_name -> google.rpc.Status + 30, // 4: buildbarn.configuration.blobstore.BlobAccessConfiguration.error:type_name -> google.rpc.Status 3, // 5: buildbarn.configuration.blobstore.BlobAccessConfiguration.sharding:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration 4, // 6: buildbarn.configuration.blobstore.BlobAccessConfiguration.mirrored:type_name -> buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration 5, // 7: buildbarn.configuration.blobstore.BlobAccessConfiguration.local:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration @@ -2204,66 +2279,69 @@ var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobs 17, // 17: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_writing:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration 18, // 18: buildbarn.configuration.blobstore.BlobAccessConfiguration.with_labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration 19, // 19: buildbarn.configuration.blobstore.BlobAccessConfiguration.deadline_enforcing:type_name -> buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess - 1, // 20: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.slow:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 21: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.fast:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 22: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 22, // 23: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.shards:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry - 1, // 24: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_a:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 25: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_b:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 26: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_a_to_b:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 10, // 27: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_b_to_a:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 23, // 28: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory - 30, // 29: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_on_block_device:type_name -> buildbarn.configuration.blockdevice.Configuration - 24, // 30: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory - 25, // 31: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_on_block_device:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice - 26, // 32: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.persistent:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent - 1, // 33: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 31, // 34: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 1, // 35: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 36: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.primary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 37: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.secondary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 38: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 1, // 39: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 32, // 40: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.aws_session:type_name -> buildbarn.configuration.cloud.aws.SessionConfiguration - 33, // 41: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.http_client:type_name -> buildbarn.configuration.http.client.Configuration - 34, // 42: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.gcp_client_options:type_name -> buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration - 1, // 43: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 35, // 44: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.local:type_name -> google.protobuf.Empty - 36, // 45: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.remote:type_name -> buildbarn.configuration.grpc.ClientConfiguration - 11, // 46: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.queued:type_name -> buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration - 35, // 47: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.noop:type_name -> google.protobuf.Empty - 10, // 48: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.deduplicating:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 12, // 49: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.concurrency_limiting:type_name -> buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration - 10, // 50: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 31, // 51: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 10, // 52: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 27, // 53: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.instance_name_prefixes:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry - 1, // 54: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 55: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 37, // 56: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_validity:type_name -> google.protobuf.Duration - 37, // 57: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.maximum_validity_jitter:type_name -> google.protobuf.Duration - 38, // 58: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_timestamp:type_name -> google.protobuf.Timestamp - 1, // 59: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.source:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 60: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.replica:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 37, // 61: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.maximum_cache_duration:type_name -> google.protobuf.Duration - 31, // 62: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 1, // 63: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 28, // 64: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry - 37, // 65: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.timeout:type_name -> google.protobuf.Duration - 1, // 66: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 36, // 67: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration.client:type_name -> buildbarn.configuration.grpc.ClientConfiguration - 1, // 68: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 21, // 69: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry.value:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard - 30, // 70: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.source:type_name -> buildbarn.configuration.blockdevice.Configuration - 31, // 71: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 37, // 72: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent.minimum_epoch_interval:type_name -> google.protobuf.Duration - 14, // 73: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry.value:type_name -> buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration - 1, // 74: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry.value:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 75, // [75:75] is the sub-list for method output_type - 75, // [75:75] is the sub-list for method input_type - 75, // [75:75] is the sub-list for extension type_name - 75, // [75:75] is the sub-list for extension extendee - 0, // [0:75] is the sub-list for field type_name + 21, // 20: buildbarn.configuration.blobstore.BlobAccessConfiguration.chunk_list_validating:type_name -> buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration + 1, // 21: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.slow:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 22: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.fast:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 10, // 23: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 23, // 24: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.shards:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry + 1, // 25: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_a:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 26: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_b:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 10, // 27: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_a_to_b:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 10, // 28: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_b_to_a:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 24, // 29: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory + 31, // 30: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_on_block_device:type_name -> buildbarn.configuration.blockdevice.Configuration + 25, // 31: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory + 26, // 32: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_on_block_device:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice + 27, // 33: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.persistent:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent + 32, // 34: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.chunking_parameters:type_name -> build.bazel.remote.execution.v2.RepMaxCdcParams + 1, // 35: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 33, // 36: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 1, // 37: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 38: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.primary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 39: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.secondary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 10, // 40: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 1, // 41: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 34, // 42: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.aws_session:type_name -> buildbarn.configuration.cloud.aws.SessionConfiguration + 35, // 43: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.http_client:type_name -> buildbarn.configuration.http.client.Configuration + 36, // 44: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.gcp_client_options:type_name -> buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration + 1, // 45: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 37, // 46: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.local:type_name -> google.protobuf.Empty + 38, // 47: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.remote:type_name -> buildbarn.configuration.grpc.ClientConfiguration + 11, // 48: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.queued:type_name -> buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration + 37, // 49: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.noop:type_name -> google.protobuf.Empty + 10, // 50: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.deduplicating:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 12, // 51: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.concurrency_limiting:type_name -> buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration + 10, // 52: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 33, // 53: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 10, // 54: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 28, // 55: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.instance_name_prefixes:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry + 1, // 56: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 57: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 39, // 58: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_validity:type_name -> google.protobuf.Duration + 39, // 59: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.maximum_validity_jitter:type_name -> google.protobuf.Duration + 40, // 60: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_timestamp:type_name -> google.protobuf.Timestamp + 1, // 61: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.source:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 62: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.replica:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 39, // 63: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.maximum_cache_duration:type_name -> google.protobuf.Duration + 33, // 64: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 1, // 65: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 29, // 66: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry + 39, // 67: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.timeout:type_name -> google.protobuf.Duration + 1, // 68: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 38, // 69: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration.client:type_name -> buildbarn.configuration.grpc.ClientConfiguration + 1, // 70: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 1, // 71: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 22, // 72: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry.value:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard + 31, // 73: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.source:type_name -> buildbarn.configuration.blockdevice.Configuration + 33, // 74: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 39, // 75: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent.minimum_epoch_interval:type_name -> google.protobuf.Duration + 14, // 76: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry.value:type_name -> buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration + 1, // 77: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry.value:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 78, // [78:78] is the sub-list for method output_type + 78, // [78:78] is the sub-list for method input_type + 78, // [78:78] is the sub-list for extension type_name + 78, // [78:78] is the sub-list for extension extendee + 0, // [0:78] is the sub-list for field type_name } func init() { @@ -2293,6 +2371,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob (*BlobAccessConfiguration_WithLabels)(nil), (*BlobAccessConfiguration_Label)(nil), (*BlobAccessConfiguration_DeadlineEnforcing)(nil), + (*BlobAccessConfiguration_ChunkListValidating)(nil), } file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5].OneofWrappers = []any{ (*LocalBlobAccessConfiguration_KeyLocationMapInMemory_)(nil), @@ -2314,7 +2393,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc)), NumEnums: 0, - NumMessages: 29, + NumMessages: 30, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/configuration/blobstore/blobstore.proto b/pkg/proto/configuration/blobstore/blobstore.proto index 68b226711..917df5514 100644 --- a/pkg/proto/configuration/blobstore/blobstore.proto +++ b/pkg/proto/configuration/blobstore/blobstore.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package buildbarn.configuration.blobstore; +import "build/bazel/remote/execution/v2/remote_execution.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto"; import "github.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto"; @@ -199,6 +200,29 @@ message BlobAccessConfiguration { // value. When gRPC calls are timed out a `DEADLINE_EXCEEDED` error // code will be returned. DeadlineEnforcingBlobAccess deadline_enforcing = 28; + + // Validate that chunk list requests fulfill the api specification. + // Chunk List Storage (CLS) is used to implement the SplitBlob and + // SpliceBlob methods from the REv2 api. The protocol has several + // demands on the state of the Content Addressable Storage (CAS) + // after those methods have been called. + // + // SplitBlob requires that that the blob as well as all chunks of + // the blob are available in the CAS, they have their lifetime + // renewed and that the resulting chunk list composes into the blob. + // This is validated by the Get path by splitting the Blob on demand + // if required. + // + // SpliceBlob requires that the blob as well as all chunks of the + // blob are present in storage, they have their lifetime renewed and + // that the supplied chunk list composes into the blob. Notably it + // does not require the chunks to follow any particular chunking + // algorithm but our implementation ensures that after any call a + // proper rep max cdc chunk list is verified even if the caller + // supplied a different chunk list. + // + // This decorator must be placed on the CLS. + ChunkListValidatingBlobAccessConfiguration chunk_list_validating = 29; } // Was 'redis'. Instead of using Redis, one may run a separate @@ -642,6 +666,18 @@ message LocalBlobAccessConfiguration { // level, e.g., on top of CompletenessCheckingBlobAccess. This can be // achieved by using HierarchicalInstanceNamesBlobAccess. bool hierarchical_instance_names = 14; + + // The chunking parameters advertised via GetCapabilities, setting + // this announces support for the SplitBlob and SpliceBlob api calls. + // + // Adding the chunking parameters does not make the underlying + // blobstore enforce the chunking parameters or implement the split + // and splice blob. There must be a configured Chunk List Storage with + // an outer ChunkValidatingBlobAccess that has view of the entire + // Content Addressable Storage (CAS). + // + // This option is only supported for the CLS. + build.bazel.remote.execution.v2.RepMaxCdcParams chunking_parameters = 15; } message ExistenceCachingBlobAccessConfiguration { @@ -937,3 +973,8 @@ message GrpcBlobAccessConfiguration { // types (AC, ICAS, etc.). bool enable_compression = 2; } + +message ChunkListValidatingBlobAccessConfiguration { + // The backend to which validated operations are delegated. + BlobAccessConfiguration backend = 1; +} From 10b73fdd0543caa364b634cb9fe2d17af3a12e8a Mon Sep 17 00:00:00 2001 From: Benjamin Ingberg Date: Tue, 9 Jun 2026 08:44:01 +0200 Subject: [PATCH 3/6] Replace CAS with CS and CLS This commit builds on top of our split and splice blob support to make it a mandatory first class feature in Buildbarn. With this commit the Content Addressable Storage (CAS) is created from two Storage configurations that work in tandem. A Chunk Storage (CS) which is content addressed and contains chunks of blobs, and a Chunk List Storage (CLS) which is addressed by a blob digest and contains a manifest describing the chunks that make up the blob. All api calls are automatically translated to use Chunk Lists created with RepMaxCDC. Effectively this means that large blobs no longer exists in the storage layer, individual chunks of the large blobs are in turn deduplicated in such a manner that the chunks are stored only once. The automatic translation makes certain that clients that are not cdc aware can still continue to use the storage backend without performing any changes. Clients which support RepMaxCDC also gets a significant reduction in the amount of blobs to transfer as they only need to transfer modified chunks rather than the entire blob. --- cmd/bb_copy/main.go | 4 +- cmd/bb_replicator/main.go | 4 +- cmd/bb_storage/BUILD.bazel | 4 + cmd/bb_storage/main.go | 83 +-- internal/mock/BUILD.bazel | 16 + pkg/blobstore/BUILD.bazel | 16 +- pkg/blobstore/authorizing_blob_access.go | 8 - pkg/blobstore/authorizing_blob_access_test.go | 18 - pkg/blobstore/blob_access.go | 2 - pkg/blobstore/cas_read_buffer_factory.go | 30 - pkg/blobstore/cdc/BUILD.bazel | 43 ++ pkg/blobstore/cdc/bypass.go | 22 + .../cdc/caching_parameters_fetcher.go | 33 + .../{chunklistvalidating => cdc}/chunker.go | 2 +- .../cdc/content_addressable_storage.go | 129 ++++ pkg/blobstore/cdc/operations.go | 213 ++++++ pkg/blobstore/cdc/parameters.go | 59 ++ pkg/blobstore/cdc/put_proto.go | 29 + pkg/blobstore/cdc/read_blob_at.go | 68 ++ .../reader_chunker.go | 4 +- .../reader_chunker_test.go | 10 +- pkg/blobstore/chunk_list_fetcher.go | 33 + pkg/blobstore/chunklist/BUILD.bazel | 22 + pkg/blobstore/chunklist/caching_fetcher.go | 33 + .../chunklist/chunk_concatenating_reader.go | 74 +++ pkg/blobstore/chunklist/chunk_fetcher.go | 26 + pkg/blobstore/chunklist/chunk_list.go | 43 ++ pkg/blobstore/chunklist/encoding.go | 56 ++ pkg/blobstore/chunklistvalidating/BUILD.bazel | 16 +- .../chunk_concatenating_reader.go | 68 -- .../chunk_list_validating_blob_access.go | 381 +++++------ .../chunk_list_validating_blob_access_test.go | 301 ++++----- .../integration/BUILD.bazel | 19 - .../chunk_list_validating_integration_test.go | 445 ------------- pkg/blobstore/cls_read_buffer_factory.go | 6 +- .../completenesschecking/BUILD.bazel | 3 + .../completeness_checking_blob_access.go | 20 +- .../completeness_checking_blob_access_test.go | 159 ++--- pkg/blobstore/configuration/BUILD.bazel | 8 +- .../configuration/ac_blob_access_creator.go | 15 +- .../configuration/cls_blob_access_creator.go | 30 +- ...s_creator.go => cs_blob_access_creator.go} | 54 +- ...eator.go => cs_blob_replicator_creator.go} | 18 +- .../configuration/new_blob_access.go | 79 ++- .../deadline_enforcing_blob_access.go | 10 - pkg/blobstore/demultiplexing_blob_access.go | 12 - .../demultiplexing_blob_access_test.go | 84 --- .../empty_blob_injecting_blob_access.go | 8 - .../empty_blob_injecting_blob_access_test.go | 47 -- pkg/blobstore/grpcclients/BUILD.bazel | 6 +- pkg/blobstore/grpcclients/cls_blob_access.go | 81 ++- .../{cas_blob_access.go => cs_blob_access.go} | 9 +- ..._access_test.go => cs_blob_access_test.go} | 30 +- pkg/blobstore/grpcservers/BUILD.bazel | 4 + .../grpcservers/byte_stream_server.go | 175 ++--- .../grpcservers/byte_stream_server_test.go | 388 +++++++---- .../content_addressable_storage_server.go | 123 ++-- ...content_addressable_storage_server_test.go | 241 +++++-- .../grpcservers/integration/BUILD.bazel | 34 + .../integration/byte_stream_server_test.go | 50 ++ ...content_addressable_storage_server_test.go | 404 ++++++++++++ .../grpcservers/integration/utils_test.go | 618 ++++++++++++++++++ ...hierarchical_instance_names_blob_access.go | 47 -- ...rchical_instance_names_blob_access_test.go | 59 -- pkg/blobstore/local/BUILD.bazel | 1 - pkg/blobstore/local/flat_blob_access_test.go | 279 -------- pkg/blobstore/metrics_blob_access.go | 18 - pkg/blobstore/mirrored/BUILD.bazel | 1 - .../mirrored/mirrored_blob_access.go | 6 - .../mirrored/mirrored_blob_access_test.go | 25 - pkg/blobstore/read_canarying_blob_access.go | 54 -- .../read_canarying_blob_access_test.go | 28 - pkg/blobstore/readcaching/BUILD.bazel | 1 - .../readcaching/read_caching_blob_access.go | 12 - .../read_caching_blob_access_test.go | 24 - pkg/blobstore/readfallback/BUILD.bazel | 1 - .../readfallback/read_fallback_blob_access.go | 12 - .../read_fallback_blob_access_test.go | 26 - pkg/blobstore/referenceexpanding/BUILD.bazel | 49 ++ .../reference_expanding_blob_access.go | 23 +- .../reference_expanding_blob_access_test.go | 48 +- pkg/blobstore/replication/BUILD.bazel | 1 - pkg/blobstore/replication/blob_replicator.go | 4 - .../concurrency_limiting_blob_replicator.go | 11 - .../deduplicating_blob_replicator.go | 11 - .../deduplicating_blob_replicator_test.go | 29 - .../replication/local_blob_replicator.go | 15 - .../replication/metrics_blob_replicator.go | 18 - .../replication/noop_blob_replicator.go | 5 - .../replication/queued_blob_replicator.go | 10 - .../queued_blob_replicator_test.go | 62 -- .../replication/remote_blob_replicator.go | 15 - .../replication/with_blob_replicator.go | 35 - pkg/blobstore/sharding/BUILD.bazel | 1 - .../sharding/sharding_blob_access.go | 9 - .../sharding/sharding_blob_access_test.go | 15 - pkg/digest/function.go | 3 + pkg/proto/blobstore/chunklist/BUILD.bazel | 24 + .../blobstore/chunklist/chunk_list.pb.go | 124 ++++ .../blobstore/chunklist/chunk_list.proto | 13 + .../bb_replicator/bb_replicator.proto | 4 +- .../configuration/bb_storage/bb_storage.pb.go | 213 ++++-- .../configuration/bb_storage/bb_storage.proto | 34 +- .../configuration/blobstore/blobstore.pb.go | 532 ++++++++------- .../configuration/blobstore/blobstore.proto | 48 +- pkg/testutil/testutil.go | 18 + pkg/ttlcache/BUILD.bazel | 14 + pkg/ttlcache/ttl_cache.go | 97 +++ 108 files changed, 4083 insertions(+), 3026 deletions(-) create mode 100644 pkg/blobstore/cdc/BUILD.bazel create mode 100644 pkg/blobstore/cdc/bypass.go create mode 100644 pkg/blobstore/cdc/caching_parameters_fetcher.go rename pkg/blobstore/{chunklistvalidating => cdc}/chunker.go (91%) create mode 100644 pkg/blobstore/cdc/content_addressable_storage.go create mode 100644 pkg/blobstore/cdc/operations.go create mode 100644 pkg/blobstore/cdc/parameters.go create mode 100644 pkg/blobstore/cdc/put_proto.go create mode 100644 pkg/blobstore/cdc/read_blob_at.go rename pkg/blobstore/{chunklistvalidating => cdc}/reader_chunker.go (95%) rename pkg/blobstore/{chunklistvalidating => cdc}/reader_chunker_test.go (88%) create mode 100644 pkg/blobstore/chunk_list_fetcher.go create mode 100644 pkg/blobstore/chunklist/BUILD.bazel create mode 100644 pkg/blobstore/chunklist/caching_fetcher.go create mode 100644 pkg/blobstore/chunklist/chunk_concatenating_reader.go create mode 100644 pkg/blobstore/chunklist/chunk_fetcher.go create mode 100644 pkg/blobstore/chunklist/chunk_list.go create mode 100644 pkg/blobstore/chunklist/encoding.go delete mode 100644 pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go delete mode 100644 pkg/blobstore/chunklistvalidating/integration/BUILD.bazel delete mode 100644 pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go rename pkg/blobstore/configuration/{cas_blob_access_creator.go => cs_blob_access_creator.go} (71%) rename pkg/blobstore/configuration/{cas_blob_replicator_creator.go => cs_blob_replicator_creator.go} (66%) rename pkg/blobstore/grpcclients/{cas_blob_access.go => cs_blob_access.go} (95%) rename pkg/blobstore/grpcclients/{cas_blob_access_test.go => cs_blob_access_test.go} (95%) create mode 100644 pkg/blobstore/grpcservers/integration/BUILD.bazel create mode 100644 pkg/blobstore/grpcservers/integration/byte_stream_server_test.go create mode 100644 pkg/blobstore/grpcservers/integration/content_addressable_storage_server_test.go create mode 100644 pkg/blobstore/grpcservers/integration/utils_test.go create mode 100644 pkg/blobstore/referenceexpanding/BUILD.bazel rename pkg/blobstore/{ => referenceexpanding}/reference_expanding_blob_access.go (91%) rename pkg/blobstore/{ => referenceexpanding}/reference_expanding_blob_access_test.go (93%) create mode 100644 pkg/proto/blobstore/chunklist/BUILD.bazel create mode 100644 pkg/proto/blobstore/chunklist/chunk_list.pb.go create mode 100644 pkg/proto/blobstore/chunklist/chunk_list.proto create mode 100644 pkg/ttlcache/BUILD.bazel create mode 100644 pkg/ttlcache/ttl_cache.go diff --git a/cmd/bb_copy/main.go b/cmd/bb_copy/main.go index 54e83489e..6d891a639 100644 --- a/cmd/bb_copy/main.go +++ b/cmd/bb_copy/main.go @@ -42,7 +42,7 @@ func main() { grpcClientFactory := grpc.NewBaseClientFactory(grpc.BaseClientDialer, nil, nil, nil) - blobAccessCreator := blobstore_configuration.NewCASBlobAccessCreator( + blobAccessCreator := blobstore_configuration.NewCSBlobAccessCreator( grpcClientFactory, int(configuration.MaximumMessageSizeBytes), bb_zstd.NewPoolFromConfiguration(nil), @@ -68,7 +68,7 @@ func main() { configuration.Replicator, source.BlobAccess, sink, - blobstore_configuration.NewCASBlobReplicatorCreator(grpcClientFactory), + blobstore_configuration.NewCSBlobReplicatorCreator(grpcClientFactory), ) if err != nil { return util.StatusWrap(err, "Failed to create replicator") diff --git a/cmd/bb_replicator/main.go b/cmd/bb_replicator/main.go index 8f21a8020..8e5b9f994 100644 --- a/cmd/bb_replicator/main.go +++ b/cmd/bb_replicator/main.go @@ -33,7 +33,7 @@ func main() { return util.StatusWrap(err, "Failed to apply global configuration options") } - blobAccessCreator := blobstore_configuration.NewCASBlobAccessCreator( + blobAccessCreator := blobstore_configuration.NewCSBlobAccessCreator( grpcClientFactory, int(configuration.MaximumMessageSizeBytes), bb_zstd.NewPoolFromConfiguration(nil), @@ -59,7 +59,7 @@ func main() { configuration.Replicator, source.BlobAccess, sink, - blobstore_configuration.NewCASBlobReplicatorCreator(grpcClientFactory), + blobstore_configuration.NewCSBlobReplicatorCreator(grpcClientFactory), ) if err != nil { return util.StatusWrap(err, "Failed to create replicator") diff --git a/cmd/bb_storage/BUILD.bazel b/cmd/bb_storage/BUILD.bazel index b9d6bc741..230e2d907 100644 --- a/cmd/bb_storage/BUILD.bazel +++ b/cmd/bb_storage/BUILD.bazel @@ -10,10 +10,13 @@ go_library( "//pkg/auth", "//pkg/auth/configuration", "//pkg/blobstore", + "//pkg/blobstore/cdc", "//pkg/blobstore/configuration", "//pkg/blobstore/grpcservers", "//pkg/builder", "//pkg/capabilities", + "//pkg/clock", + "//pkg/digest", "//pkg/global", "//pkg/grpc", "//pkg/program", @@ -21,6 +24,7 @@ go_library( "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", + "//pkg/ttlcache", "//pkg/util", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/cmd/bb_storage/main.go b/cmd/bb_storage/main.go index 92e3650f0..9a7c7e288 100644 --- a/cmd/bb_storage/main.go +++ b/cmd/bb_storage/main.go @@ -9,6 +9,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/auth" auth_configuration "github.com/buildbarn/bb-storage/pkg/auth/configuration" "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" "github.com/buildbarn/bb-storage/pkg/builder" @@ -54,55 +55,57 @@ func main() { var cacheCapabilitiesAuthorizers []auth.Authorizer // Content Addressable Storage (CAS). - var contentAddressableStorageInfo *blobstore_configuration.BlobAccessInfo - var contentAddressableStorage blobstore.BlobAccess - if configuration.ContentAddressableStorage != nil { - info, authorizedBackend, allAuthorizers, err := newScannableBlobAccess( + var contentAddressableStorage cdc.ContentAddressableStorage + var authorizedContentAddressableStorage cdc.ContentAddressableStorage + if configuration.ContentAddressableStorageServer != nil { + cas, chunkStorage, chunkListStorage, _, cdcParametersFetcher, err := blobstore_configuration.NewCASFromConfiguration( dependenciesGroup, - configuration.ContentAddressableStorage, - blobstore_configuration.NewCASBlobAccessCreator( - grpcClientFactory, - int(configuration.MaximumMessageSizeBytes), - zstdPool, - ), + configuration.ContentAddressableStorageServer.ContentAddressableStorage, grpcClientFactory, + int(configuration.MaximumMessageSizeBytes), + zstdPool, ) if err != nil { return util.StatusWrap(err, "Failed to create Content Addressable Storage") } + contentAddressableStorage = cas + + // Create authorizers. + getAuthorizer, err := auth_configuration.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(configuration.ContentAddressableStorageServer.GetAuthorizer, dependenciesGroup, grpcClientFactory) + if err != nil { + return util.StatusWrap(err, "Failed to create Get() authorizer for Content Addressable Storage") + } + putAuthorizer, err := auth_configuration.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(configuration.ContentAddressableStorageServer.PutAuthorizer, dependenciesGroup, grpcClientFactory) + if err != nil { + return util.StatusWrap(err, "Failed to create Put() authorizer for Content Addressable Storage") + } + findMissingAuthorizer, err := auth_configuration.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(configuration.ContentAddressableStorageServer.FindMissingAuthorizer, dependenciesGroup, grpcClientFactory) + if err != nil { + return util.StatusWrap(err, "Failed to create FindMissing() authorizer for Content Addressable Storage") + } + + // Create authorized versions of the backends. + authorizedChunkStorage := blobstore.NewAuthorizingBlobAccess(chunkStorage, getAuthorizer, putAuthorizer, findMissingAuthorizer) + authorizedChunkListStorage := blobstore.NewAuthorizingBlobAccess(chunkListStorage, getAuthorizer, putAuthorizer, findMissingAuthorizer) + authorizedChunkListFetcher := blobstore.NewBlobAccessChunkListFetcher(authorizedChunkListStorage, int(configuration.MaximumMessageSizeBytes)) + authorizedContentAddressableStorage = cdc.NewContentAddressableStorage( + authorizedChunkStorage, + authorizedChunkListStorage, + authorizedChunkListFetcher, + cdcParametersFetcher, + contentAddressableStorage.GetDigestKeyFormat(), + ) + // Create the Chunk Storage (CS). cacheCapabilitiesProviders = append( cacheCapabilitiesProviders, - info.BlobAccess, + chunkStorage, capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ CacheCapabilities: &remoteexecution.CacheCapabilities{ SupportedCompressors: configuration.SupportedCompressors, }, }), ) - cacheCapabilitiesAuthorizers = append(cacheCapabilitiesAuthorizers, allAuthorizers...) - contentAddressableStorageInfo = &info - contentAddressableStorage = authorizedBackend - } - - // Chunk List Storage (CLS). - var chunkListStorage blobstore.BlobAccess - if configuration.ChunkListStorage != nil { - info, authorizedBackend, allAuthorizers, err := newScannableBlobAccess( - dependenciesGroup, - configuration.ChunkListStorage, - blobstore_configuration.NewCLSBlobAccessCreator( - contentAddressableStorageInfo, - grpcClientFactory, - int(configuration.MaximumMessageSizeBytes), - ), - grpcClientFactory, - ) - if err != nil { - return util.StatusWrap(err, "Failed to create Chunk List Storage") - } - cacheCapabilitiesProviders = append(cacheCapabilitiesProviders, info.BlobAccess) - cacheCapabilitiesAuthorizers = append(cacheCapabilitiesAuthorizers, allAuthorizers...) - chunkListStorage = authorizedBackend + cacheCapabilitiesAuthorizers = append(cacheCapabilitiesAuthorizers, getAuthorizer, putAuthorizer, findMissingAuthorizer) } // Action Cache (AC). @@ -112,7 +115,7 @@ func main() { dependenciesGroup, configuration.ActionCache, blobstore_configuration.NewACBlobAccessCreator( - contentAddressableStorageInfo, + contentAddressableStorage, grpcClientFactory, int(configuration.MaximumMessageSizeBytes), ), @@ -213,10 +216,9 @@ func main() { if err := bb_grpc.NewServersFromConfigurationAndServe( configuration.GrpcServers, func(s grpc.ServiceRegistrar) { - if contentAddressableStorage != nil { + if authorizedContentAddressableStorage != nil { contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer( - contentAddressableStorage, - chunkListStorage, + authorizedContentAddressableStorage, configuration.MaximumMessageSizeBytes, ) remoteexecution.RegisterContentAddressableStorageServer( @@ -226,8 +228,7 @@ func main() { bytestream.RegisterByteStreamServer( s, grpcservers.NewByteStreamServer( - contentAddressableStorage, - 1<<16, + authorizedContentAddressableStorage, zstdPool, ), ) diff --git a/internal/mock/BUILD.bazel b/internal/mock/BUILD.bazel index 94a9a7706..8211c444a 100644 --- a/internal/mock/BUILD.bazel +++ b/internal/mock/BUILD.bazel @@ -152,6 +152,19 @@ gomock( package = "mock", ) +gomock( + name = "cdc", + out = "cdc.go", + interfaces = [ + "ContentAddressableStorage", + "ParametersFetcher", + ], + library = "//pkg/blobstore/cdc", + mockgen_model_library = "@org_uber_go_mock//mockgen/model", + mockgen_tool = "@org_uber_go_mock//mockgen", + package = "mock", +) + gomock( name = "clock", out = "clock.go", @@ -377,6 +390,7 @@ go_library( "buffer.go", "builder.go", "capabilities.go", + "cdc.go", "clock.go", "cloud_aws.go", "cloud_gcp.go", @@ -402,6 +416,8 @@ go_library( "//pkg/auth", "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/blobstore/local", "//pkg/blobstore/sharding", "//pkg/blobstore/slicing", diff --git a/pkg/blobstore/BUILD.bazel b/pkg/blobstore/BUILD.bazel index 2cd9368ef..f1a04c7f9 100644 --- a/pkg/blobstore/BUILD.bazel +++ b/pkg/blobstore/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "blob_access.go", "blob_access_message_reader.go", "cas_read_buffer_factory.go", + "chunk_list_fetcher.go", "cls_read_buffer_factory.go", "deadline_enforcing_blob_access.go", "demultiplexing_blob_access.go", @@ -23,7 +24,6 @@ go_library( "metrics_blob_access.go", "read_buffer_factory.go", "read_canarying_blob_access.go", - "reference_expanding_blob_access.go", "validation_caching_read_buffer_factory.go", "visit_topologically_sorted_tree.go", "zip_reading_blob_access.go", @@ -34,22 +34,19 @@ go_library( deps = [ "//pkg/auth", "//pkg/blobstore/buffer", + "//pkg/blobstore/chunklist", "//pkg/blobstore/slicing", "//pkg/capabilities", "//pkg/clock", - "//pkg/cloud/aws", - "//pkg/cloud/gcp", "//pkg/digest", "//pkg/eviction", + "//pkg/proto/blobstore/chunklist", "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", "//pkg/storage", "//pkg/util", - "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", - "@com_github_aws_aws_sdk_go_v2//aws", - "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", "@com_github_prometheus_client_golang//prometheus", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", @@ -70,7 +67,6 @@ go_test( "existence_caching_blob_access_test.go", "hierarchical_instance_names_blob_access_test.go", "read_canarying_blob_access_test.go", - "reference_expanding_blob_access_test.go", "validation_caching_read_buffer_factory_test.go", "visit_topologically_sorted_tree_test.go", "zip_reading_blob_access_test.go", @@ -82,16 +78,10 @@ go_test( "//pkg/blobstore/buffer", "//pkg/digest", "//pkg/eviction", - "//pkg/proto/icas", "//pkg/testutil", "//pkg/util", - "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", "@bazel_remote_apis//build/bazel/semver:semver_go_proto", - "@com_github_aws_aws_sdk_go_v2//aws", - "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", - "@com_github_aws_aws_sdk_go_v2_service_s3//types", - "@com_github_klauspost_compress//zstd", "@com_github_stretchr_testify//require", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", diff --git a/pkg/blobstore/authorizing_blob_access.go b/pkg/blobstore/authorizing_blob_access.go index 25898bdd4..cdb3b480d 100644 --- a/pkg/blobstore/authorizing_blob_access.go +++ b/pkg/blobstore/authorizing_blob_access.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/auth" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -38,13 +37,6 @@ func (ba *authorizingBlobAccess) Get(ctx context.Context, d digest.Digest) buffe return ba.BlobAccess.Get(ctx, d) } -func (ba *authorizingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - if err := auth.AuthorizeSingleInstanceName(ctx, ba.getAuthorizer, parentDigest.GetInstanceName()); err != nil { - return buffer.NewBufferFromError(util.StatusWrap(err, "Authorization")) - } - return ba.BlobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer) -} - func (ba *authorizingBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { if err := auth.AuthorizeSingleInstanceName(ctx, ba.putAuthorizer, d.GetInstanceName()); err != nil { return util.StatusWrap(err, "Authorization") diff --git a/pkg/blobstore/authorizing_blob_access_test.go b/pkg/blobstore/authorizing_blob_access_test.go index a13d994dd..6323daec5 100644 --- a/pkg/blobstore/authorizing_blob_access_test.go +++ b/pkg/blobstore/authorizing_blob_access_test.go @@ -54,24 +54,6 @@ func TestAuthorizingBlobAccess(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.PermissionDenied, "Authorization: You shall not pass"), err) }) - t.Run("GetFromComposite-Allowed", func(t *testing.T) { - getAuthorizer.EXPECT().Authorize(ctx, beepSlice).Return([]error{nil}) - blobSlicer := mock.NewMockBlobSlicer(ctrl) - baseBlobAccess.EXPECT().GetFromComposite(ctx, d, d2, blobSlicer).Return(wantBuf) - - gotBuf, err := ba.GetFromComposite(ctx, d, d2, blobSlicer).ToByteSlice(30) - require.NoError(t, err) - require.Equal(t, wantBytes, gotBuf) - }) - - t.Run("GetFromComposite-Denied", func(t *testing.T) { - blobSlicer := mock.NewMockBlobSlicer(ctrl) - getAuthorizer.EXPECT().Authorize(ctx, beepSlice).Return([]error{status.Error(codes.PermissionDenied, "You shall not pass")}) - - _, err := ba.GetFromComposite(ctx, d, d2, blobSlicer).ToByteSlice(30) - testutil.RequireEqualStatus(t, status.Error(codes.PermissionDenied, "Authorization: You shall not pass"), err) - }) - t.Run("Put-Allowed", func(t *testing.T) { putAuthorizer.EXPECT().Authorize(ctx, beepSlice).Return([]error{nil}) baseBlobAccess.EXPECT().Put(ctx, d, wantBuf).Return(nil) diff --git a/pkg/blobstore/blob_access.go b/pkg/blobstore/blob_access.go index 09d6ba963..2f2ed19dc 100644 --- a/pkg/blobstore/blob_access.go +++ b/pkg/blobstore/blob_access.go @@ -4,7 +4,6 @@ import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/capabilities" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -16,7 +15,6 @@ type BlobAccess interface { capabilities.Provider Get(ctx context.Context, digest digest.Digest) buffer.Buffer - GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) } diff --git a/pkg/blobstore/cas_read_buffer_factory.go b/pkg/blobstore/cas_read_buffer_factory.go index 2832352b8..b80cf0869 100644 --- a/pkg/blobstore/cas_read_buffer_factory.go +++ b/pkg/blobstore/cas_read_buffer_factory.go @@ -1,15 +1,12 @@ package blobstore import ( - "context" "io" - "math" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" "google.golang.org/protobuf/encoding/protowire" - "google.golang.org/protobuf/proto" ) type casReadBufferFactory struct{} @@ -30,33 +27,6 @@ func (f casReadBufferFactory) NewBufferFromReaderAt(digest digest.Digest, r buff // stored in the Content Addressable Storage (CAS). var CASReadBufferFactory ReadBufferFactory = casReadBufferFactory{} -// CASPutProto is a helper function for storing Protobuf messages in the -// Content Addressable Storage (CAS). It computes the digest of the -// message and stores it under that key. The digest is then returned, so -// that the object may be referenced. -func CASPutProto(ctx context.Context, blobAccess BlobAccess, message proto.Message, digestFunction digest.Function) (digest.Digest, error) { - b := buffer.NewProtoBufferFromProto(message, buffer.UserProvided) - sizeBytes, err := b.GetSizeBytes() - if err != nil { - b.Discard() - return digest.BadDigest, err - } - - // Compute new digest of data. - digestGenerator := digestFunction.NewGenerator(sizeBytes) - bDigest, bPut := b.CloneCopy(math.MaxInt) - if err := bDigest.IntoWriter(digestGenerator); err != nil { - bPut.Discard() - return digest.BadDigest, err - } - blobDigest := digestGenerator.Sum() - - if err := blobAccess.Put(ctx, blobDigest, bPut); err != nil { - return digest.BadDigest, err - } - return blobDigest, nil -} - // The Protobuf field numbers of the REv2 Tree's "root" and "children" // fields. These are used in combination with util.VisitProtoBytesFields() // to be able to process REv2 Tree objects in a streaming manner. diff --git a/pkg/blobstore/cdc/BUILD.bazel b/pkg/blobstore/cdc/BUILD.bazel new file mode 100644 index 000000000..41be1685d --- /dev/null +++ b/pkg/blobstore/cdc/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "cdc", + srcs = [ + "bypass.go", + "caching_parameters_fetcher.go", + "chunker.go", + "content_addressable_storage.go", + "operations.go", + "parameters.go", + "put_proto.go", + "read_blob_at.go", + "reader_chunker.go", + ], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/cdc", + visibility = ["//visibility:public"], + deps = [ + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/blobstore/chunklist", + "//pkg/capabilities", + "//pkg/digest", + "//pkg/ttlcache", + "//pkg/util", + "@com_github_buildbarn_go_cdc//:go-cdc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + "@org_golang_google_protobuf//proto", + ], +) + +go_test( + name = "cdc_test", + srcs = ["reader_chunker_test.go"], + deps = [ + ":cdc", + "//pkg/blobstore/buffer", + "//pkg/digest", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_stretchr_testify//require", + ], +) diff --git a/pkg/blobstore/cdc/bypass.go b/pkg/blobstore/cdc/bypass.go new file mode 100644 index 000000000..1564d4e85 --- /dev/null +++ b/pkg/blobstore/cdc/bypass.go @@ -0,0 +1,22 @@ +package cdc + +import "context" + +type chunkListValidationBypassKey struct{} + +// NewContextWithChunkListValidationBypass creates a derived context +// that signals downstream storage layers that the chunk list has +// already been validated (or was freshly generated) and does not need +// expensive re-validation. +func NewContextWithChunkListValidationBypass(ctx context.Context) context.Context { + return context.WithValue(ctx, chunkListValidationBypassKey{}, chunkListValidationBypassKey{}) +} + +// ChunkListValidationBypassed checks if the provided context contains +// the bypass signal. +func ChunkListValidationBypassed(ctx context.Context) bool { + if value := ctx.Value(chunkListValidationBypassKey{}); value != nil { + return true + } + return false +} diff --git a/pkg/blobstore/cdc/caching_parameters_fetcher.go b/pkg/blobstore/cdc/caching_parameters_fetcher.go new file mode 100644 index 000000000..98c7b46c6 --- /dev/null +++ b/pkg/blobstore/cdc/caching_parameters_fetcher.go @@ -0,0 +1,33 @@ +package cdc + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/ttlcache" +) + +type cachingParametersFetcher struct { + ParametersFetcher + cache *ttlcache.TTLCache[digest.InstanceName, Parameters] +} + +// NewCachingParametersFetcher decorates a ParametersFetcher with caching. +func NewCachingParametersFetcher(base ParametersFetcher, ttlCache *ttlcache.TTLCache[digest.InstanceName, Parameters]) ParametersFetcher { + return &cachingParametersFetcher{ + ParametersFetcher: base, + cache: ttlCache, + } +} + +func (f *cachingParametersFetcher) FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) { + if v, ok := f.cache.Get(instanceName); ok { + return v, nil + } + params, err := f.ParametersFetcher.FetchCDCParameters(ctx, instanceName) + if err != nil { + return Parameters{}, err + } + f.cache.Put(instanceName, params) + return params, nil +} diff --git a/pkg/blobstore/chunklistvalidating/chunker.go b/pkg/blobstore/cdc/chunker.go similarity index 91% rename from pkg/blobstore/chunklistvalidating/chunker.go rename to pkg/blobstore/cdc/chunker.go index 8badbc7e9..ac02682db 100644 --- a/pkg/blobstore/chunklistvalidating/chunker.go +++ b/pkg/blobstore/cdc/chunker.go @@ -1,4 +1,4 @@ -package chunklistvalidating +package cdc import ( "github.com/buildbarn/bb-storage/pkg/digest" diff --git a/pkg/blobstore/cdc/content_addressable_storage.go b/pkg/blobstore/cdc/content_addressable_storage.go new file mode 100644 index 000000000..4ae510f94 --- /dev/null +++ b/pkg/blobstore/cdc/content_addressable_storage.go @@ -0,0 +1,129 @@ +package cdc + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" +) + +type contentAddressableStorage struct { + chunkStorage blobstore.BlobAccess + chunkListFetcher chunklist.Fetcher + cdcParametersFetcher ParametersFetcher + chunkListStorage blobstore.BlobAccess + digestKeyFormat digest.KeyFormat +} + +// ContentAddressableStorage is an interface which describes +// interactions with a Content Addressable Storage (CAS). +type ContentAddressableStorage interface { + // GetDigestKeyFormat returns the combined digest key format of the + // CS and CLS. + GetDigestKeyFormat() digest.KeyFormat + // FetchCDCParameters fetches the CDC parameters of the CAS. + FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) + // FindMissing returns the set of digests that are not present in + // the CAS. + FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) + // FetchChunk fetches the contents of a specific chunk. + FetchChunk(ctx context.Context, d digest.Digest) ([]byte, error) + // PutChunk stores the contents of a specific chunk. + PutChunk(ctx context.Context, d digest.Digest, data []byte) error + // GetManifest returns the chunk list which constitutes a specific + // blob. + GetManifest(ctx context.Context, d digest.Digest) (chunklist.ChunkList, error) + // PutManifest stores the chunk list which constitutes a specific + // blob. + PutManifest(ctx context.Context, d digest.Digest, manifest chunklist.ChunkList) error +} + +// ContentAddressableStorage implements the chunklist.ChunkFetcher +// interface +var _ chunklist.ChunkFetcher = ContentAddressableStorage(nil) + +// ContentAddressableStorage implements the ParametersFetcher interface. +var _ ParametersFetcher = ContentAddressableStorage(nil) + +// NewContentAddressableStorage creates an interface towards the Content +// Addressable Storage (CAS) that allows access to arbitrary sized +// objects from the chunks described in the chunk list. +func NewContentAddressableStorage(chunkStorage, chunkListStorage blobstore.BlobAccess, chunkListFetcher chunklist.Fetcher, cdcParametersFetcher ParametersFetcher, digestKeyFormat digest.KeyFormat) ContentAddressableStorage { + return &contentAddressableStorage{ + chunkStorage: chunkStorage, + chunkListStorage: chunkListStorage, + chunkListFetcher: chunkListFetcher, + cdcParametersFetcher: cdcParametersFetcher, + digestKeyFormat: digestKeyFormat, + } +} + +func (cas *contentAddressableStorage) GetDigestKeyFormat() digest.KeyFormat { + return cas.digestKeyFormat +} + +func (cas *contentAddressableStorage) FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) { + return cas.cdcParametersFetcher.FetchCDCParameters(ctx, instanceName) +} + +func (cas *contentAddressableStorage) FetchChunk(ctx context.Context, d digest.Digest) ([]byte, error) { + return cas.chunkStorage.Get(ctx, d).ToByteSlice(int(d.GetSizeBytes())) +} + +func (cas *contentAddressableStorage) PutChunk(ctx context.Context, d digest.Digest, data []byte) error { + return cas.chunkStorage.Put(ctx, d, buffer.NewCASBufferFromByteSlice(d, data, buffer.UserProvided)) +} + +func (cas *contentAddressableStorage) GetManifest(ctx context.Context, d digest.Digest) (chunklist.ChunkList, error) { + return cas.chunkListFetcher.FetchChunkList(ctx, d) +} + +func (cas *contentAddressableStorage) PutManifest(ctx context.Context, d digest.Digest, manifest chunklist.ChunkList) error { + b := buffer.NewProtoBufferFromProto(chunklist.ToProto(manifest), buffer.UserProvided) + if err := cas.chunkListStorage.Put(ctx, d, b); err != nil { + return util.StatusWrap(err, "Could not save chunk list for blob") + } + return nil +} + +func (cas *contentAddressableStorage) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { + digestSets := digests.PartitionByInstanceName() + missings := make([]digest.Set, 0, len(digestSets)) + for _, digestSet := range digestSets { + // PartitionByInstanceNames guarantees non empty sets. + missing, err := cas.findMissingFromInstance(ctx, digestSet.Items()[0].GetInstanceName(), digestSet) + if err != nil { + return digest.EmptySet, err + } + missings = append(missings, missing) + } + return digest.GetUnion(missings), nil +} + +func (cas *contentAddressableStorage) findMissingFromInstance(ctx context.Context, instanceName digest.InstanceName, digests digest.Set) (digest.Set, error) { + params, err := cas.cdcParametersFetcher.FetchCDCParameters(ctx, instanceName) + if err != nil { + return digest.EmptySet, err + } + smallDigests := digest.NewSetBuilder(digests.Length()) + largeDigests := digest.NewSetBuilder(digests.Length()) + for _, d := range digests.Items() { + if IsSingleChunk(params, d) { + smallDigests.Add(d) + } else { + largeDigests.Add(d) + } + } + smallMissing, err := cas.chunkStorage.FindMissing(ctx, smallDigests.Build()) + if err != nil { + return digest.EmptySet, err + } + largeMissing, err := cas.chunkListStorage.FindMissing(ctx, largeDigests.Build()) + if err != nil { + return digest.EmptySet, err + } + return digest.GetUnion([]digest.Set{smallMissing, largeMissing}), nil +} diff --git a/pkg/blobstore/cdc/operations.go b/pkg/blobstore/cdc/operations.go new file mode 100644 index 000000000..214a049a9 --- /dev/null +++ b/pkg/blobstore/cdc/operations.go @@ -0,0 +1,213 @@ +package cdc + +import ( + "bytes" + "context" + "io" + "runtime/debug" + + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// IsSingleChunk checks if a digest is represented by a single chunk. +func IsSingleChunk(params Parameters, d digest.Digest) bool { + return d.GetSizeBytes() < 2*params.MinChunkSizeBytes +} + +// GetBytes returns the bytes of a digest from the CAS as a byteslice. +func GetBytes(ctx context.Context, cas ContentAddressableStorage, d digest.Digest) ([]byte, error) { + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch CDC parameters") + } + if IsSingleChunk(params, d) { + return cas.FetchChunk(ctx, d) + } + manifest, err := cas.GetManifest(ctx, d) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch chunk list") + } + ret := make([]byte, d.GetSizeBytes()) + for _, chunk := range manifest { + chunkBytes, err := cas.FetchChunk(ctx, chunk.Digest) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch chunk") + } + copy(ret[chunk.Offset:], chunkBytes) + } + return ret, nil +} + +// GetReadCloser returns an io.ReadCloser that reads contents of a blob +// in CAS. +func GetReadCloser(ctx context.Context, cas ContentAddressableStorage, d digest.Digest) (io.ReadCloser, error) { + return GetReadCloserAt(ctx, cas, d, 0) +} + +// GetReadCloserAt returns an io.ReadCloser that reads contents of a +// blob in seeked to a specific offset. +func GetReadCloserAt(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, offset int64) (io.ReadCloser, error) { + if offset < 0 || offset > d.GetSizeBytes() { + return nil, status.Errorf(codes.InvalidArgument, "Offset %d is outside of blob %s", offset, d) + } + + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch CDC parameters") + } + if IsSingleChunk(params, d) { + chunk, err := cas.FetchChunk(ctx, d) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(chunk[offset:])), nil + } + + manifest, err := cas.GetManifest(ctx, d) + if err != nil { + return nil, util.StatusWrap(err, "Could not fetch chunk list") + } + index, chunkOffset := chunklist.FindChunkOffset(manifest, uint64(offset)) + if index >= len(manifest) { + // Offset at the end of the blob. + return io.NopCloser(bytes.NewReader(nil)), nil + } + r := chunklist.NewChunkConcatenatingReader(ctx, manifest[index:], cas) + if chunkOffset > 0 { + if _, err := io.CopyN(io.Discard, r, chunkOffset); err != nil { + return nil, util.StatusWrap(err, "Failed to skip to read offset") + } + } + return r, nil +} + +// PutBytes inserts all chunks for a digest from it's raw bytes. +func PutBytes(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, data []byte) error { + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return util.StatusWrap(err, "Could not fetch CDC parameters") + } + if IsSingleChunk(params, d) { + return cas.PutChunk(ctx, d, data) + } + return PutReader(ctx, cas, d, bytes.NewReader(data)) +} + +// PutReader inserts all chunks for a digest from an io.Reader. +func PutReader(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, r io.Reader) error { + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return util.StatusWrap(err, "Could not fetch CDC parameters") + } + + digestFunction := d.GetDigestFunction() + chunker := NewReaderChunker(digestFunction, r, params.MinChunkSizeBytes, params.HorizonSizeBytes) + wholeGen := digestFunction.NewGenerator(d.GetSizeBytes()) + + chunkList := make(chunklist.ChunkList, 0, d.GetSizeBytes()/params.MinChunkSizeBytes+1) + var offset uint64 + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + if err != nil { + return util.StatusWrap(err, "Failed to chunk write stream") + } + + if _, err := wholeGen.Write(chunk.Data); err != nil { + return status.Error(codes.Internal, "Could not compute digest of blob") + } + + if err := cas.PutChunk(ctx, chunk.Digest, chunk.Data); err != nil { + return util.StatusWrap(err, "Failed to save chunk") + } + + chunkList = append(chunkList, chunklist.Entry{Offset: offset, Digest: chunk.Digest}) + offset += uint64(chunk.Digest.GetSizeBytes()) + } + + // Verify the whole blob against the advertised digest. + if actual := wholeGen.Sum(); actual != d { + debug.PrintStack() + return status.Errorf(codes.InvalidArgument, "Blob digest mismatch: advertised %s, actual %s", d, actual) + } + + // A single chunk is the trivial case: it already lives in the + // chunk storage and needs no chunk list. + if len(chunkList) <= 1 { + return nil + } + + // We generated and validated this chunk list ourselves. + ctx = NewContextWithChunkListValidationBypass(ctx) + if err := cas.PutManifest(ctx, d, chunkList); err != nil { + return util.StatusWrap(err, "Could not save chunk list for blob") + } + return nil +} + +// GetProto retrieves a proto message from the CAS unmarshalling it into +// the supplied message object +func GetProto[T proto.Message](ctx context.Context, cas ContentAddressableStorage, d digest.Digest, message T) (T, error) { + var zero T + bytes, err := GetBytes(ctx, cas, d) + if err != nil { + return zero, err + } + err = proto.Unmarshal(bytes, message) + if err != nil { + return zero, util.StatusWrapWithCode(err, codes.InvalidArgument, "Failed to unmarshal message") + } + return message, nil +} + +// IntoWriter streams the chunks of a blob directly into the provided +// io.Writer from the specified offset. +func IntoWriter(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, offset int64, w io.Writer) error { + if offset < 0 || offset > d.GetSizeBytes() { + return status.Errorf(codes.InvalidArgument, "Invalid offset %d for digest %s", offset, d) + } + + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return util.StatusWrapf(err, "Failed to fetch CDC parameters for instance %s", d.GetInstanceName()) + } + + if IsSingleChunk(params, d) { + chunkData, err := cas.FetchChunk(ctx, d) + if err != nil { + return err + } + if _, err := w.Write(chunkData[offset:]); err != nil { + return err + } + return nil + } + + manifest, err := cas.GetManifest(ctx, d) + if err != nil { + return err + } + index, chunkOffset := chunklist.FindChunkOffset(manifest, uint64(offset)) + for ; index < len(manifest); index++ { + chunk := manifest[index] + chunkData, err := cas.FetchChunk(ctx, chunk.Digest) + if err != nil { + return err + } + + if _, err := w.Write(chunkData[chunkOffset:]); err != nil { + return err + } + chunkOffset = 0 + } + + return nil +} diff --git a/pkg/blobstore/cdc/parameters.go b/pkg/blobstore/cdc/parameters.go new file mode 100644 index 000000000..a84cd3515 --- /dev/null +++ b/pkg/blobstore/cdc/parameters.go @@ -0,0 +1,59 @@ +package cdc + +import ( + "context" + "math" + + "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Parameters are the parameters by which to perform Content Defined +// Chunking (CDC). +type Parameters struct { + MinChunkSizeBytes int64 + HorizonSizeBytes int64 +} + +// ParametersFetcher retrieves the Content Defined Chunking (CDC) +// parameters that govern how blobs are decomposed into chunks. +type ParametersFetcher interface { + FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) +} + +type capabilitiesParametersFetcher struct { + provider capabilities.Provider +} + +// NewCapabilitiesParametersFetcher creates a ParametersFetcher that +// obtains the CDC parameters by calling GetCapabilities() on the +// provided capabilities.Provider. +func NewCapabilitiesParametersFetcher(provider capabilities.Provider) ParametersFetcher { + return &capabilitiesParametersFetcher{ + provider: provider, + } +} + +func (f *capabilitiesParametersFetcher) FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) { + capabilities, err := f.provider.GetCapabilities(ctx, instanceName) + if err != nil { + return Parameters{}, util.StatusWrap(err, "Unable to GetCapabilities to determine chunking parameters") + } + + params := capabilities.CacheCapabilities.GetRepMaxCdcParams() + if params == nil { + return Parameters{}, status.Error(codes.Unimplemented, "This backend only supports storage backends with RepMaxCDC support") + } + if params.MinChunkSizeBytes < 64 { + return Parameters{}, status.Errorf(codes.Internal, "RepMaxCDC minimum chunk size was %d bytes but a minimum of 64 bytes is required", params.MinChunkSizeBytes) + } + if params.MinChunkSizeBytes > math.MaxInt64 || params.HorizonSizeBytes > math.MaxInt64 { + return Parameters{}, status.Error(codes.Internal, "CDC Parameters could not be represented as int64") + } + + return Parameters{MinChunkSizeBytes: int64(params.MinChunkSizeBytes), HorizonSizeBytes: int64(params.HorizonSizeBytes)}, nil +} diff --git a/pkg/blobstore/cdc/put_proto.go b/pkg/blobstore/cdc/put_proto.go new file mode 100644 index 000000000..82dd45226 --- /dev/null +++ b/pkg/blobstore/cdc/put_proto.go @@ -0,0 +1,29 @@ +package cdc + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + + "google.golang.org/protobuf/proto" +) + +// PutProto is a helper function for storing Protobuf messages in the +// Content Addressable Storage (CAS). It computes the digest of the +// message and stores it under that key. The digest is then returned, so +// that the object may be referenced. +func PutProto(ctx context.Context, contentAddressableStorage ContentAddressableStorage, message proto.Message, digestFunction digest.Function) (digest.Digest, error) { + bytes, err := proto.Marshal(message) + if err != nil { + return digest.BadDigest, err + } + digestGenerator := digestFunction.NewGenerator(int64(len(bytes))) + if _, err := digestGenerator.Write(bytes); err != nil { + return digest.BadDigest, err + } + blobDigest := digestGenerator.Sum() + if err := PutBytes(ctx, contentAddressableStorage, blobDigest, bytes); err != nil { + return digest.BadDigest, err + } + return blobDigest, nil +} diff --git a/pkg/blobstore/cdc/read_blob_at.go b/pkg/blobstore/cdc/read_blob_at.go new file mode 100644 index 000000000..fcfdfd5ec --- /dev/null +++ b/pkg/blobstore/cdc/read_blob_at.go @@ -0,0 +1,68 @@ +package cdc + +import ( + "context" + "io" + + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ReadBlobAt fills buf with the contents of a blob starting at the +// given offset, following the semantics of io.ReaderAt: it returns +// the number of bytes read and io.EOF when the read reaches the end +// of the blob before filling buf completely. +// +// Blobs stored as a single chunk are read with a single Chunk Storage +// fetch. For larger blobs only the chunks covering the requested +// range are fetched, so random access reads do not stream the full +// blob. +func ReadBlobAt(ctx context.Context, cas ContentAddressableStorage, d digest.Digest, buf []byte, offset int64) (int, error) { + if offset < 0 || offset > d.GetSizeBytes() { + return 0, status.Errorf(codes.InvalidArgument, "Offset %d is outside of blob %s", offset, d) + } + if len(buf) == 0 { + return 0, nil + } + + params, err := cas.FetchCDCParameters(ctx, d.GetInstanceName()) + if err != nil { + return 0, util.StatusWrap(err, "Could not fetch CDC parameters") + } + if IsSingleChunk(params, d) { + chunk, err := cas.FetchChunk(ctx, d) + if err != nil { + return 0, err + } + n := copy(buf, chunk[offset:]) + if n < len(buf) { + return n, io.EOF + } + return n, nil + } + + manifest, err := cas.GetManifest(ctx, d) + if err != nil { + return 0, util.StatusWrap(err, "Could not fetch chunk list") + } + index, chunkOffset := chunklist.FindChunkOffset(manifest, uint64(offset)) + + n := 0 + for n < len(buf) && index < len(manifest) { + chunkBytes, err := cas.FetchChunk(ctx, manifest[index].Digest) + if err != nil { + return n, util.StatusWrap(err, "Could not fetch chunk") + } + n += copy(buf[n:], chunkBytes[chunkOffset:]) + chunkOffset = 0 + index++ + } + if n < len(buf) { + return n, io.EOF + } + return n, nil +} diff --git a/pkg/blobstore/chunklistvalidating/reader_chunker.go b/pkg/blobstore/cdc/reader_chunker.go similarity index 95% rename from pkg/blobstore/chunklistvalidating/reader_chunker.go rename to pkg/blobstore/cdc/reader_chunker.go index 0d250dabd..ad7717d72 100644 --- a/pkg/blobstore/chunklistvalidating/reader_chunker.go +++ b/pkg/blobstore/cdc/reader_chunker.go @@ -1,4 +1,4 @@ -package chunklistvalidating +package cdc import ( "bufio" @@ -12,7 +12,6 @@ import ( type readerChunker struct { cdcChunker cdc.ContentDefinedChunker - reader io.Reader digestFunction digest.Function } @@ -47,7 +46,6 @@ func NewReaderChunker(digestFunction digest.Function, reader io.Reader, minChunk int(minChunkSizeBytes), int(horizonSizeBytes), ), - reader, digestFunction, } } diff --git a/pkg/blobstore/chunklistvalidating/reader_chunker_test.go b/pkg/blobstore/cdc/reader_chunker_test.go similarity index 88% rename from pkg/blobstore/chunklistvalidating/reader_chunker_test.go rename to pkg/blobstore/cdc/reader_chunker_test.go index c5630d427..d92778bbd 100644 --- a/pkg/blobstore/chunklistvalidating/reader_chunker_test.go +++ b/pkg/blobstore/cdc/reader_chunker_test.go @@ -1,4 +1,4 @@ -package chunklistvalidating_test +package cdc_test import ( "io" @@ -7,7 +7,7 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/stretchr/testify/require" ) @@ -32,7 +32,7 @@ func FuzzReaderChunker(f *testing.F) { reader := buffer.NewValidatedBufferFromByteSlice(originalData).ToReader() defer reader.Close() - chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + chunker := cdc.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) composedData := make([]byte, 0, dataSizeBytes) var numberOfChunks int @@ -76,7 +76,7 @@ func TestReaderChunkerSmallBlob(t *testing.T) { defer reader.Close() digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) - chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + chunker := cdc.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) chunks := make([][]byte, 0, 1) for { @@ -99,7 +99,7 @@ func TestReaderChunkerEmptyBlob(t *testing.T) { defer reader.Close() digestFunc := digest.MustNewFunction("", remoteexecution.DigestFunction_SHA256) - chunker := chunklistvalidating.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) + chunker := cdc.NewReaderChunker(digestFunc, reader, minChunkSize, horizonLookaheadBytes) chunk, err := chunker.NextChunk() require.ErrorIs(t, io.EOF, err) diff --git a/pkg/blobstore/chunk_list_fetcher.go b/pkg/blobstore/chunk_list_fetcher.go new file mode 100644 index 000000000..0817e2e3c --- /dev/null +++ b/pkg/blobstore/chunk_list_fetcher.go @@ -0,0 +1,33 @@ +package blobstore + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/util" +) + +type blobAccessChunkListFetcher struct { + chunkListStorage BlobAccess + maximumMessageSizeBytes int +} + +// NewBlobAccessChunkListFetcher creates a ChunkListFetcher that reads +// and decodes chunk lists from the provided BlobAccess. +func NewBlobAccessChunkListFetcher(chunkListStorage BlobAccess, maximumMessageSizeBytes int) chunklist.Fetcher { + return &blobAccessChunkListFetcher{ + chunkListStorage: chunkListStorage, + maximumMessageSizeBytes: maximumMessageSizeBytes, + } +} + +func (f *blobAccessChunkListFetcher) FetchChunkList(ctx context.Context, d digest.Digest) (chunklist.ChunkList, error) { + msg, err := f.chunkListStorage.Get(ctx, d).ToProto(&chunklist_pb.ChunkList{}, f.maximumMessageSizeBytes) + if err != nil { + return nil, util.StatusWrap(err, "Failed to fetch chunk list") + } + proto := msg.(*chunklist_pb.ChunkList) + return chunklist.NewChunkListFromProto(proto, d.GetInstanceName()) +} diff --git a/pkg/blobstore/chunklist/BUILD.bazel b/pkg/blobstore/chunklist/BUILD.bazel new file mode 100644 index 000000000..cd8927589 --- /dev/null +++ b/pkg/blobstore/chunklist/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "chunklist", + srcs = [ + "caching_fetcher.go", + "chunk_concatenating_reader.go", + "chunk_fetcher.go", + "chunk_list.go", + "encoding.go", + ], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist", + visibility = ["//visibility:public"], + deps = [ + "//pkg/digest", + "//pkg/proto/blobstore/chunklist", + "//pkg/ttlcache", + "//pkg/util", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) diff --git a/pkg/blobstore/chunklist/caching_fetcher.go b/pkg/blobstore/chunklist/caching_fetcher.go new file mode 100644 index 000000000..2b51933ca --- /dev/null +++ b/pkg/blobstore/chunklist/caching_fetcher.go @@ -0,0 +1,33 @@ +package chunklist + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/ttlcache" +) + +type cachingFetcher struct { + Fetcher + cache *ttlcache.TTLCache[digest.Digest, ChunkList] +} + +// NewCachingFetcher decorates a Fetcher with a cache. +func NewCachingFetcher(base Fetcher, cache *ttlcache.TTLCache[digest.Digest, ChunkList]) Fetcher { + return &cachingFetcher{ + Fetcher: base, + cache: cache, + } +} + +func (f *cachingFetcher) FetchChunkList(ctx context.Context, d digest.Digest) (ChunkList, error) { + if v, ok := f.cache.Get(d); ok { + return v, nil + } + chunkList, err := f.Fetcher.FetchChunkList(ctx, d) + if err != nil { + return nil, err + } + f.cache.Put(d, chunkList) + return chunkList, nil +} diff --git a/pkg/blobstore/chunklist/chunk_concatenating_reader.go b/pkg/blobstore/chunklist/chunk_concatenating_reader.go new file mode 100644 index 000000000..dbaf6e42a --- /dev/null +++ b/pkg/blobstore/chunklist/chunk_concatenating_reader.go @@ -0,0 +1,74 @@ +package chunklist + +import ( + "context" + "io" + + "github.com/buildbarn/bb-storage/pkg/util" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// chunkConcatenatingReadCloser is an io.ReadCloser that stitches +// together the contents of a blob based on a ChunkList. +type chunkConcatenatingReadCloser struct { + ctx context.Context + chunkFetcher ChunkFetcher + chunkList ChunkList + + currentChunkIndex int + currentChunkData []byte + currentChunkOffset int + closed bool +} + +// NewChunkConcatenatingReader creates an io.ReadCloser that yields the +// concatenated contents of the chunks in a ChunkList. +func NewChunkConcatenatingReader(ctx context.Context, chunkList ChunkList, chunkFetcher ChunkFetcher) io.ReadCloser { + return &chunkConcatenatingReadCloser{ + ctx: ctx, + chunkFetcher: chunkFetcher, + chunkList: chunkList, + } +} + +func (r *chunkConcatenatingReadCloser) Read(p []byte) (int, error) { + if r.closed { + return 0, status.Error(codes.Internal, "Reader is already closed") + } + + // Advance to the next chunk if the current one is exhausted, + // skipping any zero-length chunks. + for r.currentChunkOffset >= len(r.currentChunkData) { + if r.currentChunkIndex >= len(r.chunkList) { + return 0, io.EOF + } + + chunkDigest := r.chunkList[r.currentChunkIndex].Digest + chunkData, err := r.chunkFetcher.FetchChunk(r.ctx, chunkDigest) + if err != nil { + return 0, util.StatusWrapf(err, "Failed to fetch chunk at index %d", r.currentChunkIndex) + } + + r.currentChunkData = chunkData + r.currentChunkOffset = 0 + r.currentChunkIndex++ + } + + // Copy as much data as available from the current chunk into p. + n := copy(p, r.currentChunkData[r.currentChunkOffset:]) + r.currentChunkOffset += n + + if r.currentChunkOffset >= len(r.currentChunkData) { + r.currentChunkData = nil + } + + return n, nil +} + +func (r *chunkConcatenatingReadCloser) Close() error { + r.closed = true + r.currentChunkData = nil + return nil +} diff --git a/pkg/blobstore/chunklist/chunk_fetcher.go b/pkg/blobstore/chunklist/chunk_fetcher.go new file mode 100644 index 000000000..146567714 --- /dev/null +++ b/pkg/blobstore/chunklist/chunk_fetcher.go @@ -0,0 +1,26 @@ +package chunklist + +import ( + "context" + + "github.com/buildbarn/bb-storage/pkg/digest" +) + +// ChunkFetcher retrieves the contents of an individual chunk. +type ChunkFetcher interface { + FetchChunk(ctx context.Context, digest digest.Digest) ([]byte, error) +} + +type functionChunkFetcher struct { + fetcher func(ctx context.Context, digest digest.Digest) ([]byte, error) +} + +// NewChunkFetcherFromFunction creates a chunk fetcher that wraps a +// function which fetches chunks. +func NewChunkFetcherFromFunction(fetcher func(ctx context.Context, digest digest.Digest) ([]byte, error)) ChunkFetcher { + return functionChunkFetcher{fetcher: fetcher} +} + +func (f functionChunkFetcher) FetchChunk(ctx context.Context, digest digest.Digest) ([]byte, error) { + return f.fetcher(ctx, digest) +} diff --git a/pkg/blobstore/chunklist/chunk_list.go b/pkg/blobstore/chunklist/chunk_list.go new file mode 100644 index 000000000..894565ae8 --- /dev/null +++ b/pkg/blobstore/chunklist/chunk_list.go @@ -0,0 +1,43 @@ +package chunklist + +import ( + "context" + "slices" + + "github.com/buildbarn/bb-storage/pkg/digest" +) + +// Entry represents a single chunk within a chunk list. +type Entry struct { + Offset uint64 + Digest digest.Digest +} + +// ChunkList represents the ordered list of chunks that compose a blob. +type ChunkList []Entry + +// Fetcher retrieves a ChunkList for a digest. +type Fetcher interface { + FetchChunkList(ctx context.Context, digest digest.Digest) (ChunkList, error) +} + +// FindChunkOffset returns the index of the chunk containing the given +// offset and the offset within that chunk. +func FindChunkOffset(chunkList ChunkList, off uint64) (index int, chunkOffset int64) { + i, ok := slices.BinarySearchFunc(chunkList, off, func(e Entry, offset uint64) int { + next := e.Offset + uint64(e.Digest.GetSizeBytes()) + if offset < e.Offset { + return 1 + } + if offset >= next { + return -1 + } + return 0 + }) + + if ok { + return i, int64(off - chunkList[i].Offset) + } + + return len(chunkList), 0 +} diff --git a/pkg/blobstore/chunklist/encoding.go b/pkg/blobstore/chunklist/encoding.go new file mode 100644 index 000000000..a6fbf12c5 --- /dev/null +++ b/pkg/blobstore/chunklist/encoding.go @@ -0,0 +1,56 @@ +package chunklist + +import ( + "bytes" + + "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/util" +) + +// EncodeToBinary converts a ChunkList into a compact binary +// representation. Each digest is encoded using +// digest.Digest.GetCompactBinary(). +func EncodeToBinary(chunkDigests []digest.Digest) []byte { + var buf bytes.Buffer + for _, d := range chunkDigests { + buf.Write(d.GetCompactBinary()) + } + return buf.Bytes() +} + +// DecodeFromBinary parses a compact binary representation of a chunk +// list back into a ChunkList. +func DecodeFromBinary(data []byte, instanceName digest.InstanceName) (ChunkList, error) { + r := bytes.NewReader(data) + var cl ChunkList + var offset uint64 + for r.Len() > 0 { + d, err := instanceName.NewDigestFromCompactBinary(r) + if err != nil { + return nil, util.StatusWrap(err, "Failed to parse binary coded digest in chunk list") + } + cl = append(cl, Entry{ + Offset: offset, + Digest: d, + }) + offset += uint64(d.GetSizeBytes()) + } + return cl, nil +} + +// ToProto converts a ChunkList to the protobuf storage format. +func ToProto(cl ChunkList) *chunklist_pb.ChunkList { + digests := make([]digest.Digest, 0, len(cl)) + for _, entry := range cl { + digests = append(digests, entry.Digest) + } + return &chunklist_pb.ChunkList{ + Data: EncodeToBinary(digests), + } +} + +// NewChunkListFromProto converts a protobuf ChunkList into a ChunkList. +func NewChunkListFromProto(proto *chunklist_pb.ChunkList, instanceName digest.InstanceName) (ChunkList, error) { + return DecodeFromBinary(proto.Data, instanceName) +} diff --git a/pkg/blobstore/chunklistvalidating/BUILD.bazel b/pkg/blobstore/chunklistvalidating/BUILD.bazel index 1bb49c7f0..cac7baa20 100644 --- a/pkg/blobstore/chunklistvalidating/BUILD.bazel +++ b/pkg/blobstore/chunklistvalidating/BUILD.bazel @@ -2,24 +2,19 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "chunklistvalidating", - srcs = [ - "chunk_concatenating_reader.go", - "chunk_list_validating_blob_access.go", - "chunker.go", - "reader_chunker.go", - ], + srcs = ["chunk_list_validating_blob_access.go"], importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating", visibility = ["//visibility:public"], deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/digest", + "//pkg/proto/blobstore/chunklist", "//pkg/util", - "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", - "@com_github_buildbarn_go_cdc//:go-cdc", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", - "@org_golang_x_sync//errgroup", ], ) @@ -28,13 +23,14 @@ go_test( srcs = [ "chunk_list_validating_blob_access_test.go", "fake_blob_access_test.go", - "reader_chunker_test.go", ], deps = [ ":chunklistvalidating", "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/chunklist", "//pkg/digest", + "//pkg/proto/blobstore/chunklist", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", "@com_github_stretchr_testify//require", "@org_golang_google_grpc//codes", diff --git a/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go b/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go deleted file mode 100644 index 643af130e..000000000 --- a/pkg/blobstore/chunklistvalidating/chunk_concatenating_reader.go +++ /dev/null @@ -1,68 +0,0 @@ -package chunklistvalidating - -import ( - "context" - "io" - - "github.com/buildbarn/bb-storage/pkg/blobstore" - "github.com/buildbarn/bb-storage/pkg/digest" - "github.com/buildbarn/bb-storage/pkg/util" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -// chunkConcatenatingReader is a helper utility that implements the -// io.ReadCloser api over a series of digest.Digest objectes fetched -// sequentially from the CAS. -type chunkConcatenatingReader struct { - ctx context.Context - contentAddressableStorage blobstore.BlobAccess - chunkDigests []digest.Digest - currentIndex int - currentReader io.ReadCloser - closed bool -} - -func (r *chunkConcatenatingReader) Read(p []byte) (int, error) { - if r.closed { - return 0, status.Error(codes.Internal, "Reader is already closed") - } - for { - if r.currentReader == nil { - if r.currentIndex >= len(r.chunkDigests) { - return 0, io.EOF - } - chunkDigest := r.chunkDigests[r.currentIndex] - b := r.contentAddressableStorage.Get(r.ctx, chunkDigest) - r.currentReader = b.ToReader() - r.currentIndex++ - } - - n, err := r.currentReader.Read(p) - if n > 0 { - return n, nil - } - if err == io.EOF { - err = r.currentReader.Close() - r.currentReader = nil - if err != nil { - return 0, err - } - continue - } - if err != nil { - _ = r.currentReader.Close() - r.currentReader = nil - return 0, util.StatusWrap(err, "Failed to read chunk") - } - } -} - -func (r *chunkConcatenatingReader) Close() (err error) { - r.closed = true - if r.currentReader != nil { - err = r.currentReader.Close() - r.currentReader = nil - } - return err -} diff --git a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go index 351ea3a0f..acb88c372 100644 --- a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go +++ b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access.go @@ -5,20 +5,24 @@ import ( "io" "slices" - remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/util" - "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) type chunkListValidatingBlobAccess struct { blobstore.BlobAccess - contentAddressableStorage blobstore.BlobAccess - maximumMessageSizeBytes int + cdcParametersFetcher cdc.ParametersFetcher + chunkListFetcher chunklist.Fetcher + chunkFetcher chunklist.ChunkFetcher + chunkStorage blobstore.BlobAccess + maximumMessageSizeBytes int } // NewChunkListValidatingBlobAccess creates a wrapper around a Chunk @@ -31,71 +35,53 @@ type chunkListValidatingBlobAccess struct { // This validation is fairly expensive and validation should only be // done at a single layer as close as possible to the CAS where the full // view of the CAS is available. -func NewChunkListValidatingBlobAccess(chunkListStorage, contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int) blobstore.BlobAccess { +func NewChunkListValidatingBlobAccess(chunkListStorage, chunkStorage blobstore.BlobAccess, maximumMessageSizeBytes int) blobstore.BlobAccess { return &chunkListValidatingBlobAccess{ - BlobAccess: chunkListStorage, - contentAddressableStorage: contentAddressableStorage, - maximumMessageSizeBytes: maximumMessageSizeBytes, + BlobAccess: chunkListStorage, + cdcParametersFetcher: cdc.NewCapabilitiesParametersFetcher(chunkStorage), + chunkListFetcher: blobstore.NewBlobAccessChunkListFetcher(chunkListStorage, maximumMessageSizeBytes), + chunkStorage: chunkStorage, + maximumMessageSizeBytes: maximumMessageSizeBytes, + chunkFetcher: chunklist.NewChunkFetcherFromFunction(func(ctx context.Context, digest digest.Digest) ([]byte, error) { + return chunkStorage.Get(ctx, digest).ToByteSlice(maximumMessageSizeBytes) + }), } } -// Fetch the chunking parameters from the GetCapabilities -// implementation. -func (ba *chunkListValidatingBlobAccess) getValidChunkingParameters(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.RepMaxCdcParams, error) { - capabilities, err := ba.BlobAccess.GetCapabilities(ctx, instanceName) - if err != nil { - return nil, util.StatusWrap(err, "Unable to GetCapabilities to determine chunking parameters") - } - - params := capabilities.CacheCapabilities.GetRepMaxCdcParams() - if params == nil { - return nil, status.Error(codes.Unimplemented, "This backend only supports upstream servers with RepMaxCDC support") - } - if params.MinChunkSizeBytes < 64 { - return nil, status.Errorf(codes.Internal, "RepMaxCDC minimum chunk size was %d bytes but a minimum of 64 bytes is required", params.MinChunkSizeBytes) +// Get the split result from the downstream blob access, should one +// exist return it only if all its constituent chunks exist. +func (ba *chunkListValidatingBlobAccess) getComplete(ctx context.Context, d digest.Digest) buffer.Buffer { + // Verify the existence of the blob itself against the chunk + // list storage. This renews the lifetime of the blob even when + // the chunk list is served from a caching chunk list storage's + // local cache, because the FindMissing is punched through to + // the authoritative chunk list storage. + missing, err := ba.BlobAccess.FindMissing(ctx, d.ToSingletonSet()) + if err != nil || !missing.Empty() { + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob could not be found")) } - return params, nil -} - -// Check the downstream blob access if this particular blob has already -// been split. If that's the case and all the chunks are still there we -// can return early. In case of errors we will return nil and continue -// with the regular code path. -func (ba *chunkListValidatingBlobAccess) checkSplitResult(ctx context.Context, d digest.Digest) buffer.Buffer { - b1, b2 := ba.BlobAccess.Get(ctx, d).CloneCopy(ba.maximumMessageSizeBytes) - responseMsg, err := b1.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + storedChunkList, err := ba.chunkListFetcher.FetchChunkList(ctx, d) if err != nil { - b2.Discard() - return nil + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Failed to get chunk list")) } - splitBlobResponse := responseMsg.(*remoteexecution.SplitBlobResponse) - digestFunction := d.GetDigestFunction() - digestSetBuilder := digest.NewSetBuilder(len(splitBlobResponse.ChunkDigests)) - digestSetBuilder.Add(d) - - for _, chunkDigestProto := range splitBlobResponse.ChunkDigests { - chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) - if err != nil { - b2.Discard() - return nil - } - digestSetBuilder.Add(chunkDigest) + digestSetBuilder := digest.NewSetBuilder(len(storedChunkList)) + for _, entry := range storedChunkList { + digestSetBuilder.Add(entry.Digest) } - missing, err := ba.contentAddressableStorage.FindMissing(ctx, digestSetBuilder.Build()) + missing, err = ba.chunkStorage.FindMissing(ctx, digestSetBuilder.Build()) if err == nil && missing.Empty() { - return b2 + return buffer.NewProtoBufferFromProto(chunklist.ToProto(storedChunkList), buffer.UserProvided) } - b2.Discard() - return nil + return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob could not be found")) } -// Get returns a valid SplitResult for the given digest chunking the +// Get returns a valid chunk list for the given digest, chunking the // blob and storing the chunk list if needed. func (ba *chunkListValidatingBlobAccess) Get(ctx context.Context, d digest.Digest) buffer.Buffer { - params, err := ba.getValidChunkingParameters(ctx, d.GetInstanceName()) + params, err := ba.cdcParametersFetcher.FetchCDCParameters(ctx, d.GetInstanceName()) if err != nil { return buffer.NewBufferFromError(err) } @@ -105,8 +91,8 @@ func (ba *chunkListValidatingBlobAccess) Get(ctx context.Context, d digest.Diges // original blob. We verify the existence of the blob in CAS and // break out early. blobSize := d.GetSizeBytes() - if uint64(blobSize) < 2*params.MinChunkSizeBytes { - missing, err := ba.contentAddressableStorage.FindMissing(ctx, d.ToSingletonSet()) + if blobSize < 2*params.MinChunkSizeBytes { + missing, err := ba.chunkStorage.FindMissing(ctx, d.ToSingletonSet()) if err != nil { return buffer.NewBufferFromError(util.StatusWrap(err, "Failed to verify blob existence")) } @@ -114,111 +100,50 @@ func (ba *chunkListValidatingBlobAccess) Get(ctx context.Context, d digest.Diges return buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found in CAS")) } - response := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{d.GetProto()}, - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + chunkList := chunklist.ChunkList{ + {Offset: 0, Digest: d}, } - - return buffer.NewProtoBufferFromProto(response, buffer.UserProvided) + return buffer.NewProtoBufferFromProto(chunklist.ToProto(chunkList), buffer.UserProvided) } - // Check if we have already computed the result for this blob. - if result := ba.checkSplitResult(ctx, d); result != nil { - return result - } - - // Fallthrough case, compute the chunk list, upload the chunks and - // store the chunk list. - blobReader := ba.contentAddressableStorage.Get(ctx, d).ToReader() - defer blobReader.Close() - chunker := NewReaderChunker(d.GetDigestFunction(), blobReader, int64(params.MinChunkSizeBytes), int64(params.HorizonSizeBytes)) - - chunkDigests := make([]*remoteexecution.Digest, 0, uint64(blobSize)/params.MinChunkSizeBytes+1) - - for { - chunk, err := chunker.NextChunk() - if err == io.EOF { - break - } - if err != nil { - return buffer.NewBufferFromError(err) - } - - missing, err := ba.contentAddressableStorage.FindMissing(ctx, chunk.Digest.ToSingletonSet()) - if err != nil { - return buffer.NewBufferFromError(err) - } - if !missing.Empty() { - if err := ba.contentAddressableStorage.Put(ctx, chunk.Digest, buffer.NewValidatedBufferFromByteSlice(chunk.Data)); err != nil { - return buffer.NewBufferFromError(err) - } - } - - chunkDigests = append(chunkDigests, chunk.Digest.GetProto()) - } - - response := &remoteexecution.SplitBlobResponse{ - ChunkDigests: chunkDigests, - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, - } - - b1, b2 := buffer.NewProtoBufferFromProto(response, buffer.UserProvided).CloneCopy(ba.maximumMessageSizeBytes) + // Return upstream split result if complete. + return ba.getComplete(ctx, d) +} - if err := ba.BlobAccess.Put(ctx, d, b1); err != nil { - b2.Discard() - return buffer.NewBufferFromError(util.StatusWrap(err, "Failed to store the split blob response")) +// matchesStoredChunkList checks if the user-provided chunk digests +// match the chunk list already stored for the given digest. +func (ba *chunkListValidatingBlobAccess) matchesStoredChunkList(ctx context.Context, d digest.Digest, userChunkList chunklist.ChunkList) bool { + msg, err := ba.BlobAccess.Get(ctx, d).ToProto(&chunklist_pb.ChunkList{}, ba.maximumMessageSizeBytes) + if err != nil { + return false } - return b2 -} - -func (ba *chunkListValidatingBlobAccess) matchesStoredChunkList(ctx context.Context, d digest.Digest, userResponse *remoteexecution.SplitBlobResponse) bool { - existingMsg, err := ba.BlobAccess.Get(ctx, d).ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + storedChunkList, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), d.GetInstanceName()) if err != nil { return false } - cachedResponse := existingMsg.(*remoteexecution.SplitBlobResponse) - return slices.EqualFunc(cachedResponse.ChunkDigests, userResponse.ChunkDigests, - func(c, u *remoteexecution.Digest) bool { - return c.Hash == u.Hash && c.SizeBytes == u.SizeBytes - }) + return slices.Equal(userChunkList, storedChunkList) } func (ba *chunkListValidatingBlobAccess) Put(ctx context.Context, d digest.Digest, b buffer.Buffer) error { - // Parse the buffer as a SplitBlobResponse - msg, err := b.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + // Parse the buffer as a ChunkList proto. + msg, err := b.ToProto(&chunklist_pb.ChunkList{}, ba.maximumMessageSizeBytes) if err != nil { - return util.StatusWrap(err, "Failed to parse input as SplitBlobResponse") + return util.StatusWrap(err, "Failed to parse input as ChunkList") } - userResponse := msg.(*remoteexecution.SplitBlobResponse) - - digestFunction := d.GetDigestFunction() - var userChunks []digest.Digest - digestSetBuilder := digest.NewSetBuilder(len(userResponse.ChunkDigests)) - for _, chunkDigestProto := range userResponse.ChunkDigests { - chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) - if err != nil { - return status.Errorf(codes.InvalidArgument, "Invalid chunk digest: %v", err) - } - digestSetBuilder.Add(chunkDigest) - userChunks = append(userChunks, chunkDigest) + userChunkList, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), d.GetInstanceName()) + if err != nil { + return util.StatusWrap(err, "Failed to decode chunk list") } - // Check that all referenced chunks are present in storage. - missing, err := ba.contentAddressableStorage.FindMissing(ctx, digestSetBuilder.Build()) + params, err := ba.cdcParametersFetcher.FetchCDCParameters(ctx, d.GetInstanceName()) if err != nil { - return util.StatusWrap(err, "Failed to check existence of chunks") - } - if !missing.Empty() { - return status.Error(codes.NotFound, "At least one chunk in the chunk list was not found") + return err } - // Check the trivial cases without hitting the downstream blob - // stores. - // No chunks given, blob must be the empty blob. - if len(userChunks) == 0 { + if len(userChunkList) == 0 { if d.GetSizeBytes() != 0 { return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") } @@ -227,118 +152,134 @@ func (ba *chunkListValidatingBlobAccess) Put(ctx context.Context, d digest.Diges } return nil } - // Single chunk given, the blob must be equal to the chunk. At this - // point we have already verified the presence of the chunk so we do - // not have to verify the presence of the blob. - if len(userChunks) == 1 { - if d != userChunks[0] { - return status.Error(codes.InvalidArgument, "Chunk list does not compose to blob") - } - return nil - } - chunksMatchesStoredLists := ba.matchesStoredChunkList(ctx, d, userResponse) - missing, err = ba.contentAddressableStorage.FindMissing(ctx, d.ToSingletonSet()) + // Check that all referenced chunks are present in storage. + userChunkList, err = ba.flattenChunks(ctx, params, userChunkList) if err != nil { - return util.StatusWrap(err, "Failed to check existence of blob") + return status.Error(codes.NotFound, "At least one chunk is missing from storage.") } - blobExistsInCAS := missing.Empty() - // The request is identical to an already existing chunk list with - // content we have verified exists in CAS. - if blobExistsInCAS && chunksMatchesStoredLists { + // Chunk list is marked for validation bypass, push it directy to + // downstream blob store. + if cdc.ChunkListValidationBypassed(ctx) { + return ba.BlobAccess.Put(ctx, d, b) + } + + // Check the trivial case without hitting the downstream blob + // stores. + + if ba.matchesStoredChunkList(ctx, d, userChunkList) { return nil } // No more shortcuts available go through the heavy path of // concatenating/verifying and chunking the blobs. - params, err := ba.getValidChunkingParameters(ctx, d.GetInstanceName()) - if err != nil { - return err + canonicalChunkList := make(chunklist.ChunkList, 0, len(userChunkList)) + offset := uint64(0) + reader := chunklist.NewChunkConcatenatingReader(ctx, userChunkList, ba.chunkFetcher) + digestFunction := d.GetDigestFunction() + wholeGen := digestFunction.NewGenerator(d.GetSizeBytes()) + chunker := cdc.NewReaderChunker(d.GetDigestFunction(), reader, params.MinChunkSizeBytes, params.HorizonSizeBytes) + for { + chunk, err := chunker.NextChunk() + if err == io.EOF { + break + } + if err != nil { + return err + } + + if _, err := wholeGen.Write(chunk.Data); err != nil { + return status.Error(codes.Internal, "Could not compute digest of blob") + } + + missing, err := ba.chunkStorage.FindMissing(ctx, chunk.Digest.ToSingletonSet()) + if err != nil { + return err + } + if !missing.Empty() { + if err := ba.chunkStorage.Put(ctx, chunk.Digest, buffer.NewValidatedBufferFromByteSlice(chunk.Data)); err != nil { + return util.StatusWrap(err, "Failed to save chunk") + } + } + canonicalChunkList = append(canonicalChunkList, chunklist.Entry{Digest: chunk.Digest, Offset: offset}) + offset += uint64(chunk.Digest.GetSizeBytes()) } - reader := &chunkConcatenatingReader{ - ctx: ctx, - contentAddressableStorage: ba.contentAddressableStorage, - chunkDigests: userChunks, + // Verify the whole blob against the advertised digest. + if actual := wholeGen.Sum(); actual != d { + return status.Errorf(codes.InvalidArgument, "Blob digest mismatch: advertised %s, actual %s", d, actual) } - blobBuffer := buffer.NewCASBufferFromReader(d, reader, buffer.UserProvided) - b1, b2 := blobBuffer.CloneStream() + // Store the canonical chunk list. + canonicalBuffer := buffer.NewProtoBufferFromProto(chunklist.ToProto(canonicalChunkList), buffer.UserProvided) + if err := ba.BlobAccess.Put(ctx, d, canonicalBuffer); err != nil { + return util.StatusWrap(err, "Failed to save canonical chunk list") + } + return nil +} - // Stream 1: Uploads the blob to CAS. - group, gCtx := errgroup.WithContext(ctx) - group.Go(func() error { - if blobExistsInCAS { - // Upload unnecessary, blob already exists in CAS. - b1.Discard() - return nil +func (ba *chunkListValidatingBlobAccess) flattenChunks(ctx context.Context, params cdc.Parameters, userChunkList chunklist.ChunkList) (chunklist.ChunkList, error) { + maxChunkSize := int64(2*params.MinChunkSizeBytes - 1) + bigDigests := digest.NewSetBuilder(len(userChunkList)) + for _, entry := range userChunkList { + if entry.Digest.GetSizeBytes() > maxChunkSize { + bigDigests.Add(entry.Digest) } - return ba.contentAddressableStorage.Put(gCtx, d, b1) - }) - - // Stream 2: Chunk the stream to compute the digest and cache the - // canonical chunks. - var canonicalChunkDigests []*remoteexecution.Digest - group.Go(func() error { - b2Reader := b2.ToReader() - defer b2Reader.Close() - chunker := NewReaderChunker(d.GetDigestFunction(), b2Reader, int64(params.MinChunkSizeBytes), int64(params.HorizonSizeBytes)) - for { - chunk, err := chunker.NextChunk() - if err == io.EOF { - return nil - } - if err != nil { - return err - } - - missing, err := ba.contentAddressableStorage.FindMissing(gCtx, chunk.Digest.ToSingletonSet()) + } + missing, err := ba.BlobAccess.FindMissing(ctx, bigDigests.Build()) + if err != nil { + return nil, util.StatusWrap(err, "Error checking for chunk lists of big chunks") + } + if !missing.Empty() { + return nil, status.Error(codes.NotFound, "Chunk lists not found for big chunks") + } + flattenedChunkList := make(chunklist.ChunkList, 0, len(userChunkList)) + flattenedChunksBuilder := digest.NewSetBuilder(len(userChunkList)) + for _, outerEntry := range userChunkList { + outerDigest := outerEntry.Digest + if outerDigest.GetSizeBytes() <= maxChunkSize { + flattenedChunkList = append(flattenedChunkList, outerEntry) + flattenedChunksBuilder.Add(outerDigest) + } else { + innerChunkList, err := ba.chunkListFetcher.FetchChunkList(ctx, outerDigest) if err != nil { - return err + return nil, util.StatusWrap(err, "Error fetching inner chunk list") } - if !missing.Empty() { - if err := ba.contentAddressableStorage.Put(gCtx, chunk.Digest, buffer.NewValidatedBufferFromByteSlice(chunk.Data)); err != nil { - return util.StatusWrap(err, "Failed to save chunk") + for _, innerChunkEntry := range innerChunkList { + innerEntry := chunklist.Entry{ + Digest: innerChunkEntry.Digest, + Offset: innerChunkEntry.Offset + outerEntry.Offset, } + flattenedChunkList = append(flattenedChunkList, innerEntry) + flattenedChunksBuilder.Add(innerEntry.Digest) } - canonicalChunkDigests = append(canonicalChunkDigests, chunk.Digest.GetProto()) } - }) - - // Wait for the full blob validation and upload to complete. - if err := group.Wait(); err != nil { - return util.StatusWrap(err, "Failed to splice the blob") } - - // Store the canonical response. - canonicalResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: canonicalChunkDigests, - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + missing, err = ba.chunkStorage.FindMissing(ctx, flattenedChunksBuilder.Build()) + if err != nil { + return nil, util.StatusWrap(err, "Error checking for existence of flattened chunks.") } - canonicalBuffer := buffer.NewProtoBufferFromProto(canonicalResponse, buffer.UserProvided) - if err := ba.BlobAccess.Put(ctx, d, canonicalBuffer); err != nil { - return util.StatusWrap(err, "Failed to save canonical chunk list") + if !missing.Empty() { + return nil, status.Error(codes.NotFound, "At least one chunk among flattened chunks are missing.") } - return nil + return flattenedChunkList, nil } func (ba *chunkListValidatingBlobAccess) findMissingChunks(ctx context.Context, d digest.Digest) (digest.Set, error) { - splitBlobResponseProto, err := ba.BlobAccess.Get(ctx, d).ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) + msg, err := ba.BlobAccess.Get(ctx, d).ToProto(&chunklist_pb.ChunkList{}, ba.maximumMessageSizeBytes) if err != nil { return digest.EmptySet, err } - splitBlobResponse := splitBlobResponseProto.(*remoteexecution.SplitBlobResponse) - digestFunction := d.GetDigestFunction() - builder := digest.NewSetBuilder(len(splitBlobResponse.ChunkDigests)) - for _, chunkDigestProto := range splitBlobResponse.ChunkDigests { - chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) - if err != nil { - return digest.EmptySet, util.StatusWrapf(err, "Invalid chunk digest %#v", chunkDigestProto) - } - builder.Add(chunkDigest) + storedChunkList, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), d.GetInstanceName()) + if err != nil { + return digest.EmptySet, util.StatusWrap(err, "Failed to decode chunk list") + } + builder := digest.NewSetBuilder(len(storedChunkList)) + for _, entry := range storedChunkList { + builder.Add(entry.Digest) } - return ba.contentAddressableStorage.FindMissing(ctx, builder.Build()) + return ba.chunkStorage.FindMissing(ctx, builder.Build()) } func (ba *chunkListValidatingBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { diff --git a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go index 3830e720b..340d4e23f 100644 --- a/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go +++ b/pkg/blobstore/chunklistvalidating/chunk_list_validating_blob_access_test.go @@ -7,8 +7,10 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/chunklistvalidating" "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" @@ -25,6 +27,33 @@ func mustComputeDigest(t *testing.T, digestFunction digest.Function, data []byte return generator.Sum() } +// makeChunkListBuffer creates a buffer containing a ChunkList proto +// from a list of chunk digests, suitable for passing to Put(). +func makeChunkListBuffer(chunkDigests ...digest.Digest) buffer.Buffer { + cl := make(chunklist.ChunkList, 0, len(chunkDigests)) + var offset uint64 + for _, d := range chunkDigests { + cl = append(cl, chunklist.Entry{Offset: offset, Digest: d}) + offset += uint64(d.GetSizeBytes()) + } + return buffer.NewProtoBufferFromProto(chunklist.ToProto(cl), buffer.UserProvided) +} + +// getChunkListDigests reads a ChunkList proto from a buffer and +// returns the chunk digests it contains. +func getChunkListDigests(t *testing.T, buf buffer.Buffer, instanceName digest.InstanceName) []digest.Digest { + t.Helper() + msg, err := buf.ToProto(&chunklist_pb.ChunkList{}, maximumMessageSizeBytes) + require.NoError(t, err) + cl, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), instanceName) + require.NoError(t, err) + digests := make([]digest.Digest, 0, len(cl)) + for _, entry := range cl { + digests = append(digests, entry.Digest) + } + return digests +} + var testCDCParams = &remoteexecution.RepMaxCdcParams{ MinChunkSizeBytes: 1024, HorizonSizeBytes: 8 * 1024, @@ -34,8 +63,8 @@ var maximumMessageSizeBytes = 1024 * 1024 func TestChunkListValidatingBlobAccessGetTrivialSmallBlob(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) @@ -45,92 +74,98 @@ func TestChunkListValidatingBlobAccessGetTrivialSmallBlob(t *testing.T) { require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) fakeCAS.ResetTouches() - msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) - require.NoError(t, err) + chunkDigests := getChunkListDigests(t, validatingCLS.Get(ctx, blobDigest), digestFunction.GetInstanceName()) - splitResponse := msg.(*remoteexecution.SplitBlobResponse) - require.Len(t, splitResponse.ChunkDigests, 1) - require.Equal(t, blobDigest.GetProto().Hash, splitResponse.ChunkDigests[0].Hash) + require.Len(t, chunkDigests, 1) + require.Equal(t, blobDigest, chunkDigests[0]) require.Greater(t, fakeCAS.GetTouches(blobDigest), 0, "Blob did not have its lifetime renewed.") } -func TestChunkListValidatingBlobAccessGetLargeBlob(t *testing.T) { +func TestChunkListValidatingBlobAccessGetExtendsLifetimes(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) + // The blob must be at least 2*MinChunkSizeBytes, so that Get + // goes down the path of returning the stored chunk list. + blobData := bytes.Repeat([]byte("testdatafortests"), 250) + blobData = blobData[:2048] + chunk1Data := blobData[:len(blobData)/2] + chunk2Data := blobData[len(blobData)/2:] + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) - blobData := bytes.Repeat([]byte("test_data_pattern_"), 6000) blobDigest := mustComputeDigest(t, digestFunction, blobData) + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + require.NoError(t, fakeCAS.Put(ctx, chunk2Digest, buffer.NewValidatedBufferFromByteSlice(chunk2Data))) + require.NoError(t, fakeCLS.Put(ctx, blobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest))) - require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) - - msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) - require.NoError(t, err) + // Reset touches. + fakeCLS.ResetTouches() + fakeCAS.ResetTouches() - splitResponse := msg.(*remoteexecution.SplitBlobResponse) - require.Greater(t, len(splitResponse.ChunkDigests), 1, "Blob should have been divided into multiple chunks.") + // Perform a cached split. + cachedDigests := getChunkListDigests(t, validatingCLS.Get(ctx, blobDigest), digestFunction.GetInstanceName()) + require.Equal(t, []digest.Digest{chunk1Digest, chunk2Digest}, cachedDigests) - for _, chunkProto := range splitResponse.ChunkDigests { - chunkDigest, err := digestFunction.NewDigestFromProto(chunkProto) - require.NoError(t, err) + // The original blob's chunk list MUST have had its lifetime + // extended. + require.Greater(t, fakeCLS.GetTouches(blobDigest), 0, "Original blob's chunk list lifetime was not extended during call to Get") - require.Greater(t, fakeCAS.GetTouches(chunkDigest), 0, "Chunk generated by CDC did not have its lifetime renewed.") - missing, err := fakeCAS.FindMissing(ctx, chunkDigest.ToSingletonSet()) - require.NoError(t, err) - require.True(t, missing.Empty(), "Chunk generated by CDC was not saved to the CAS.") + // Every chunk MUST have had its lifetime extended. + for _, chunkDigest := range cachedDigests { + require.Greater(t, fakeCAS.GetTouches(chunkDigest), 0, "Chunk's lifetime was not extended during call to Get") } - - cachedMsg, err := fakeCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) - require.NoError(t, err) - require.Equal(t, len(splitResponse.ChunkDigests), len(cachedMsg.(*remoteexecution.SplitBlobResponse).ChunkDigests)) } -func TestChunkListValidatingBlobAccessGetExtendsLifetimes(t *testing.T) { +func TestChunkListValidatingBlobAccessGetLargeBlobMissingUnderlyingChunk(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) - blobData := bytes.Repeat([]byte("test_data_pattern_"), 6000) // ~108KB - blobDigest := mustComputeDigest(t, digestFunction, blobData) - require.NoError(t, fakeCAS.Put(ctx, blobDigest, buffer.NewValidatedBufferFromByteSlice(blobData))) - // Split the blob to populate the CAS and CLS for this blob. - msg, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) - require.NoError(t, err) - splitResponse := msg.(*remoteexecution.SplitBlobResponse) + chunk1Data := bytes.Repeat([]byte("A"), 1500) + chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) + require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) + chunk2Data := bytes.Repeat([]byte("B"), 1500) + chunk2Digest := mustComputeDigest(t, digestFunction, chunk2Data) + // Chunk 2 is not uploaded to the chunk storage. - // Reset touches. - fakeCAS.ResetTouches() + expectedFullData := append(chunk1Data, chunk2Data...) + blobDigest := mustComputeDigest(t, digestFunction, expectedFullData) + require.NoError(t, fakeCLS.Put(ctx, blobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest))) - // Perform a cached split. - msgCached, err := validatingCLS.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) - require.NoError(t, err) - cachedResponse := msgCached.(*remoteexecution.SplitBlobResponse) - require.Equal(t, len(splitResponse.ChunkDigests), len(cachedResponse.ChunkDigests)) + _, err := validatingCLS.Get(ctx, blobDigest).ToProto(&chunklist_pb.ChunkList{}, maximumMessageSizeBytes) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err), "Incorrect error code from Get request: %s", err.Error()) +} + +func TestChunkListValidatingBlobAccessGetMissingBlob(t *testing.T) { + ctx := context.Background() - // The original blob MUST have had its lifetime extended - require.Greater(t, fakeCAS.GetTouches(blobDigest), 0, "Original blob's lifetime was not extended during call to SplitBlob") - require.Greater(t, fakeCLS.GetTouches(blobDigest), 0, "Original blob's chunk list lifetime was not extended during call to SplitBlob") + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) + validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) - // Every chunk MUST have been touched in the CAS - for _, chunkProto := range cachedResponse.ChunkDigests { - chunkDigest, err := digestFunction.NewDigestFromProto(chunkProto) - require.NoError(t, err) - require.Greater(t, fakeCAS.GetTouches(chunkDigest), 0, "Chunk's lifetime was not extended during call to SplitBlob") - } + digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) + ghostDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) + + _, err := validatingCLS.Get(ctx, ghostDigest).ToProto(&chunklist_pb.ChunkList{}, maximumMessageSizeBytes) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) } func TestChunkListValidatingBlobAccessPutManualSplice(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) @@ -146,15 +181,7 @@ func TestChunkListValidatingBlobAccessPutManualSplice(t *testing.T) { expectedFullData := []byte("Hello, World!") fullBlobDigest := mustComputeDigest(t, digestFunction, expectedFullData) - splitResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{ - chunk1Digest.GetProto(), - chunk2Digest.GetProto(), - }, - } - reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) - - err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) + err := validatingCLS.Put(ctx, fullBlobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest)) require.NoError(t, err) composedData, err := fakeCAS.Get(ctx, fullBlobDigest).ToByteSlice(len(expectedFullData)) @@ -165,15 +192,15 @@ func TestChunkListValidatingBlobAccessPutManualSplice(t *testing.T) { func TestChunkListValidatingBlobAccessPutCanonicalization(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) - blobData := bytes.Repeat([]byte("canonicalization_test_data"), 4000) // ~104KB - chunk1Data := blobData[:10] - chunk2Data := blobData[10:] + blobData := bytes.Repeat([]byte("testdatafortests"), 250) + chunk1Data := blobData[:len(blobData)/2] + chunk2Data := blobData[len(blobData)/2:] chunk1Digest := mustComputeDigest(t, digestFunction, chunk1Data) require.NoError(t, fakeCAS.Put(ctx, chunk1Digest, buffer.NewValidatedBufferFromByteSlice(chunk1Data))) @@ -183,46 +210,27 @@ func TestChunkListValidatingBlobAccessPutCanonicalization(t *testing.T) { fullBlobDigest := mustComputeDigest(t, digestFunction, blobData) - splitResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{ - chunk1Digest.GetProto(), - chunk2Digest.GetProto(), - }, - } - reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) - - err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) - require.NoError(t, err) - - composedData, err := fakeCAS.Get(ctx, fullBlobDigest).ToByteSlice(200000) - require.NoError(t, err) - require.Equal(t, blobData, composedData) - - canonicalBuffer := fakeCLS.Get(ctx, fullBlobDigest) - canonicalProto, err := canonicalBuffer.ToProto(&remoteexecution.SplitBlobResponse{}, maximumMessageSizeBytes) + err := validatingCLS.Put(ctx, fullBlobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest)) require.NoError(t, err) - canonicalResp := canonicalProto.(*remoteexecution.SplitBlobResponse) - require.Greater(t, len(canonicalResp.ChunkDigests), 0) - require.NotEqual(t, chunk1Digest.GetProto().Hash, canonicalResp.ChunkDigests[0].Hash, "Server should not have echoed back the non-standard chunks") + // The stored chunk list should be the canonical CDC chunking, + // not the non-standard chunks that were provided. + canonicalDigests := getChunkListDigests(t, fakeCLS.Get(ctx, fullBlobDigest), digestFunction.GetInstanceName()) + require.Greater(t, len(canonicalDigests), 0) + require.NotEqual(t, chunk1Digest, canonicalDigests[0], "Server should not have echoed back the non-standard chunks") } func TestChunkListValidatingBlobAccessPutMissingChunk(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) chunkDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) - splitResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{chunkDigest.GetProto()}, - } - reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) - - err := validatingCLS.Put(ctx, chunkDigest, reqBuffer) + err := validatingCLS.Put(ctx, chunkDigest, makeChunkListBuffer(chunkDigest)) require.Error(t, err) require.Equal(t, codes.NotFound, status.Code(err)) } @@ -230,8 +238,8 @@ func TestChunkListValidatingBlobAccessPutMissingChunk(t *testing.T) { func TestChunkListValidatingBlobAccessPutDigestMismatch(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) @@ -242,40 +250,30 @@ func TestChunkListValidatingBlobAccessPutDigestMismatch(t *testing.T) { wrongBlobDigest := mustComputeDigest(t, digestFunction, []byte("Different data")) - splitResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{chunkDigest.GetProto()}, - } - reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) - - err := validatingCLS.Put(ctx, wrongBlobDigest, reqBuffer) + err := validatingCLS.Put(ctx, wrongBlobDigest, makeChunkListBuffer(chunkDigest)) require.Error(t, err) - require.Contains(t, err.Error(), "does not compose to blob") + require.Equal(t, codes.InvalidArgument, status.Code(err), "Incorrect error code from Put request: %s", err.Error()) } func TestChunkListValidatingBlobAccessPutEmptyBlob(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) emptyDigest := mustComputeDigest(t, digestFunction, nil) - splitResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{}, - } - reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) - - err := validatingCLS.Put(ctx, emptyDigest, reqBuffer) + err := validatingCLS.Put(ctx, emptyDigest, makeChunkListBuffer()) require.NoError(t, err) } func TestChunkListValidatingBlobAccessPutRepeatedChunks(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) @@ -291,20 +289,10 @@ func TestChunkListValidatingBlobAccessPutRepeatedChunks(t *testing.T) { expectedData := []byte("AABA") expectedDigest := mustComputeDigest(t, digestFunction, expectedData) - splitResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{ - digestA.GetProto(), - digestA.GetProto(), - digestB.GetProto(), - digestA.GetProto(), - }, - } - reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) - - err := validatingCLS.Put(ctx, expectedDigest, reqBuffer) + err := validatingCLS.Put(ctx, expectedDigest, makeChunkListBuffer(digestA, digestA, digestB, digestA)) require.NoError(t, err) - composedData, err := fakeCAS.Get(ctx, expectedDigest).ToByteSlice(100) + composedData, err := fakeCAS.Get(ctx, expectedDigest).ToByteSlice(len(expectedData)) require.NoError(t, err) require.Equal(t, expectedData, composedData) } @@ -312,8 +300,8 @@ func TestChunkListValidatingBlobAccessPutRepeatedChunks(t *testing.T) { func TestChunkListValidatingBlobAccessPutInlineEmptyChunk(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) @@ -327,23 +315,15 @@ func TestChunkListValidatingBlobAccessPutInlineEmptyChunk(t *testing.T) { expectedDigest := mustComputeDigest(t, digestFunction, chunkData) - splitResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{ - chunkDigest.GetProto(), - emptyDigest.GetProto(), - }, - } - reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) - - err := validatingCLS.Put(ctx, expectedDigest, reqBuffer) + err := validatingCLS.Put(ctx, expectedDigest, makeChunkListBuffer(chunkDigest, emptyDigest)) require.NoError(t, err) } func TestChunkListValidatingBlobAccessPutExtendsLifetimes(t *testing.T) { ctx := context.Background() - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) + fakeCAS := newFakeBlobAccess(testCDCParams) + fakeCLS := newFakeBlobAccess(nil) validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, maximumMessageSizeBytes) digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) @@ -361,23 +341,16 @@ func TestChunkListValidatingBlobAccessPutExtendsLifetimes(t *testing.T) { fakeCAS.ResetTouches() - splitResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{ - chunk1Digest.GetProto(), - chunk2Digest.GetProto(), - }, - } - reqBuffer := buffer.NewProtoBufferFromProto(splitResponse, buffer.UserProvided) - - err := validatingCLS.Put(ctx, fullBlobDigest, reqBuffer) + err := validatingCLS.Put(ctx, fullBlobDigest, makeChunkListBuffer(chunk1Digest, chunk2Digest)) // From the REAPI, the server may either process the splice and - // return OK, OR it may return ALREADY_EXISTS if the blob is already - // composed and the server chooses not to extend the lifetime of the - // user's specific chunks. + // return OK, OR it may return ALREADY_EXISTS if the blob is + // already composed and the server chooses not to extend the + // lifetime of the user's specific chunks. if status.Code(err) == codes.AlreadyExists { - // The server is free not to touch the user's chunks. However, - // it MUST still have verified/touched the original blob. + // The server is free not to touch the user's chunks. + // However, it MUST still have verified/touched the original + // blob. require.Greater(t, fakeCAS.GetTouches(fullBlobDigest), 0, "Composed blob lifetime was not extended during SpliceBlob") require.Greater(t, fakeCLS.GetTouches(fullBlobDigest), 0, "Composed blob chunk list lifetime was not extended during SpliceBlob") } else { @@ -392,19 +365,3 @@ func TestChunkListValidatingBlobAccessPutExtendsLifetimes(t *testing.T) { require.Greater(t, fakeCLS.GetTouches(fullBlobDigest), 0, "Composed blob chunk list lifetime was not extended during SpliceBlob") } } - -func TestChunkListValidatingBlobAccessGetMissingBlob(t *testing.T) { - ctx := context.Background() - - fakeCAS := newFakeBlobAccess(nil) - fakeCLS := newFakeBlobAccess(testCDCParams) - validatingCLS := chunklistvalidating.NewChunkListValidatingBlobAccess(fakeCLS, fakeCAS, 1024*1024) - - digestFunction := digest.MustNewFunction("instance", remoteexecution.DigestFunction_SHA256) - ghostDigest := mustComputeDigest(t, digestFunction, []byte("ghost")) - - _, err := validatingCLS.Get(ctx, ghostDigest).ToProto(&remoteexecution.SplitBlobResponse{}, 1024*1024) - - require.Error(t, err) - require.Equal(t, codes.NotFound, status.Code(err)) -} diff --git a/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel b/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel deleted file mode 100644 index ca423091e..000000000 --- a/pkg/blobstore/chunklistvalidating/integration/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_go//go:def.bzl", "go_test") - -go_test( - name = "integration_test", - srcs = ["chunk_list_validating_integration_test.go"], - data = ["//cmd/bb_storage"], - env = { - "BB_STORAGE_RUNFILE_PATH": "$(rlocationpath //cmd/bb_storage:bb_storage)", - }, - deps = [ - "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", - "@com_github_stretchr_testify//require", - "@org_golang_google_grpc//:grpc", - "@org_golang_google_grpc//codes", - "@org_golang_google_grpc//credentials/insecure", - "@org_golang_google_grpc//status", - "@rules_go//go/runfiles", - ], -) diff --git a/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go b/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go deleted file mode 100644 index 033ec595c..000000000 --- a/pkg/blobstore/chunklistvalidating/integration/chunk_list_validating_integration_test.go +++ /dev/null @@ -1,445 +0,0 @@ -package integration - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "fmt" - "math/rand" - "os" - "os/exec" - "strings" - "testing" - "time" - - remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" - "github.com/bazelbuild/rules_go/go/runfiles" - "github.com/stretchr/testify/require" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/status" -) - -type serverParams struct { - disableCLS bool - socketPath string - upstreamSocketPath string -} - -func escapeJSON(s string) string { - s = strings.ReplaceAll(s, "\\", "\\\\") - s = strings.ReplaceAll(s, "\"", "\\\"") - s = strings.ReplaceAll(s, "'", "\\'") - s = strings.ReplaceAll(s, "\n", "\\n") - return s -} - -func storageConfig(params serverParams) string { - return fmt.Sprintf(` -local cls = %t; -local listenPath = '%s'; -{ - grpcServers: [{ - listenPaths: [listenPath], - authenticationPolicy: { allow: {} }, - }], - maximumMessageSizeBytes: 4 * 1024 * 1024, - contentAddressableStorage: { - backend: { - 'local': { - keyLocationMapInMemory: { entries: 1024 }, - keyLocationMapMaximumGetAttempts: 32, - keyLocationMapMaximumPutAttempts: 64, - oldBlocks: 1, - currentBlocks: 1, - newBlocks: 1, - blocksInMemory: { blockSizeBytes: 1024 * 1024 }, - }, - }, - getAuthorizer: { allow: {} }, - putAuthorizer: { allow: {} }, - findMissingAuthorizer: { allow: {} }, - }, - chunkListStorage: if !cls then null else { - backend: { - 'local': { - keyLocationMapInMemory: { entries: 1024 }, - keyLocationMapMaximumGetAttempts: 32, - keyLocationMapMaximumPutAttempts: 64, - oldBlocks: 1, - currentBlocks: 1, - newBlocks: 1, - blocksInMemory: { blockSizeBytes: 1024 * 1024 }, - chunkingParameters: { - minChunkSizeBytes: 256, - horizonSizeBytes: 8*256, - } - }, - }, - getAuthorizer: { allow: {} }, - putAuthorizer: { allow: {} }, - findMissingAuthorizer: { allow: {} }, - }, -} -`, !params.disableCLS, escapeJSON(params.socketPath)) -} - -func frontendConfig(params serverParams) string { - return fmt.Sprintf(` -local cls = %t; -local listenPath = '%s'; -// unix:// doesn't work under Windows. -// https://github.com/grpc/grpc-go/issues/8675 -local upstreamAddress = 'unix:%s'; -{ - grpcServers: [{ - listenPaths: [listenPath], - authenticationPolicy: { allow: {} }, - }], - maximumMessageSizeBytes: 4 * 1024 * 1024, - contentAddressableStorage: { - backend: { grpc: { client: { address: upstreamAddress } } }, - getAuthorizer: { allow: {} }, - putAuthorizer: { allow: {} }, - findMissingAuthorizer: { allow: {} }, - }, - chunkListStorage: if !cls then null else { - backend: { chunkListValidating: { backend: { grpc: { client: { address: upstreamAddress } } } } }, - getAuthorizer: { allow: {} }, - putAuthorizer: { allow: {} }, - findMissingAuthorizer: { allow: {} }, - }, -} -`, !params.disableCLS, escapeJSON(params.socketPath), escapeJSON(params.upstreamSocketPath)) -} - -func writeConfigFile(name, content string) (file *os.File, err error) { - if file, err = os.CreateTemp("", name); err != nil { - return nil, err - } - if _, err = file.WriteString(content); err != nil { - return nil, err - } - if err = file.Close(); err != nil { - return nil, err - } - return file, nil -} - -func setupServer(t *testing.T, name, config string) func() { - rf, err := runfiles.New() - if err != nil { - t.Fatalf("Failed to initialize runfiles: %v", err) - } - runfilePath := os.Getenv("BB_STORAGE_RUNFILE_PATH") - require.NotEmpty(t, runfilePath, "BB_STORAGE_RUNFILE_PATH environment variable is not set") - - bbStoragePath, err := rf.Rlocation(runfilePath) - require.NoError(t, err) - - configFile, err := writeConfigFile(name, config) - require.NoError(t, err) - - cmd := exec.Command(bbStoragePath, configFile.Name()) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Start() - require.NoError(t, err) - - return func() { - cmd.Process.Kill() - cmd.Wait() - os.Remove(configFile.Name()) - } -} - -func createSocketPath(t *testing.T) string { - t.Helper() - socketFile, err := os.CreateTemp("", "bb_*.sock") - require.NoError(t, err) - socketPath := socketFile.Name() - socketFile.Close() - err = os.Remove(socketPath) - require.NoError(t, err) - return socketPath -} - -func setupServers(t *testing.T, storageParams, frontendParams serverParams) (func(), remoteexecution.CapabilitiesClient, remoteexecution.ContentAddressableStorageClient) { - storageParams.socketPath = createSocketPath(t) - closeStorage := setupServer(t, "storage", storageConfig(storageParams)) - require.Eventually(t, func() bool { - _, err := os.Stat(storageParams.socketPath) - return err == nil - }, 1*time.Second, 10*time.Millisecond, "Storage server did not start.") - - frontendParams.socketPath = createSocketPath(t) - frontendParams.upstreamSocketPath = storageParams.socketPath - closeFrontend := setupServer(t, "frontend", frontendConfig(frontendParams)) - require.Eventually(t, func() bool { - _, err := os.Stat(frontendParams.socketPath) - return err == nil - }, 1*time.Second, 10*time.Millisecond, "Frontend server did not start.") - - conn, err := grpc.NewClient(fmt.Sprintf("unix:%s", frontendParams.socketPath), grpc.WithTransportCredentials(insecure.NewCredentials())) - require.NoError(t, err) - - return func() { - closeStorage() - closeFrontend() - conn.Close() - os.Remove(storageParams.socketPath) - os.Remove(frontendParams.socketPath) - }, remoteexecution.NewCapabilitiesClient(conn), remoteexecution.NewContentAddressableStorageClient(conn) -} - -func TestChunkListValidatingCapabilities(t *testing.T) { - tests := []struct { - name string - storageParams serverParams - frontendParams serverParams - expectSupport bool - }{ - {"Enabled In Both", serverParams{}, serverParams{}, true}, - {"Disabled in Storage", serverParams{disableCLS: true}, serverParams{}, false}, - {"Disabled in Frontend", serverParams{}, serverParams{disableCLS: true}, false}, - {"Disabled in Both", serverParams{disableCLS: true}, serverParams{disableCLS: true}, false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - closer, capabilitiesClient, _ := setupServers(t, tc.storageParams, tc.frontendParams) - defer closer() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - capabilities, err := capabilitiesClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ - InstanceName: "", - }) - require.NoError(t, err) - - cacheCaps := capabilities.CacheCapabilities - require.NotNil(t, cacheCaps) - - if tc.expectSupport { - require.True(t, cacheCaps.SpliceBlobSupport) - require.True(t, cacheCaps.SplitBlobSupport) - - chunkingParameters := cacheCaps.GetRepMaxCdcParams() - require.NotNil(t, chunkingParameters) - require.Equal(t, uint64(256), chunkingParameters.GetMinChunkSizeBytes()) - require.Equal(t, uint64(2048), chunkingParameters.GetHorizonSizeBytes()) - } else { - require.False(t, cacheCaps.SpliceBlobSupport) - require.False(t, cacheCaps.SplitBlobSupport) - require.Nil(t, cacheCaps.GetRepMaxCdcParams()) - } - }) - } -} - -func computeDigest(data []byte) *remoteexecution.Digest { - hash := sha256.Sum256(data) - return &remoteexecution.Digest{ - Hash: hex.EncodeToString(hash[:]), - SizeBytes: int64(len(data)), - } -} - -func makeRandomData(t *testing.T, size int, seed int64) []byte { - t.Helper() - data := make([]byte, size) - r := rand.New(rand.NewSource(seed)) - _, err := r.Read(data) - require.NoError(t, err) - return data -} - -func uploadBlob(ctx context.Context, t *testing.T, cas remoteexecution.ContentAddressableStorageClient, data []byte) *remoteexecution.Digest { - t.Helper() - digest := computeDigest(data) - req := &remoteexecution.BatchUpdateBlobsRequest{ - Requests: []*remoteexecution.BatchUpdateBlobsRequest_Request{ - {Digest: digest, Data: data}, - }, - DigestFunction: remoteexecution.DigestFunction_SHA256, - } - res, err := cas.BatchUpdateBlobs(ctx, req) - require.NoError(t, err) - require.NotEmpty(t, res.Responses, "server returned empty responses array") - status := res.Responses[0].GetStatus() - require.Equal(t, int32(0), status.GetCode(), status.GetMessage()) - return digest -} - -func findMissingBlobs(ctx context.Context, t *testing.T, cas remoteexecution.ContentAddressableStorageClient, digests []*remoteexecution.Digest) []*remoteexecution.Digest { - t.Helper() - req := &remoteexecution.FindMissingBlobsRequest{ - BlobDigests: digests, - DigestFunction: remoteexecution.DigestFunction_SHA256, - } - res, err := cas.FindMissingBlobs(ctx, req) - require.NoError(t, err) - return res.MissingBlobDigests -} - -func TestRepMaxCDCSplitAndSpliceBehaviors(t *testing.T) { - minChunkSize := int64(256) - - t.Run("RoundTripSplitThenSplice", func(t *testing.T) { - closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) - defer closer() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - dataSize := (minChunkSize * 4) + 128 - data := makeRandomData(t, int(dataSize), 0) - blobDigest := uploadBlob(ctx, t, casClient, data) - - splitReq := &remoteexecution.SplitBlobRequest{ - BlobDigest: blobDigest, - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, - DigestFunction: remoteexecution.DigestFunction_SHA256, - } - splitRes, err := casClient.SplitBlob(ctx, splitReq) - require.NoError(t, err) - - spliceReq := &remoteexecution.SpliceBlobRequest{ - BlobDigest: blobDigest, - ChunkDigests: splitRes.ChunkDigests, - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, - DigestFunction: remoteexecution.DigestFunction_SHA256, - } - spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) - require.NoError(t, err) - require.Equal(t, blobDigest.Hash, spliceRes.BlobDigest.Hash) - }) - - t.Run("SpliceNonStandardChunkingThenSplit", func(t *testing.T) { - closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) - defer closer() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - blobData := makeRandomData(t, int(minChunkSize*2), 0) - - chunk1 := blobData[:1] - chunk2 := blobData[1:] - - digest1 := uploadBlob(ctx, t, casClient, chunk1) - digest2 := uploadBlob(ctx, t, casClient, chunk2) - expectedDigest := computeDigest(blobData) - - spliceReq := &remoteexecution.SpliceBlobRequest{ - BlobDigest: expectedDigest, - ChunkDigests: []*remoteexecution.Digest{digest1, digest2}, - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, - DigestFunction: remoteexecution.DigestFunction_SHA256, - } - spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) - require.NoError(t, err) - require.Equal(t, expectedDigest.Hash, spliceRes.BlobDigest.Hash) - - splitReq := &remoteexecution.SplitBlobRequest{ - BlobDigest: expectedDigest, - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, - DigestFunction: remoteexecution.DigestFunction_SHA256, - } - splitRes, err := casClient.SplitBlob(ctx, splitReq) - require.NoError(t, err) - - require.NotEmpty(t, splitRes.ChunkDigests) - - // Check that it didn't just echo our chunks back - isEcho := len(splitRes.ChunkDigests) == 2 && - splitRes.ChunkDigests[0].Hash == digest1.Hash && - splitRes.ChunkDigests[1].Hash == digest2.Hash - require.False(t, isEcho, "Server echoed non-standard chunks") - - var totalSize int64 - for _, c := range splitRes.ChunkDigests { - totalSize += c.SizeBytes - } - require.Equal(t, expectedDigest.SizeBytes, totalSize) - }) - - t.Run("SpliceAlreadyExistsOrNoop", func(t *testing.T) { - closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) - defer closer() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - blobData := append([]byte("This blob will be fully uploaded before we try to splice it."), makeRandomData(t, 16, 0)...) - expectedDigest := uploadBlob(ctx, t, casClient, blobData) - - chunk1 := blobData[:10] - chunk2 := blobData[10:] - digest1 := uploadBlob(ctx, t, casClient, chunk1) - digest2 := uploadBlob(ctx, t, casClient, chunk2) - - spliceReq := &remoteexecution.SpliceBlobRequest{ - BlobDigest: expectedDigest, - ChunkDigests: []*remoteexecution.Digest{digest1, digest2}, - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, - DigestFunction: remoteexecution.DigestFunction_SHA256, - } - - spliceRes, err := casClient.SpliceBlob(ctx, spliceReq) - - if err != nil { - require.Equal(t, codes.AlreadyExists, status.Code(err), "Expected OK or ALREADY_EXISTS") - } else { - require.Equal(t, expectedDigest.Hash, spliceRes.BlobDigest.Hash) - } - }) - - t.Run("ValidationSpliceBlobRejections", func(t *testing.T) { - closer, _, casClient := setupServers(t, serverParams{}, serverParams{}) - defer closer() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - validData := makeRandomData(t, 512, 0) - validDigest := uploadBlob(ctx, t, casClient, validData) - ghostDigest := computeDigest([]byte("I do not exist")) - - tests := []struct { - name string - req *remoteexecution.SpliceBlobRequest - expectError codes.Code - }{ - { - name: "Missing Chunk", - req: &remoteexecution.SpliceBlobRequest{ - BlobDigest: ghostDigest, - ChunkDigests: []*remoteexecution.Digest{ghostDigest}, - }, - expectError: codes.NotFound, - }, - { - name: "Digest Mismatch", - req: &remoteexecution.SpliceBlobRequest{ - BlobDigest: computeDigest([]byte("Fake target")), - ChunkDigests: []*remoteexecution.Digest{validDigest}, - }, - expectError: codes.InvalidArgument, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tc.req.ChunkingFunction = remoteexecution.ChunkingFunction_REP_MAX_CDC - tc.req.DigestFunction = remoteexecution.DigestFunction_SHA256 - - _, err := casClient.SpliceBlob(ctx, tc.req) - require.Error(t, err) - require.Equal(t, tc.expectError, status.Code(err)) - }) - } - }) -} diff --git a/pkg/blobstore/cls_read_buffer_factory.go b/pkg/blobstore/cls_read_buffer_factory.go index 7735f064b..071781c55 100644 --- a/pkg/blobstore/cls_read_buffer_factory.go +++ b/pkg/blobstore/cls_read_buffer_factory.go @@ -3,19 +3,19 @@ package blobstore import ( "io" - remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" ) type clsReadBufferFactory struct{} func (clsReadBufferFactory) NewBufferFromByteSlice(digest digest.Digest, data []byte, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { - return buffer.NewProtoBufferFromByteSlice(&remoteexecution.SplitBlobResponse{}, data, buffer.BackendProvided(dataIntegrityCallback)) + return buffer.NewProtoBufferFromByteSlice(&chunklist_pb.ChunkList{}, data, buffer.BackendProvided(dataIntegrityCallback)) } func (clsReadBufferFactory) NewBufferFromReader(digest digest.Digest, r io.ReadCloser, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { - return buffer.NewProtoBufferFromReader(&remoteexecution.SplitBlobResponse{}, r, buffer.BackendProvided(dataIntegrityCallback)) + return buffer.NewProtoBufferFromReader(&chunklist_pb.ChunkList{}, r, buffer.BackendProvided(dataIntegrityCallback)) } func (f clsReadBufferFactory) NewBufferFromReaderAt(digest digest.Digest, r buffer.ReadAtCloser, sizeBytes int64, dataIntegrityCallback buffer.DataIntegrityCallback) buffer.Buffer { diff --git a/pkg/blobstore/completenesschecking/BUILD.bazel b/pkg/blobstore/completenesschecking/BUILD.bazel index 17cf2c617..33a9d5b79 100644 --- a/pkg/blobstore/completenesschecking/BUILD.bazel +++ b/pkg/blobstore/completenesschecking/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/util", @@ -25,6 +26,8 @@ go_test( ":completenesschecking", "//internal/mock", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/digest", "//pkg/testutil", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go b/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go index 992d94d4d..115aad87b 100644 --- a/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go +++ b/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go @@ -7,6 +7,7 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -21,7 +22,7 @@ import ( type findMissingQueue struct { context context.Context digestFunction digest.Function - contentAddressableStorage blobstore.BlobAccess + contentAddressableStorage cdc.ContentAddressableStorage batchSize int pending digest.SetBuilder @@ -73,7 +74,7 @@ func (q *findMissingQueue) finalize() error { type completenessCheckingBlobAccess struct { blobstore.BlobAccess - contentAddressableStorage blobstore.BlobAccess + contentAddressableStorage cdc.ContentAddressableStorage batchSize int maximumMessageSizeBytes int maximumTotalTreeSizeBytes int64 @@ -93,7 +94,7 @@ type completenessCheckingBlobAccess struct { // needs to be rebuilt. By calling it, Bazel indicates that all // associated output files must remain present during the build for // forward progress to be made. -func NewCompletenessCheckingBlobAccess(actionCache, contentAddressableStorage blobstore.BlobAccess, batchSize, maximumMessageSizeBytes int, maximumTotalTreeSizeBytes int64) blobstore.BlobAccess { +func NewCompletenessCheckingBlobAccess(actionCache blobstore.BlobAccess, contentAddressableStorage cdc.ContentAddressableStorage, batchSize, maximumMessageSizeBytes int, maximumTotalTreeSizeBytes int64) blobstore.BlobAccess { return &completenessCheckingBlobAccess{ BlobAccess: actionCache, contentAddressableStorage: contentAddressableStorage, @@ -151,8 +152,10 @@ func (ba *completenessCheckingBlobAccess) checkCompleteness(ctx context.Context, return status.Errorf(codes.NotFound, "Combined size of all output directories exceeds maximum limit of %d bytes", ba.maximumTotalTreeSizeBytes) } remainingTreeSizeBytes -= sizeBytes - - r := ba.contentAddressableStorage.Get(ctx, treeDigest).ToReader() + r, err := cdc.GetReadCloser(ctx, ba.contentAddressableStorage, treeDigest) + if err != nil { + return err + } if err := util.VisitProtoBytesFields(r, func(fieldNumber protowire.Number, offsetBytes, sizeBytes int64, fieldReader io.Reader) error { if fieldNumber == blobstore.TreeRootFieldNumber || fieldNumber == blobstore.TreeChildrenFieldNumber { directoryMessage, err := buffer.NewProtoBufferFromReader( @@ -185,13 +188,6 @@ func (ba *completenessCheckingBlobAccess) checkCompleteness(ctx context.Context, } return nil }); err != nil { - // Any errors generated above may be caused by - // data corruption on the Tree object. Force - // reading the Tree until completion, and prefer - // read errors over any errors generated above. - if _, copyErr := io.Copy(io.Discard, r); copyErr != nil { - err = copyErr - } r.Close() return util.StatusWrapf(err, "Output directory %#v", outputDirectory.Path) } diff --git a/pkg/blobstore/completenesschecking/completeness_checking_blob_access_test.go b/pkg/blobstore/completenesschecking/completeness_checking_blob_access_test.go index 0fe3d6820..b789c4889 100644 --- a/pkg/blobstore/completenesschecking/completeness_checking_blob_access_test.go +++ b/pkg/blobstore/completenesschecking/completeness_checking_blob_access_test.go @@ -2,12 +2,13 @@ package completenesschecking_test import ( "context" - "io" "testing" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/internal/mock" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/completenesschecking" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/testutil" @@ -24,7 +25,8 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) actionCache := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) + completenessCheckingBlobAccess := completenesschecking.NewCompletenessCheckingBlobAccess( actionCache, contentAddressableStorage, @@ -149,13 +151,27 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { buffer.BackendProvided(dataIntegrityCallback.Call), ), ) - contentAddressableStorage.EXPECT().Get( + treeDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) + contentAddressableStorage.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("hello"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 2, HorizonSizeBytes: 4}, nil) + contentAddressableStorage.EXPECT().GetManifest( ctx, - digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - ).Return(buffer.NewBufferFromError(status.Error(codes.Internal, "Hard disk has a case of the Mondays"))) + treeDigest, + ).Return( + chunklist.ChunkList{{Offset: 0, Digest: treeDigest}}, + nil, + ) + // The tree's only chunk fails while the tree is being + // traversed. + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + treeDigest, + ).Return(nil, status.Error(codes.Internal, "Hard disk has a case of the Mondays")) _, err := completenessCheckingBlobAccess.Get(ctx, actionDigest).ToProto(&remoteexecution.ActionResult{}, 1000) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Output directory \"bazel-out/foo\": Hard disk has a case of the Mondays"), err) + testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Output directory \"bazel-out/foo\": Failed to fetch chunk at index 0: Hard disk has a case of the Mondays"), err) }) t.Run("GetTreeTooLarge", func(t *testing.T) { @@ -185,102 +201,6 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Combined size of all output directories exceeds maximum limit of 10000 bytes"), err) }) - t.Run("GetTreeDataCorruption", func(t *testing.T) { - // Because Tree objects are processed in a streaming - // fashion, it may be the case that we call - // FindMissing() against the CAS, even though we later - // discover that the Tree object was corrupted. - // - // This means that even if FindMissing() reports objects - // as being absent, we cannot terminate immediately. We - // must process the Tree object in its entirety. - dataIntegrityCallback1 := mock.NewMockDataIntegrityCallback(ctrl) - dataIntegrityCallback1.EXPECT().Call(true) - actionCache.EXPECT().Get(ctx, actionDigest).Return( - buffer.NewProtoBufferFromProto( - &remoteexecution.ActionResult{ - OutputDirectories: []*remoteexecution.OutputDirectory{ - { - Path: "bazel-out/foo", - TreeDigest: &remoteexecution.Digest{ - Hash: "8f0450aa5f4602d93968daba6f2e7611", - SizeBytes: 4000, - }, - }, - }, - }, - buffer.BackendProvided(dataIntegrityCallback1.Call), - ), - ) - - treeReader := mock.NewMockReadCloser(ctrl) - treeReader.EXPECT().Read(gomock.Any()). - DoAndReturn(func(p []byte) (int, error) { - treeData, err := proto.Marshal(&remoteexecution.Tree{ - Root: &remoteexecution.Directory{ - Files: []*remoteexecution.FileNode{ - { - Digest: &remoteexecution.Digest{ - Hash: "024ced29f1fdef2f644f34a071ade5be", - SizeBytes: 1, - }, - }, - { - Digest: &remoteexecution.Digest{ - Hash: "8b3b146b1c4df062a2dc35168cbf4ce6", - SizeBytes: 2, - }, - }, - { - Digest: &remoteexecution.Digest{ - Hash: "4a4a6ebb3f8b062653cb957cbdc047d9", - SizeBytes: 3, - }, - }, - { - Digest: &remoteexecution.Digest{ - Hash: "69778ed3e4dcf4e0c40df49e4ca5bd37", - SizeBytes: 4, - }, - }, - { - Digest: &remoteexecution.Digest{ - Hash: "ff7816e0353299e801a30e37aee1758c", - SizeBytes: 5, - }, - }, - }, - }, - }) - require.NoError(t, err) - return copy(p, treeData), nil - }) - treeReader.EXPECT().Read(gomock.Any()). - DoAndReturn(func(p []byte) (int, error) { - return copy(p, "Garbage"), io.EOF - }) - treeReader.EXPECT().Close() - dataIntegrityCallback2 := mock.NewMockDataIntegrityCallback(ctrl) - dataIntegrityCallback2.EXPECT().Call(false) - treeDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8f0450aa5f4602d93968daba6f2e7611", 4000) - contentAddressableStorage.EXPECT().Get(ctx, treeDigest).Return( - buffer.NewCASBufferFromReader(treeDigest, treeReader, buffer.BackendProvided(dataIntegrityCallback2.Call)), - ) - contentAddressableStorage.EXPECT().FindMissing( - ctx, - digest.NewSetBuilder(0). - Add(treeDigest). - Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "024ced29f1fdef2f644f34a071ade5be", 1)). - Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b3b146b1c4df062a2dc35168cbf4ce6", 2)). - Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "4a4a6ebb3f8b062653cb957cbdc047d9", 3)). - Add(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "69778ed3e4dcf4e0c40df49e4ca5bd37", 4)). - Build(), - ).Return(digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "4a4a6ebb3f8b062653cb957cbdc047d9", 3).ToSingletonSet(), nil) - - _, err := completenessCheckingBlobAccess.Get(ctx, actionDigest).ToProto(&remoteexecution.ActionResult{}, 1000) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Output directory \"bazel-out/foo\": Buffer is 210 bytes in size, while 4000 bytes were expected"), err) - }) - t.Run("Success", func(t *testing.T) { // Successful checking of existence of dependencies. // Below is an ActionResult that contains five @@ -335,12 +255,7 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { buffer.BackendProvided(dataIntegrityCallback1.Call), ), ) - dataIntegrityCallback2 := mock.NewMockDataIntegrityCallback(ctrl) - dataIntegrityCallback2.EXPECT().Call(true) - contentAddressableStorage.EXPECT().Get( - ctx, - digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 200), - ).Return(buffer.NewProtoBufferFromProto(&remoteexecution.Tree{ + tree := &remoteexecution.Tree{ Root: &remoteexecution.Directory{ // Directory digests should not be part of // FindMissing(), as references to directories @@ -383,7 +298,25 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { }, {}, }, - }, buffer.BackendProvided(dataIntegrityCallback2.Call))) + } + treeBytes, err := proto.Marshal(tree) + require.NoError(t, err) + treeDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 200) + contentAddressableStorage.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("hello"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 2, HorizonSizeBytes: 4}, nil) + contentAddressableStorage.EXPECT().GetManifest( + ctx, + treeDigest, + ).Return( + chunklist.ChunkList{{Offset: 0, Digest: treeDigest}}, + nil, + ) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + treeDigest, + ).Return(treeBytes, nil) contentAddressableStorage.EXPECT().FindMissing( ctx, digest.NewSetBuilder(0). @@ -410,3 +343,11 @@ func TestCompletenessCheckingBlobAccess(t *testing.T) { testutil.RequireEqualProto(t, &actionResult, actualResult) }) } + +func mustNewInstanceName(name string) digest.InstanceName { + instanceName, err := digest.NewInstanceName(name) + if err != nil { + panic(err) + } + return instanceName +} diff --git a/pkg/blobstore/configuration/BUILD.bazel b/pkg/blobstore/configuration/BUILD.bazel index 8882b2b2e..79c4cedef 100644 --- a/pkg/blobstore/configuration/BUILD.bazel +++ b/pkg/blobstore/configuration/BUILD.bazel @@ -6,9 +6,9 @@ go_library( "ac_blob_access_creator.go", "blob_access_creator.go", "blob_replicator_creator.go", - "cas_blob_access_creator.go", - "cas_blob_replicator_creator.go", "cls_blob_access_creator.go", + "cs_blob_access_creator.go", + "cs_blob_replicator_creator.go", "fsac_blob_access_creator.go", "icas_blob_access_creator.go", "icas_blob_replicator_creator.go", @@ -22,6 +22,8 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/blobstore", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/blobstore/chunklistvalidating", "//pkg/blobstore/completenesschecking", "//pkg/blobstore/grpcclients", @@ -29,6 +31,7 @@ go_library( "//pkg/blobstore/mirrored", "//pkg/blobstore/readcaching", "//pkg/blobstore/readfallback", + "//pkg/blobstore/referenceexpanding", "//pkg/blobstore/replication", "//pkg/blobstore/sharding", "//pkg/blockdevice", @@ -46,6 +49,7 @@ go_library( "//pkg/proto/configuration/blobstore", "//pkg/proto/configuration/digest", "//pkg/random", + "//pkg/ttlcache", "//pkg/util", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/configuration/ac_blob_access_creator.go b/pkg/blobstore/configuration/ac_blob_access_creator.go index 14999a233..b521dabb7 100644 --- a/pkg/blobstore/configuration/ac_blob_access_creator.go +++ b/pkg/blobstore/configuration/ac_blob_access_creator.go @@ -3,6 +3,7 @@ package configuration import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/completenesschecking" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" "github.com/buildbarn/bb-storage/pkg/capabilities" @@ -31,7 +32,7 @@ type acBlobAccessCreator struct { protoBlobAccessCreator protoBlobReplicatorCreator - contentAddressableStorage *BlobAccessInfo + contentAddressableStorage cdc.ContentAddressableStorage grpcClientFactory grpc.ClientFactory maximumMessageSizeBytes int } @@ -39,7 +40,7 @@ type acBlobAccessCreator struct { // NewACBlobAccessCreator creates a BlobAccessCreator that can be // provided to NewBlobAccessFromConfiguration() to construct a // BlobAccess that is suitable for accessing the Action Cache. -func NewACBlobAccessCreator(contentAddressableStorage *BlobAccessInfo, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { +func NewACBlobAccessCreator(contentAddressableStorage cdc.ContentAddressableStorage, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { return &acBlobAccessCreator{ contentAddressableStorage: contentAddressableStorage, grpcClientFactory: grpcClientFactory, @@ -100,20 +101,22 @@ func (bac *acBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gro return BlobAccessInfo{ BlobAccess: completenesschecking.NewCompletenessCheckingBlobAccess( base.BlobAccess, - bac.contentAddressableStorage.BlobAccess, + bac.contentAddressableStorage, blobstore.RecommendedFindMissingDigestsCount, bac.maximumMessageSizeBytes, backend.CompletenessChecking.MaximumTotalTreeSizeBytes, ), - DigestKeyFormat: base.DigestKeyFormat.Combine(bac.contentAddressableStorage.DigestKeyFormat), + DigestKeyFormat: base.DigestKeyFormat.Combine(bac.contentAddressableStorage.GetDigestKeyFormat()), }, "completeness_checking", nil case *pb.BlobAccessConfiguration_Grpc: - client, err := bac.grpcClientFactory.NewClientFromConfiguration(backend.Grpc.Client, terminationGroup) + grpc := backend.Grpc + client, err := bac.grpcClientFactory.NewClientFromConfiguration(grpc.Client, terminationGroup) if err != nil { return BlobAccessInfo{}, "", err } + ba := grpcclients.NewACBlobAccess(client, bac.maximumMessageSizeBytes) return BlobAccessInfo{ - BlobAccess: grpcclients.NewACBlobAccess(client, bac.maximumMessageSizeBytes), + BlobAccess: ba, DigestKeyFormat: digest.KeyWithInstance, }, "grpc", nil default: diff --git a/pkg/blobstore/configuration/cls_blob_access_creator.go b/pkg/blobstore/configuration/cls_blob_access_creator.go index af2e695c9..7fd9e9c49 100644 --- a/pkg/blobstore/configuration/cls_blob_access_creator.go +++ b/pkg/blobstore/configuration/cls_blob_access_creator.go @@ -10,27 +10,25 @@ import ( "github.com/buildbarn/bb-storage/pkg/grpc" "github.com/buildbarn/bb-storage/pkg/program" pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) type clsBlobAccessCreator struct { protoBlobAccessCreator protoBlobReplicatorCreator - contentAddressableStorage *BlobAccessInfo - grpcClientFactory grpc.ClientFactory - maximumMessageSizeBytes int + chunkStorage *BlobAccessInfo + grpcClientFactory grpc.ClientFactory + maximumMessageSizeBytes int } // NewCLSBlobAccessCreator creates a BlobAccessCreator that can be // provided to NewBlobAccessFromConfiguration() to construct a // BlobAccess that is suitable for querying for chunk list. -func NewCLSBlobAccessCreator(contentAddressableStorage *BlobAccessInfo, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { +func NewCLSBlobAccessCreator(chunkStorage *BlobAccessInfo, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { return &clsBlobAccessCreator{ - contentAddressableStorage: contentAddressableStorage, - grpcClientFactory: grpcClientFactory, - maximumMessageSizeBytes: maximumMessageSizeBytes, + chunkStorage: chunkStorage, + grpcClientFactory: grpcClientFactory, + maximumMessageSizeBytes: maximumMessageSizeBytes, } } @@ -49,10 +47,6 @@ func (clsBlobAccessCreator) GetDefaultCapabilitiesProvider() capabilities.Provid func (bac *clsBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { switch backend := configuration.Backend.(type) { case *pb.BlobAccessConfiguration_ChunkListValidating: - if bac.contentAddressableStorage == nil { - return BlobAccessInfo{}, "", status.Error(codes.InvalidArgument, "Chunk list validation can only be enabled if a Content Addressable Storage is configured") - } - base, err := nestedCreator.NewNestedBlobAccess(backend.ChunkListValidating.Backend, bac) if err != nil { return BlobAccessInfo{}, "", err @@ -60,19 +54,21 @@ func (bac *clsBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr return BlobAccessInfo{ BlobAccess: chunklistvalidating.NewChunkListValidatingBlobAccess( base.BlobAccess, - bac.contentAddressableStorage.BlobAccess, + bac.chunkStorage.BlobAccess, bac.maximumMessageSizeBytes, ), - DigestKeyFormat: base.DigestKeyFormat.Combine(bac.contentAddressableStorage.DigestKeyFormat), + DigestKeyFormat: base.DigestKeyFormat.Combine(bac.chunkStorage.DigestKeyFormat), }, "chunk_list_validating", nil case *pb.BlobAccessConfiguration_Grpc: - client, err := bac.grpcClientFactory.NewClientFromConfiguration(backend.Grpc.Client, terminationGroup) + grpc := backend.Grpc + client, err := bac.grpcClientFactory.NewClientFromConfiguration(grpc.Client, terminationGroup) if err != nil { return BlobAccessInfo{}, "", err } + ba := grpcclients.NewCLSBlobAccess(client, bac.maximumMessageSizeBytes) return BlobAccessInfo{ - BlobAccess: grpcclients.NewCLSBlobAccess(client, bac.maximumMessageSizeBytes), + BlobAccess: ba, DigestKeyFormat: digest.KeyWithInstance, }, "grpc", nil diff --git a/pkg/blobstore/configuration/cas_blob_access_creator.go b/pkg/blobstore/configuration/cs_blob_access_creator.go similarity index 71% rename from pkg/blobstore/configuration/cas_blob_access_creator.go rename to pkg/blobstore/configuration/cs_blob_access_creator.go index ac215dd68..87db8201b 100644 --- a/pkg/blobstore/configuration/cas_blob_access_creator.go +++ b/pkg/blobstore/configuration/cs_blob_access_creator.go @@ -8,8 +8,10 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" "github.com/buildbarn/bb-storage/pkg/blobstore/local" + "github.com/buildbarn/bb-storage/pkg/blobstore/referenceexpanding" "github.com/buildbarn/bb-storage/pkg/capabilities" "github.com/buildbarn/bb-storage/pkg/cloud/aws" "github.com/buildbarn/bb-storage/pkg/cloud/gcp" @@ -28,27 +30,26 @@ import ( "cloud.google.com/go/storage" ) -var casCapabilitiesProvider = capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ +var csCapabilitiesProvider = capabilities.NewStaticProvider(&remoteexecution.ServerCapabilities{ CacheCapabilities: &remoteexecution.CacheCapabilities{ DigestFunctions: digest.SupportedDigestFunctions, // MaxBatchTotalSize: Not used by Bazel yet. }, }) -type casBlobAccessCreator struct { - casBlobReplicatorCreator +type csBlobAccessCreator struct { + csBlobReplicatorCreator maximumMessageSizeBytes int zstdPool bb_zstd.Pool } -// NewCASBlobAccessCreator creates a BlobAccessCreator that can be +// NewCSBlobAccessCreator creates a BlobAccessCreator that can be // provided to NewBlobAccessFromConfiguration() to construct a -// BlobAccess that is suitable for accessing the Content Addressable -// Storage. -func NewCASBlobAccessCreator(grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) BlobAccessCreator { - return &casBlobAccessCreator{ - casBlobReplicatorCreator: casBlobReplicatorCreator{ +// BlobAccess that is suitable for accessing the Chunk Storage. +func NewCSBlobAccessCreator(grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) BlobAccessCreator { + return &csBlobAccessCreator{ + csBlobReplicatorCreator: csBlobReplicatorCreator{ grpcClientFactory: grpcClientFactory, }, maximumMessageSizeBytes: maximumMessageSizeBytes, @@ -56,27 +57,27 @@ func NewCASBlobAccessCreator(grpcClientFactory grpc.ClientFactory, maximumMessag } } -func (casBlobAccessCreator) GetBaseDigestKeyFormat() digest.KeyFormat { +func (csBlobAccessCreator) GetBaseDigestKeyFormat() digest.KeyFormat { return digest.KeyWithoutInstance } -func (casBlobAccessCreator) GetReadBufferFactory() blobstore.ReadBufferFactory { +func (csBlobAccessCreator) GetReadBufferFactory() blobstore.ReadBufferFactory { return blobstore.CASReadBufferFactory } -func (casBlobAccessCreator) GetDefaultCapabilitiesProvider() capabilities.Provider { - return casCapabilitiesProvider +func (csBlobAccessCreator) GetDefaultCapabilitiesProvider() capabilities.Provider { + return csCapabilitiesProvider } -func (casBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks int) (local.BlockListGrowthPolicy, error) { +func (csBlobAccessCreator) NewBlockListGrowthPolicy(currentBlocks, newBlocks int) (local.BlockListGrowthPolicy, error) { return local.NewImmutableBlockListGrowthPolicy(currentBlocks, newBlocks), nil } -func (casBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) { +func (csBlobAccessCreator) NewHierarchicalInstanceNamesLocalBlobAccess(keyLocationMap local.KeyLocationMap, locationBlobMap local.LocationBlobMap, globalLock *sync.RWMutex, capabilitiesProvider capabilities.Provider) (blobstore.BlobAccess, error) { return local.NewHierarchicalCASBlobAccess(keyLocationMap, locationBlobMap, globalLock, capabilitiesProvider), nil } -func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { +func (bac *csBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Group, configuration *pb.BlobAccessConfiguration, nestedCreator NestedBlobAccessCreator) (BlobAccessInfo, string, error) { switch backend := configuration.Backend.(type) { case *pb.BlobAccessConfiguration_ExistenceCaching: base, err := nestedCreator.NewNestedBlobAccess(backend.ExistenceCaching.Backend, bac) @@ -92,7 +93,8 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr DigestKeyFormat: base.DigestKeyFormat, }, "existence_caching", nil case *pb.BlobAccessConfiguration_Grpc: - client, err := bac.grpcClientFactory.NewClientFromConfiguration(backend.Grpc.Client, terminationGroup) + grpc := backend.Grpc + client, err := bac.grpcClientFactory.NewClientFromConfiguration(grpc.Client, terminationGroup) if err != nil { return BlobAccessInfo{}, "", err } @@ -100,10 +102,11 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr if backend.Grpc.EnableCompression { zstdPool = bac.zstdPool } + ba := grpcclients.NewCSBlobAccess(client, uuid.NewRandom, 64<<10, zstdPool) // TODO: Should we provide a configuration option, so // that digest.KeyWithoutInstance can be used? return BlobAccessInfo{ - BlobAccess: grpcclients.NewCASBlobAccess(client, uuid.NewRandom, 64<<10, zstdPool), + BlobAccess: ba, DigestKeyFormat: digest.KeyWithInstance, }, "grpc", nil case *pb.BlobAccessConfiguration_ReferenceExpanding: @@ -124,15 +127,12 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr return BlobAccessInfo{}, "", err } - var contentAddressableStorage blobstore.BlobAccess + var contentAddressableStorage cdc.ContentAddressableStorage if backend.ReferenceExpanding.ContentAddressableStorage != nil { - backend, err := nestedCreator.NewNestedBlobAccess(backend.ReferenceExpanding.ContentAddressableStorage, bac) - if err != nil { - return BlobAccessInfo{}, "", err - } - contentAddressableStorage = backend.BlobAccess + contentAddressableStorage, _, _, _, _, err = NewCASFromConfiguration(terminationGroup, backend.ReferenceExpanding.ContentAddressableStorage, bac.grpcClientFactory, bac.maximumMessageSizeBytes, bac.zstdPool) } else { - contentAddressableStorage = blobstore.NewErrorBlobAccess(status.Error(codes.Unimplemented, "No Content Addressable Storage configured")) + // TODO: is an equivalent of this needed? + // contentAddressableStorage = blobstore.NewErrorBlobAccess(status.Error(codes.Unimplemented, "No Content Addressable Storage configured")) } awsConfig, err := aws.NewConfigFromConfiguration(backend.ReferenceExpanding.AwsSession, "S3ReferenceExpandingBlobAccess") @@ -159,7 +159,7 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr } return BlobAccessInfo{ - BlobAccess: blobstore.NewReferenceExpandingBlobAccess( + BlobAccess: referenceexpanding.NewReferenceExpandingBlobAccess( indirectContentAddressableStorage.BlobAccess, contentAddressableStorage, &http.Client{ @@ -177,7 +177,7 @@ func (bac *casBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gr } } -func (casBlobAccessCreator) WrapTopLevelBlobAccess(blobAccess blobstore.BlobAccess) blobstore.BlobAccess { +func (csBlobAccessCreator) WrapTopLevelBlobAccess(blobAccess blobstore.BlobAccess) blobstore.BlobAccess { // For the Content Addressable Storage it is required that the empty // blob is always present. This decorator ensures that requests // for the empty blob never contact the storage backend. diff --git a/pkg/blobstore/configuration/cas_blob_replicator_creator.go b/pkg/blobstore/configuration/cs_blob_replicator_creator.go similarity index 66% rename from pkg/blobstore/configuration/cas_blob_replicator_creator.go rename to pkg/blobstore/configuration/cs_blob_replicator_creator.go index ed3172fef..215ebaf03 100644 --- a/pkg/blobstore/configuration/cas_blob_replicator_creator.go +++ b/pkg/blobstore/configuration/cs_blob_replicator_creator.go @@ -11,25 +11,25 @@ import ( "google.golang.org/grpc/status" ) -type casBlobReplicatorCreator struct { +type csBlobReplicatorCreator struct { grpcClientFactory grpc.ClientFactory } -// NewCASBlobReplicatorCreator creates a BlobReplicatorCreator that can +// NewCSBlobReplicatorCreator creates a BlobReplicatorCreator that can // be provided to NewBlobReplicatorFromConfiguration() to construct a -// BlobReplicator that is suitable for replicating Content Addressable -// Storage objects. -func NewCASBlobReplicatorCreator(grpcClientFactory grpc.ClientFactory) BlobReplicatorCreator { - return &casBlobReplicatorCreator{ +// BlobReplicator that is suitable for replicating Chunk Storage +// objects. +func NewCSBlobReplicatorCreator(grpcClientFactory grpc.ClientFactory) BlobReplicatorCreator { + return &csBlobReplicatorCreator{ grpcClientFactory: grpcClientFactory, } } -func (casBlobReplicatorCreator) GetStorageTypeName() string { - return "cas" +func (csBlobReplicatorCreator) GetStorageTypeName() string { + return "cs" } -func (brc *casBlobReplicatorCreator) NewCustomBlobReplicator(terminationGroup program.Group, configuration *pb.BlobReplicatorConfiguration, source blobstore.BlobAccess, sink BlobAccessInfo) (replication.BlobReplicator, error) { +func (brc *csBlobReplicatorCreator) NewCustomBlobReplicator(terminationGroup program.Group, configuration *pb.BlobReplicatorConfiguration, source blobstore.BlobAccess, sink BlobAccessInfo) (replication.BlobReplicator, error) { switch mode := configuration.Mode.(type) { case *pb.BlobReplicatorConfiguration_Deduplicating: base, err := NewBlobReplicatorFromConfiguration(terminationGroup, mode.Deduplicating, source, sink, brc) diff --git a/pkg/blobstore/configuration/new_blob_access.go b/pkg/blobstore/configuration/new_blob_access.go index 07e52041c..fff54563a 100644 --- a/pkg/blobstore/configuration/new_blob_access.go +++ b/pkg/blobstore/configuration/new_blob_access.go @@ -9,6 +9,8 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/local" "github.com/buildbarn/bb-storage/pkg/blobstore/mirrored" "github.com/buildbarn/bb-storage/pkg/blobstore/readcaching" @@ -26,6 +28,7 @@ import ( pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore" digest_pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/digest" "github.com/buildbarn/bb-storage/pkg/random" + "github.com/buildbarn/bb-storage/pkg/ttlcache" "github.com/buildbarn/bb-storage/pkg/util" bb_zstd "github.com/buildbarn/bb-storage/pkg/zstd" "github.com/fxtlabs/primes" @@ -633,16 +636,12 @@ func NewBlobAccessFromConfiguration(terminationGroup program.Group, configuratio }, nil } -// NewCASAndACBlobAccessFromConfiguration is a convenience function to -// create BlobAccess objects for both the Content Addressable Storage -// and Action Cache. Most Buildbarn components tend to require access to -// both these data stores. -func NewCASAndACBlobAccessFromConfiguration(terminationGroup program.Group, configuration *pb.BlobstoreConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (blobstore.BlobAccess, blobstore.BlobAccess, error) { - contentAddressableStorage, err := NewBlobAccessFromConfiguration( - terminationGroup, - configuration.GetContentAddressableStorage(), - NewCASBlobAccessCreator(grpcClientFactory, maximumMessageSizeBytes, zstdPool), - ) +// NewCASAndACFromConfiguration is a convenience function to create a +// Content Addressable Storage (CAS) and a BlobAccess for the Action +// Cache. Most Buildbarn components tend to require access to both these +// data stores. +func NewCASAndACFromConfiguration(terminationGroup program.Group, configuration *pb.BlobstoreConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (cdc.ContentAddressableStorage, blobstore.BlobAccess, error) { + contentAddressableStorage, _, _, _, _, err := NewCASFromConfiguration(terminationGroup, configuration.ContentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes, zstdPool) if err != nil { return nil, nil, util.StatusWrap(err, "Failed to create Content Addressable Storage") } @@ -651,7 +650,7 @@ func NewCASAndACBlobAccessFromConfiguration(terminationGroup program.Group, conf terminationGroup, configuration.GetActionCache(), NewACBlobAccessCreator( - &contentAddressableStorage, + contentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes, ), @@ -660,5 +659,61 @@ func NewCASAndACBlobAccessFromConfiguration(terminationGroup program.Group, conf return nil, nil, util.StatusWrap(err, "Failed to create Action Cache") } - return contentAddressableStorage.BlobAccess, actionCache.BlobAccess, nil + return contentAddressableStorage, actionCache.BlobAccess, nil +} + +// NewCASFromConfiguration is a convenience function to create a +// ContentAddressableStorage (CAS) and its constituent parts from +// configuration. +func NewCASFromConfiguration(terminationGroup program.Group, configuration *pb.ContentAddressableStorageConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (cdc.ContentAddressableStorage, blobstore.BlobAccess, blobstore.BlobAccess, chunklist.Fetcher, cdc.ParametersFetcher, error) { + chunkStorageInfo, err := NewBlobAccessFromConfiguration( + terminationGroup, + configuration.GetChunkStorage(), + NewCSBlobAccessCreator(grpcClientFactory, maximumMessageSizeBytes, zstdPool), + ) + if err != nil { + return nil, nil, nil, nil, nil, util.StatusWrap(err, "Failed to create Chunk Storage") + } + chunkStorage := chunkStorageInfo.BlobAccess + + chunkListStorageInfo, err := NewBlobAccessFromConfiguration( + terminationGroup, + configuration.GetChunkListStorage(), + NewCLSBlobAccessCreator(&chunkStorageInfo, grpcClientFactory, maximumMessageSizeBytes), + ) + if err != nil { + return nil, nil, nil, nil, nil, util.StatusWrap(err, "Failed to create Chunk List Storage") + } + chunkListStorage := chunkListStorageInfo.BlobAccess + var chunkListFetcher chunklist.Fetcher = blobstore.NewBlobAccessChunkListFetcher(chunkListStorage, maximumMessageSizeBytes) + if configuration.GetChunkListCache() != nil { + cache, err := ttlcache.NewTTLCacheFromConfiguration[digest.Digest, chunklist.ChunkList]( + configuration.ChunkListCache, + clock.SystemClock, + "ChunkListCache", + ) + if err != nil { + return nil, nil, nil, nil, nil, util.StatusWrap(err, "Failed to create chunk list cache") + } + chunkListFetcher = chunklist.NewCachingFetcher(chunkListFetcher, cache) + } + + // The chunking parameters are a property of the Chunk Storage + // (CS), so the CDC parameters are fetched from there. + cdcParametersFetcher := cdc.NewCapabilitiesParametersFetcher(chunkStorage) + if configuration.GetCdcParameterCache() != nil { + cache, err := ttlcache.NewTTLCacheFromConfiguration[digest.InstanceName, cdc.Parameters]( + configuration.CdcParameterCache, + clock.SystemClock, + "CDCParameterCache", + ) + if err != nil { + return nil, nil, nil, nil, nil, util.StatusWrap(err, "Failed to create cdc parameter cache") + } + cdcParametersFetcher = cdc.NewCachingParametersFetcher(cdcParametersFetcher, cache) + } + + contentAddressableStorage := cdc.NewContentAddressableStorage(chunkStorage, chunkListStorage, chunkListFetcher, cdcParametersFetcher, chunkStorageInfo.DigestKeyFormat.Combine(chunkListStorageInfo.DigestKeyFormat)) + + return contentAddressableStorage, chunkStorage, chunkListStorage, chunkListFetcher, cdcParametersFetcher, nil } diff --git a/pkg/blobstore/deadline_enforcing_blob_access.go b/pkg/blobstore/deadline_enforcing_blob_access.go index 0faad1f16..79b9db92e 100644 --- a/pkg/blobstore/deadline_enforcing_blob_access.go +++ b/pkg/blobstore/deadline_enforcing_blob_access.go @@ -6,7 +6,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -40,15 +39,6 @@ func (d *deadlineEnforcingBlobAccess) Get(ctx context.Context, digest digest.Dig ) } -func (d *deadlineEnforcingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - ctxWithTimeout, cancel := context.WithTimeout(ctx, d.timeout) - - return buffer.WithErrorHandler( - d.delegate.GetFromComposite(ctxWithTimeout, parentDigest, childDigest, slicer), - &contextCancelingErrorHandler{cancel: cancel}, - ) -} - func (d *deadlineEnforcingBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { ctxWithTimeout, cancel := context.WithTimeout(ctx, d.timeout) defer cancel() diff --git a/pkg/blobstore/demultiplexing_blob_access.go b/pkg/blobstore/demultiplexing_blob_access.go index 0835162bb..3cb4c7040 100644 --- a/pkg/blobstore/demultiplexing_blob_access.go +++ b/pkg/blobstore/demultiplexing_blob_access.go @@ -5,7 +5,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -49,17 +48,6 @@ func (ba *demultiplexingBlobAccess) Get(ctx context.Context, digest digest.Diges ) } -func (ba *demultiplexingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - backend, backendName, patcher, err := ba.getBackend(parentDigest.GetInstanceName()) - if err != nil { - return buffer.NewBufferFromError(err) - } - return buffer.WithErrorHandler( - backend.GetFromComposite(ctx, patcher.PatchDigest(parentDigest), patcher.PatchDigest(childDigest), slicer), - backendNamePrefixingErrorHandler{backendName: backendName}, - ) -} - func (ba *demultiplexingBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { backend, backendName, patcher, err := ba.getBackend(digest.GetInstanceName()) if err != nil { diff --git a/pkg/blobstore/demultiplexing_blob_access_test.go b/pkg/blobstore/demultiplexing_blob_access_test.go index da2e68446..5f0f6e150 100644 --- a/pkg/blobstore/demultiplexing_blob_access_test.go +++ b/pkg/blobstore/demultiplexing_blob_access_test.go @@ -87,90 +87,6 @@ func TestDemultiplexingBlobAccessGet(t *testing.T) { }) } -func TestDemultiplexingBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - demultiplexedBlobAccessGetter := mock.NewMockDemultiplexedBlobAccessGetter(ctrl) - blobAccess := blobstore.NewDemultiplexingBlobAccess(demultiplexedBlobAccessGetter.Call) - - t.Run("UnknownInstanceName", func(t *testing.T) { - // This request cannot be forwarded to any backend. - demultiplexedBlobAccessGetter.EXPECT().Call(util.Must(digest.NewInstanceName("unknown"))).Return( - nil, - "", - digest.NoopInstanceNamePatcher, - status.Error(codes.InvalidArgument, "Unknown instance name"), - ) - blobSlicer := mock.NewMockBlobSlicer(ctrl) - - _, err := blobAccess.GetFromComposite( - ctx, - digest.MustNewDigest("unknown", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("unknown", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Unknown instance name"), err) - }) - - t.Run("BackendFailure", func(t *testing.T) { - // Error messages should have the backend name prepended. - baseBlobAccess := mock.NewMockBlobAccess(ctrl) - demultiplexedBlobAccessGetter.EXPECT().Call(util.Must(digest.NewInstanceName("hello/world"))).Return( - baseBlobAccess, - "Primary", - digest.NewInstanceNamePatcher( - util.Must(digest.NewInstanceName("hello")), - util.Must(digest.NewInstanceName("goodbye")), - ), - nil, - ) - blobSlicer := mock.NewMockBlobSlicer(ctrl) - baseBlobAccess.EXPECT().GetFromComposite( - ctx, - digest.MustNewDigest("goodbye/world", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("goodbye/world", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).Return(buffer.NewBufferFromError(status.Error(codes.Internal, "Server on fire"))) - - _, err := blobAccess.GetFromComposite( - ctx, - digest.MustNewDigest("hello/world", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("hello/world", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Backend \"Primary\": Server on fire"), err) - }) - - t.Run("Success", func(t *testing.T) { - baseBlobAccess := mock.NewMockBlobAccess(ctrl) - demultiplexedBlobAccessGetter.EXPECT().Call(util.Must(digest.NewInstanceName("hello/world"))).Return( - baseBlobAccess, - "Primary", - digest.NewInstanceNamePatcher( - util.Must(digest.NewInstanceName("hello")), - util.Must(digest.NewInstanceName("goodbye")), - ), - nil, - ) - blobSlicer := mock.NewMockBlobSlicer(ctrl) - baseBlobAccess.EXPECT().GetFromComposite( - ctx, - digest.MustNewDigest("goodbye/world", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("goodbye/world", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("ell"))) - - data, err := blobAccess.GetFromComposite( - ctx, - digest.MustNewDigest("hello/world", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5), - digest.MustNewDigest("hello/world", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 3), - blobSlicer, - ).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("ell"), data) - }) -} - func TestDemultiplexingBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/empty_blob_injecting_blob_access.go b/pkg/blobstore/empty_blob_injecting_blob_access.go index 33eb8caa7..f6d27f05d 100644 --- a/pkg/blobstore/empty_blob_injecting_blob_access.go +++ b/pkg/blobstore/empty_blob_injecting_blob_access.go @@ -4,7 +4,6 @@ import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -45,13 +44,6 @@ func (ba *emptyBlobInjectingBlobAccess) Get(ctx context.Context, digest digest.D return ba.BlobAccess.Get(ctx, digest) } -func (ba *emptyBlobInjectingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - if childDigest.GetSizeBytes() == 0 { - return buffer.NewCASBufferFromByteSlice(childDigest, nil, buffer.UserProvided) - } - return ba.BlobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer) -} - func (ba *emptyBlobInjectingBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { if digest.GetSizeBytes() == 0 { _, err := b.ToByteSlice(0) diff --git a/pkg/blobstore/empty_blob_injecting_blob_access_test.go b/pkg/blobstore/empty_blob_injecting_blob_access_test.go index 518f99a27..83fb50dd2 100644 --- a/pkg/blobstore/empty_blob_injecting_blob_access_test.go +++ b/pkg/blobstore/empty_blob_injecting_blob_access_test.go @@ -63,53 +63,6 @@ func TestEmptyBlobInjectingBlobAccessGet(t *testing.T) { }) } -func TestEmptyBlobInjectingBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - baseBlobAccess := mock.NewMockBlobAccess(ctrl) - blobAccess := blobstore.NewEmptyBlobInjectingBlobAccess(baseBlobAccess) - parentDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_SHA256, "d0607e3f454f88e2925ef995f549503b7f67541dc7ac0dcf4662265aed7a749f", 1000) - slicer := mock.NewMockBlobSlicer(ctrl) - - t.Run("NonEmptySuccess", func(t *testing.T) { - // Requests for non-empty blobs should be forwarded. - childDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "7fc56270e7a70fa81a5935b72eacbe29", 1) - baseBlobAccess.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewValidatedBufferFromByteSlice([]byte("A")), - ) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(1) - require.NoError(t, err) - require.Equal(t, []byte("A"), data) - }) - - t.Run("NonEmptyFailure", func(t *testing.T) { - // Errors from the backend should be propagated. - childDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "7fc56270e7a70fa81a5935b72eacbe29", 1) - baseBlobAccess.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewBufferFromError( - status.Error(codes.Internal, "Server on fire"), - ), - ) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(1) - testutil.RequireEqualStatus(t, err, status.Error(codes.Internal, "Server on fire")) - }) - - t.Run("EmptySuccess", func(t *testing.T) { - // Requests for the empty blob should be processed directly. - data, err := blobAccess.GetFromComposite(ctx, parentDigest, digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "d41d8cd98f00b204e9800998ecf8427e", 0), slicer).ToByteSlice(0) - require.NoError(t, err) - require.Empty(t, data) - }) - - t.Run("EmptyInvalid", func(t *testing.T) { - // Validation should still be performed on empty blobs. - _, err := blobAccess.GetFromComposite(ctx, parentDigest, digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "3e25960a79dbc69b674cd4ec67a72c62", 0), slicer).ToByteSlice(0) - testutil.RequireEqualStatus(t, err, status.Error(codes.InvalidArgument, "Buffer has checksum d41d8cd98f00b204e9800998ecf8427e, while 3e25960a79dbc69b674cd4ec67a72c62 was expected")) - }) -} - func TestEmptyBlobInjectingBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/grpcclients/BUILD.bazel b/pkg/blobstore/grpcclients/BUILD.bazel index bb518ecf6..81d2e8c44 100644 --- a/pkg/blobstore/grpcclients/BUILD.bazel +++ b/pkg/blobstore/grpcclients/BUILD.bazel @@ -4,8 +4,8 @@ go_library( name = "grpcclients", srcs = [ "ac_blob_access.go", - "cas_blob_access.go", "cls_blob_access.go", + "cs_blob_access.go", "fsac_blob_access.go", "icas_blob_access.go", "iscc_blob_access.go", @@ -15,8 +15,10 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/chunklist", "//pkg/blobstore/slicing", "//pkg/digest", + "//pkg/proto/blobstore/chunklist", "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", @@ -34,7 +36,7 @@ go_library( go_test( name = "grpcclients_test", - srcs = ["cas_blob_access_test.go"], + srcs = ["cs_blob_access_test.go"], deps = [ ":grpcclients", "//internal/mock", diff --git a/pkg/blobstore/grpcclients/cls_blob_access.go b/pkg/blobstore/grpcclients/cls_blob_access.go index a27e5ea84..bdc84b212 100644 --- a/pkg/blobstore/grpcclients/cls_blob_access.go +++ b/pkg/blobstore/grpcclients/cls_blob_access.go @@ -6,12 +6,12 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" + chunklist_pb "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist" "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) type clsBlobAccess struct { @@ -21,7 +21,7 @@ type clsBlobAccess struct { } // NewCLSBlobAccess creates a BlobAccess that relays any requests to a -// gRPC server that implements the split and splice api calls of a +// gRPC server that implements the split and splice API calls of a // remoteexecution.ContentAddressableStorage service. func NewCLSBlobAccess(client grpc.ClientConnInterface, maximumMessageSizeBytes int) blobstore.BlobAccess { return &clsBlobAccess{ @@ -31,17 +31,32 @@ func NewCLSBlobAccess(client grpc.ClientConnInterface, maximumMessageSizeBytes i } } -func (ba *clsBlobAccess) Get(ctx context.Context, digest digest.Digest) buffer.Buffer { - digestFunction := digest.GetDigestFunction() +func (ba *clsBlobAccess) Get(ctx context.Context, blobDigest digest.Digest) buffer.Buffer { + digestFunction := blobDigest.GetDigestFunction() splitBlobsResponse, err := ba.contentAddressableStorageClient.SplitBlob(ctx, &remoteexecution.SplitBlobRequest{ InstanceName: digestFunction.GetInstanceName().String(), - BlobDigest: digest.GetProto(), + BlobDigest: blobDigest.GetProto(), DigestFunction: digestFunction.GetEnumValue(), }) if err != nil { return buffer.NewBufferFromError(err) } - return buffer.NewProtoBufferFromProto(splitBlobsResponse, buffer.BackendProvided(buffer.Irreparable(digest))) + + // Convert wire format to storage format. + chunkDigests := make([]digest.Digest, 0, len(splitBlobsResponse.ChunkDigests)) + for _, proto := range splitBlobsResponse.ChunkDigests { + d, err := digestFunction.NewDigestFromProto(proto) + if err != nil { + return buffer.NewBufferFromError(err) + } + chunkDigests = append(chunkDigests, d) + } + return buffer.NewProtoBufferFromProto( + &chunklist_pb.ChunkList{ + Data: chunklist.EncodeToBinary(chunkDigests), + }, + buffer.BackendProvided(buffer.Irreparable(blobDigest)), + ) } func (ba *clsBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { @@ -49,39 +64,46 @@ func (ba *clsBlobAccess) GetFromComposite(ctx context.Context, parentDigest, chi return b } -func (ba *clsBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - splitBlobResponseProto, err := b.ToProto(&remoteexecution.SplitBlobResponse{}, ba.maximumMessageSizeBytes) +func (ba *clsBlobAccess) Put(ctx context.Context, blobDigest digest.Digest, b buffer.Buffer) error { + msg, err := b.ToProto(&chunklist_pb.ChunkList{}, ba.maximumMessageSizeBytes) + if err != nil { + return err + } + + // Convert storage format to wire format. + chunkList, err := chunklist.NewChunkListFromProto(msg.(*chunklist_pb.ChunkList), blobDigest.GetInstanceName()) if err != nil { return err } - splitBlobResponse := splitBlobResponseProto.(*remoteexecution.SplitBlobResponse) - digestFunction := digest.GetDigestFunction() + chunkDigests := make([]*remoteexecution.Digest, 0, len(chunkList)) + for _, entry := range chunkList { + chunkDigests = append(chunkDigests, entry.Digest.GetProto()) + } + + digestFunction := blobDigest.GetDigestFunction() _, err = ba.contentAddressableStorageClient.SpliceBlob(ctx, &remoteexecution.SpliceBlobRequest{ InstanceName: digestFunction.GetInstanceName().String(), DigestFunction: digestFunction.GetEnumValue(), - ChunkDigests: splitBlobResponse.GetChunkDigests(), - ChunkingFunction: splitBlobResponse.GetChunkingFunction(), - BlobDigest: digest.GetProto(), + ChunkDigests: chunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + BlobDigest: blobDigest.GetProto(), }) return err } func (ba *clsBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { - missing := digest.NewSetBuilder(digests.Length()) - for _, d := range digests.Items() { - _, err := ba.contentAddressableStorageClient.SplitBlob(ctx, &remoteexecution.SplitBlobRequest{ - InstanceName: d.GetInstanceName().String(), - BlobDigest: d.GetProto(), - DigestFunction: d.GetDigestFunction().GetEnumValue(), - ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, - }) - if status.Code(err) == codes.NotFound { - missing.Add(d) - } else if err != nil { - return digest.EmptySet, err - } - } - return missing.Build(), nil + // Semantically an REv2 server which supports the Split and Splice + // apis should be able to answer the SplitBlob call for any blob + // which it has in its storage. Thus we can safely say that we are + // able to Get a chunk list from an upstream server as long as it + // has the blob. We can therefore reuse the existing + // FindMissingBlobs API for this purpose. + // + // In Buildbarn we implement this on the server side by segregating + // FMB requests for blobs larger than the maximum chunk size to the + // Chunk List Storage (CLS) and to the Chunk Storage (CS) for other + // blobs. + return findMissingBlobsInternal(ctx, digests, ba.contentAddressableStorageClient) } func (ba *clsBlobAccess) GetCapabilities(ctx context.Context, instanceName digest.InstanceName) (*remoteexecution.ServerCapabilities, error) { @@ -95,7 +117,6 @@ func (ba *clsBlobAccess) GetCapabilities(ctx context.Context, instanceName diges CacheCapabilities: &remoteexecution.CacheCapabilities{ SplitBlobSupport: cacheCapabilities.SplitBlobSupport, SpliceBlobSupport: cacheCapabilities.SpliceBlobSupport, - RepMaxCdcParams: cacheCapabilities.RepMaxCdcParams, }, }, nil } diff --git a/pkg/blobstore/grpcclients/cas_blob_access.go b/pkg/blobstore/grpcclients/cs_blob_access.go similarity index 95% rename from pkg/blobstore/grpcclients/cas_blob_access.go rename to pkg/blobstore/grpcclients/cs_blob_access.go index 8c7b3abca..3e853a729 100644 --- a/pkg/blobstore/grpcclients/cas_blob_access.go +++ b/pkg/blobstore/grpcclients/cs_blob_access.go @@ -33,7 +33,7 @@ type casBlobAccess struct { zstdPool bb_zstd.Pool } -// NewCASBlobAccess creates a BlobAccess handle that relays any requests +// NewCSBlobAccess creates a BlobAccess handle that relays any requests // to a gRPC service that implements the bytestream.ByteStream and // remoteexecution.ContentAddressableStorage services. Those are the // services that Bazel uses to access blobs stored in the Content @@ -41,7 +41,7 @@ type casBlobAccess struct { // // If zstdPool is non-nil, the client will use ZSTD compression for // ByteStream operations if the server supports it. -func NewCASBlobAccess(client grpc.ClientConnInterface, uuidGenerator util.UUIDGenerator, readChunkSize int, zstdPool bb_zstd.Pool) blobstore.BlobAccess { +func NewCSBlobAccess(client grpc.ClientConnInterface, uuidGenerator util.UUIDGenerator, readChunkSize int, zstdPool bb_zstd.Pool) blobstore.BlobAccess { return &casBlobAccess{ byteStreamClient: bytestream.NewByteStreamClient(client), contentAddressableStorageClient: remoteexecution.NewContentAddressableStorageClient(client), @@ -395,7 +395,10 @@ func (ba *casBlobAccess) GetCapabilities(ctx context.Context, instanceName diges // issue batch operations. return &remoteexecution.ServerCapabilities{ CacheCapabilities: &remoteexecution.CacheCapabilities{ - DigestFunctions: digest.RemoveUnsupportedDigestFunctions(cacheCapabilities.DigestFunctions), + DigestFunctions: digest.RemoveUnsupportedDigestFunctions(cacheCapabilities.DigestFunctions), + SplitBlobSupport: cacheCapabilities.SplitBlobSupport, + SpliceBlobSupport: cacheCapabilities.SpliceBlobSupport, + RepMaxCdcParams: cacheCapabilities.RepMaxCdcParams, }, DeprecatedApiVersion: serverCapabilities.DeprecatedApiVersion, LowApiVersion: serverCapabilities.LowApiVersion, diff --git a/pkg/blobstore/grpcclients/cas_blob_access_test.go b/pkg/blobstore/grpcclients/cs_blob_access_test.go similarity index 95% rename from pkg/blobstore/grpcclients/cas_blob_access_test.go rename to pkg/blobstore/grpcclients/cs_blob_access_test.go index 4465d1ded..de60aa128 100644 --- a/pkg/blobstore/grpcclients/cas_blob_access_test.go +++ b/pkg/blobstore/grpcclients/cs_blob_access_test.go @@ -37,12 +37,12 @@ func newTestZstdPool(maxEncoders, maxDecoders int64) bb_zstd.Pool { ) } -func TestCASBlobAccessPut(t *testing.T) { +func TestCSBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, nil) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, nil) blobDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) uuid := uuid.Must(uuid.Parse("7d659e5f-0e4b-48f0-ad9f-3489db6e103b")) @@ -180,7 +180,7 @@ func TestCASBlobAccessGet(t *testing.T) { client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, nil) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, nil) t.Run("Success", func(t *testing.T) { blobDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) @@ -276,12 +276,12 @@ func TestCASBlobAccessGet(t *testing.T) { }) } -func TestCASBlobAccessGetCapabilities(t *testing.T) { +func TestCSBlobAccessGetCapabilities(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, nil) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, nil) t.Run("BackendFailure", func(t *testing.T) { client.EXPECT().Invoke( @@ -377,12 +377,12 @@ func TestCASBlobAccessGetCapabilities(t *testing.T) { }) } -func TestCASBlobAccessPutWithCompression(t *testing.T) { +func TestCSBlobAccessPutWithCompression(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, newTestZstdPool(16, 16)) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, newTestZstdPool(16, 16)) expectGetCapabilitiesWithZSTD(client) @@ -455,12 +455,12 @@ func expectGetCapabilitiesWithZSTD(client *mock.MockClientConnInterface) { }).AnyTimes() } -func TestCASBlobAccessGetWithCompression(t *testing.T) { +func TestCSBlobAccessGetWithCompression(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 100, newTestZstdPool(16, 16)) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 100, newTestZstdPool(16, 16)) expectGetCapabilitiesWithZSTD(client) @@ -500,14 +500,14 @@ func TestCASBlobAccessGetWithCompression(t *testing.T) { }) } -func TestCASBlobAccessPutPoolExhaustion(t *testing.T) { +func TestCSBlobAccessPutPoolExhaustion(t *testing.T) { // Create a pool with only 1 concurrent encoder to test backpressure. pool := bb_zstd.NewBoundedPool(1, 1, nil, nil) ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, pool) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, pool) expectGetCapabilitiesWithZSTD(client) @@ -579,7 +579,7 @@ func TestCASBlobAccessPutPoolExhaustion(t *testing.T) { wg.Wait() } -func TestCASBlobAccessPutPoolReleasesEncoder(t *testing.T) { +func TestCSBlobAccessPutPoolReleasesEncoder(t *testing.T) { // Pool with 1 encoder: if encoder isn't released after the first Put, // the second Put would deadlock. pool := bb_zstd.NewBoundedPool(1, 1, nil, nil) @@ -587,7 +587,7 @@ func TestCASBlobAccessPutPoolReleasesEncoder(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 10, pool) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 10, pool) expectGetCapabilitiesWithZSTD(client) @@ -619,7 +619,7 @@ func TestCASBlobAccessPutPoolReleasesEncoder(t *testing.T) { } } -func TestCASBlobAccessGetPoolReleasesDecoder(t *testing.T) { +func TestCSBlobAccessGetPoolReleasesDecoder(t *testing.T) { // Pool with 1 decoder: if decoder isn't released after the first Get, // the second Get would deadlock. pool := bb_zstd.NewBoundedPool(1, 1, nil, nil) @@ -627,7 +627,7 @@ func TestCASBlobAccessGetPoolReleasesDecoder(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) client := mock.NewMockClientConnInterface(ctrl) uuidGenerator := mock.NewMockUUIDGenerator(ctrl) - blobAccess := grpcclients.NewCASBlobAccess(client, uuidGenerator.Call, 100, pool) + blobAccess := grpcclients.NewCSBlobAccess(client, uuidGenerator.Call, 100, pool) expectGetCapabilitiesWithZSTD(client) diff --git a/pkg/blobstore/grpcservers/BUILD.bazel b/pkg/blobstore/grpcservers/BUILD.bazel index 0069854c1..d2ea7ebf7 100644 --- a/pkg/blobstore/grpcservers/BUILD.bazel +++ b/pkg/blobstore/grpcservers/BUILD.bazel @@ -15,6 +15,8 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/digest", "//pkg/proto/fsac", "//pkg/proto/icas", @@ -44,6 +46,8 @@ go_test( ":grpcservers", "//internal/mock", "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", "//pkg/digest", "//pkg/proto/fsac", "//pkg/proto/icas", diff --git a/pkg/blobstore/grpcservers/byte_stream_server.go b/pkg/blobstore/grpcservers/byte_stream_server.go index 0f9cb18a2..ffda7b44b 100644 --- a/pkg/blobstore/grpcservers/byte_stream_server.go +++ b/pkg/blobstore/grpcservers/byte_stream_server.go @@ -6,8 +6,7 @@ import ( "io" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" - "github.com/buildbarn/bb-storage/pkg/blobstore" - "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" bb_zstd "github.com/buildbarn/bb-storage/pkg/zstd" @@ -18,19 +17,17 @@ import ( ) type byteStreamServer struct { - blobAccess blobstore.BlobAccess - readChunkSize int - zstdPool bb_zstd.Pool + contentAddressableStorage cdc.ContentAddressableStorage + zstdPool bb_zstd.Pool } // NewByteStreamServer creates a GRPC service for reading blobs from and // writing blobs to a BlobAccess. It is used by Bazel to access the // Content Addressable Storage (CAS). -func NewByteStreamServer(blobAccess blobstore.BlobAccess, readChunkSize int, zstdPool bb_zstd.Pool) bytestream.ByteStreamServer { +func NewByteStreamServer(contentAddressableStorage cdc.ContentAddressableStorage, zstdPool bb_zstd.Pool) bytestream.ByteStreamServer { return &byteStreamServer{ - blobAccess: blobAccess, - readChunkSize: readChunkSize, - zstdPool: zstdPool, + contentAddressableStorage: contentAddressableStorage, + zstdPool: zstdPool, } } @@ -38,41 +35,33 @@ func (s *byteStreamServer) Read(in *bytestream.ReadRequest, out bytestream.ByteS if in.ReadLimit != 0 { return status.Error(codes.Unimplemented, "This service does not support downloading partial files") } - digest, compressor, err := digest.NewDigestFromByteStreamReadPath(in.ResourceName) + d, compressor, err := digest.NewDigestFromByteStreamReadPath(in.ResourceName) if err != nil { return err } + if in.ReadOffset < 0 { + return status.Errorf(codes.InvalidArgument, "Negative read offset: %d", in.ReadOffset) + } + if in.ReadOffset > d.GetSizeBytes() { + return status.Errorf(codes.InvalidArgument, "Buffer is %d bytes in size, while a read at offset %d was requested", d.GetSizeBytes(), in.ReadOffset) + } + ctx := out.Context() + var w io.Writer = readStreamWriter{out: out} switch compressor { case remoteexecution.Compressor_IDENTITY: - r := s.blobAccess.Get(ctx, digest).ToChunkReader(in.ReadOffset, s.readChunkSize) - defer r.Close() - - for { - readBuf, readErr := r.Read() - if readErr == io.EOF { - return nil - } - if readErr != nil { - return readErr - } - if writeErr := out.Send(&bytestream.ReadResponse{Data: readBuf}); writeErr != nil { - return writeErr - } - } - case remoteexecution.Compressor_ZSTD: - b := s.blobAccess.Get(ctx, digest) - encoder, err := s.zstdPool.NewEncoder(ctx, &readStreamWriter{out: out}) + encoder, err := s.zstdPool.NewEncoder(ctx, w) if err != nil { - b.Discard() return status.Errorf(codes.ResourceExhausted, "Failed to acquire ZSTD encoder: %v", err) } defer encoder.Close() - return b.IntoWriter(encoder) + w = encoder + return cdc.IntoWriter(ctx, s.contentAddressableStorage, d, in.ReadOffset, encoder) default: return status.Errorf(codes.Unimplemented, "This service does not support downloading compression type: %s", compressor) } + return cdc.IntoWriter(ctx, s.contentAddressableStorage, d, in.ReadOffset, w) } // readStreamWriter adapts the ByteStream_ReadServer to an io.Writer. @@ -80,56 +69,13 @@ type readStreamWriter struct { out bytestream.ByteStream_ReadServer } -func (w *readStreamWriter) Write(p []byte) (int, error) { +func (w readStreamWriter) Write(p []byte) (int, error) { if err := w.out.Send(&bytestream.ReadResponse{Data: p}); err != nil { return 0, err } return len(p), nil } -type byteStreamWriteServerChunkReader struct { - stream bytestream.ByteStream_WriteServer - writeOffset int64 - data []byte - finishedWrite bool -} - -func (r *byteStreamWriteServerChunkReader) setRequest(request *bytestream.WriteRequest) error { - if r.finishedWrite { - return status.Error(codes.InvalidArgument, "Client closed stream twice") - } - if request.WriteOffset != r.writeOffset { - return status.Errorf(codes.InvalidArgument, "Attempted to write at offset %d, while %d was expected", request.WriteOffset, r.writeOffset) - } - - r.writeOffset += int64(len(request.Data)) - r.data = request.Data - r.finishedWrite = request.FinishWrite - return nil -} - -func (r *byteStreamWriteServerChunkReader) Read() ([]byte, error) { - // Read next chunk if no data is present. - if len(r.data) == 0 { - request, err := r.stream.Recv() - if err != nil { - if err == io.EOF && !r.finishedWrite { - return nil, status.Error(codes.InvalidArgument, "Client closed stream without finishing write") - } - return nil, err - } - if err := r.setRequest(request); err != nil { - return nil, err - } - } - - data := r.data - r.data = nil - return data, nil -} - -func (byteStreamWriteServerChunkReader) Close() {} - func (s *byteStreamServer) Write(stream bytestream.ByteStream_WriteServer) error { request, err := stream.Recv() if err != nil { @@ -138,45 +84,51 @@ func (s *byteStreamServer) Write(stream bytestream.ByteStream_WriteServer) error } return err } - digest, compressor, err := digest.NewDigestFromByteStreamWritePath(request.ResourceName) + + d, compressor, err := digest.NewDigestFromByteStreamWritePath(request.ResourceName) if err != nil { return err } + + ctx := stream.Context() + rawReader := &writeStreamReader{ + stream: stream, + nextOffset: int64(len(request.Data)), + finished: request.FinishWrite, + pendingData: request.Data, + } + + var r io.Reader switch compressor { case remoteexecution.Compressor_IDENTITY: - return s.writeIdentity(stream, request, digest) + r = rawReader case remoteexecution.Compressor_ZSTD: - return s.writeZstd(stream, request, digest) + zr, err := bb_zstd.NewReadCloser(ctx, s.zstdPool, rawReader) + if err != nil { + return util.StatusWrap(err, "Failed to acquire ZSTD decoder") + } + defer zr.Close() + r = zr default: return status.Errorf(codes.Unimplemented, "This service does not support uploading compression type: %s", compressor) } -} -func (s *byteStreamServer) writeIdentity(stream bytestream.ByteStream_WriteServer, request *bytestream.WriteRequest, digest digest.Digest) error { - r := &byteStreamWriteServerChunkReader{stream: stream} - if err := r.setRequest(request); err != nil { - return err - } - if err := s.blobAccess.Put( - stream.Context(), - digest, - buffer.NewCASBufferFromChunkReader(digest, r, buffer.UserProvided), - ); err != nil { + if err := cdc.PutReader(ctx, s.contentAddressableStorage, d, r); err != nil { return err } return stream.SendAndClose(&bytestream.WriteResponse{ - CommittedSize: digest.GetSizeBytes(), + CommittedSize: rawReader.nextOffset, }) } -type zstdWriteStreamReader struct { +type writeStreamReader struct { stream bytestream.ByteStream_WriteServer nextOffset int64 finished bool pendingData []byte } -func (r *zstdWriteStreamReader) Read(p []byte) (n int, err error) { +func (r *writeStreamReader) Read(p []byte) (n int, err error) { if len(r.pendingData) > 0 { n = copy(p, r.pendingData) r.pendingData = r.pendingData[n:] @@ -184,7 +136,18 @@ func (r *zstdWriteStreamReader) Read(p []byte) (n int, err error) { } if r.finished { - return 0, io.EOF + // The client indicated the write was finished. Check for trailing garbage. + req, err := r.stream.Recv() + if err == nil { + if req.FinishWrite { + return 0, status.Error(codes.InvalidArgument, "Client closed stream twice") + } + return 0, status.Error(codes.InvalidArgument, "Client sent extra data after finishing write") + } + if errors.Is(err, io.EOF) { + return 0, io.EOF + } + return 0, err } req, err := r.stream.Recv() @@ -207,36 +170,10 @@ func (r *zstdWriteStreamReader) Read(p []byte) (n int, err error) { return n, nil } -func (zstdWriteStreamReader) Close() error { +func (writeStreamReader) Close() error { return nil } -func (s *byteStreamServer) writeZstd(stream bytestream.ByteStream_WriteServer, request *bytestream.WriteRequest, digest digest.Digest) error { - ctx := stream.Context() - streamReader := &zstdWriteStreamReader{ - stream: stream, - nextOffset: int64(len(request.Data)), - finished: request.FinishWrite, - pendingData: request.Data, - } - - zstdReader, err := bb_zstd.NewReadCloser(ctx, s.zstdPool, streamReader) - if err != nil { - return util.StatusWrap(err, "Failed to acquire ZSTD decoder") - } - - if err := s.blobAccess.Put( - ctx, - digest, - buffer.NewCASBufferFromReader(digest, zstdReader, buffer.UserProvided), - ); err != nil { - return err - } - return stream.SendAndClose(&bytestream.WriteResponse{ - CommittedSize: streamReader.nextOffset, - }) -} - func (byteStreamServer) QueryWriteStatus(ctx context.Context, in *bytestream.QueryWriteStatusRequest) (*bytestream.QueryWriteStatusResponse, error) { return nil, status.Error(codes.Unimplemented, "This service does not support querying write status") } diff --git a/pkg/blobstore/grpcservers/byte_stream_server_test.go b/pkg/blobstore/grpcservers/byte_stream_server_test.go index 542eb40f3..04a09ded6 100644 --- a/pkg/blobstore/grpcservers/byte_stream_server_test.go +++ b/pkg/blobstore/grpcservers/byte_stream_server_test.go @@ -10,7 +10,8 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/internal/mock" - "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/testutil" @@ -33,8 +34,8 @@ func TestByteStreamServer(t *testing.T) { // Create an RPC server/client pair. l := bufconn.Listen(1 << 20) server := grpc.NewServer() - blobAccess := mock.NewMockBlobAccess(ctrl) - bytestream.RegisterByteStreamServer(server, grpcservers.NewByteStreamServer(blobAccess, 10, bb_zstd.NewUnboundedPool( + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) + bytestream.RegisterByteStreamServer(server, grpcservers.NewByteStreamServer(contentAddressableStorage, bb_zstd.NewUnboundedPool( []zstd.EOption{zstd.WithEncoderConcurrency(1)}, []zstd.DOption{zstd.WithDecoderConcurrency(1)}, ))) @@ -89,12 +90,14 @@ func TestByteStreamServer(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Invalid digest size: -42 bytes"), err) }) + singleChunkParameters := cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20} + twoChunksParameters := cdc.Parameters{MinChunkSizeBytes: 10, HorizonSizeBytes: 20} + t.Run("ReadSuccessEmptyInstance", func(t *testing.T) { // Attempt to fetch the small blob without an instance name. - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 5), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) + digest1 := digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "09f7e02f1290be211da707a266f153b3", 5) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return([]byte("Hello"), nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "blobs/09f7e02f1290be211da707a266f153b3/5", @@ -109,10 +112,34 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadSuccessNonEmptyInstance", func(t *testing.T) { // Attempt to fetch the large blob with an instance name. - blobAccess.EXPECT().Get( + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "3538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "0538d378083b9afa5ffad767f7269509", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "1538d378083b9afa5ffad767f7269509", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "2538d378083b9afa5ffad767f7269509", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( + gomock.Any(), + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest1, + ).Return([]byte("This is a "), nil) + contentAddressableStorage.EXPECT().FetchChunk( gomock.Any(), - digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "3538d378083b9afa5ffad767f7269509", 22), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("This is a long message"))) + chunkDigest2, + ).Return([]byte("long messa"), nil) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest3, + ).Return([]byte("ge"), nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "debian8/blobs/3538d378083b9afa5ffad767f7269509/22", @@ -134,10 +161,9 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadZSTDCompression", func(t *testing.T) { // Test reading with ZSTD compression. originalData := []byte("This is a test message that should be compressed with ZSTD") - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "8b2c3f8a9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f61", 58), - ).Return(buffer.NewValidatedBufferFromByteSlice(originalData)) + digest1 := digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "8b2c3f8a9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f61", 58) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return(originalData, nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "compressed-blobs/zstd/8b2c3f8a9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f61/58", @@ -162,7 +188,6 @@ func TestByteStreamServer(t *testing.T) { require.NoError(t, err) require.Equal(t, originalData, decompressedData) }) - t.Run("ReadZSTDCompressionLargeData", func(t *testing.T) { // Test reading large data with ZSTD compression. originalData := make([]byte, 100000) @@ -170,10 +195,9 @@ func TestByteStreamServer(t *testing.T) { originalData[i] = byte(i % 256) } - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", 100000), - ).Return(buffer.NewValidatedBufferFromByteSlice(originalData)) + digest1 := digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", 100000) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return(originalData, nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "compressed-blobs/zstd/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2/100000", @@ -214,16 +238,12 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadNegativeReadOffset", func(t *testing.T) { // Attempt to fetch a blob with a negative offset. - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "6fc422233a40a75a1f028e11c3cd1140", 7), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Goodbye"))) - req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "ubuntu1804/blobs/6fc422233a40a75a1f028e11c3cd1140/7", ReadOffset: -4, }) require.NoError(t, err) + _, err = req.Recv() testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Negative read offset: -4"), err) }) @@ -231,11 +251,6 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadOffsetBeyondEnd", func(t *testing.T) { // Attempt to fetch a blob with a offset beyond the size // of the blob. - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "ad3c8ac9eef32188da352082244b3598", 13), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("short message"))) - req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "ubuntu1804/blobs/ad3c8ac9eef32188da352082244b3598/13", ReadOffset: 100, @@ -247,10 +262,9 @@ func TestByteStreamServer(t *testing.T) { t.Run("ReadSuccessWithOffset", func(t *testing.T) { // Attempt to fetch a lblob with an instance name and offset. - blobAccess.EXPECT().Get( - gomock.Any(), - digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "da39a3ee5e6b4b0d3255bfef95601890", 19), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("This offset message"))) + digest1 := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_MD5, "da39a3ee5e6b4b0d3255bfef95601890", 19) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("ubuntu1804")).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return([]byte("This offset message"), nil) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "ubuntu1804/blobs/da39a3ee5e6b4b0d3255bfef95601890/19", @@ -259,20 +273,185 @@ func TestByteStreamServer(t *testing.T) { require.NoError(t, err) readResponse, err := req.Recv() require.NoError(t, err) - require.Equal(t, []byte(" offset me"), readResponse.Data) + require.Equal(t, []byte(" offset message"), readResponse.Data) + _, err = req.Recv() + require.Equal(t, io.EOF, err) + }) + + t.Run("ReadSuccessChunkedWithOffset", func(t *testing.T) { + // Fetch the tail of a multi-chunk blob starting in the + // middle of the second chunk. + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "4538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "4538d378083b9afa5ffad767f7269508", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "4538d378083b9afa5ffad767f7269507", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "4538d378083b9afa5ffad767f7269506", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( + gomock.Any(), + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + // The first chunk lies completely before the read offset + // and must not be fetched. + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest2, + ).Return([]byte("long messa"), nil) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest3, + ).Return([]byte("ge"), nil) + + req, err := client.Read(ctx, &bytestream.ReadRequest{ + ResourceName: "debian8/blobs/4538d378083b9afa5ffad767f7269509/22", + ReadOffset: 14, + }) + require.NoError(t, err) + readResponse, err := req.Recv() + require.NoError(t, err) + require.Equal(t, []byte(" messa"), readResponse.Data) readResponse, err = req.Recv() require.NoError(t, err) - require.Equal(t, []byte("ssage"), readResponse.Data) + require.Equal(t, []byte("ge"), readResponse.Data) _, err = req.Recv() require.Equal(t, io.EOF, err) }) - t.Run("ReadNonexistentBlob", func(t *testing.T) { - // Attempt to fetch a nonexistent blob. - blobAccess.EXPECT().Get( + t.Run("ReadSuccessChunkedWithOffsetAtChunkBoundary", func(t *testing.T) { + // A read offset that coincides with a chunk boundary + // should start the stream at the beginning of that chunk. + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "5538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "5538d378083b9afa5ffad767f7269508", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "5538d378083b9afa5ffad767f7269507", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "5538d378083b9afa5ffad767f7269506", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( gomock.Any(), - digest.MustNewDigest("fedora28", remoteexecution.DigestFunction_MD5, "09f34d28e9c8bb445ec996388968a9e8", 7), - ).Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found"))) + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest2, + ).Return([]byte("long messa"), nil) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest3, + ).Return([]byte("ge"), nil) + + req, err := client.Read(ctx, &bytestream.ReadRequest{ + ResourceName: "debian8/blobs/5538d378083b9afa5ffad767f7269509/22", + ReadOffset: 10, + }) + require.NoError(t, err) + readResponse, err := req.Recv() + require.NoError(t, err) + require.Equal(t, []byte("long messa"), readResponse.Data) + readResponse, err = req.Recv() + require.NoError(t, err) + require.Equal(t, []byte("ge"), readResponse.Data) + _, err = req.Recv() + require.Equal(t, io.EOF, err) + }) + + t.Run("ReadSuccessChunkedWithOffsetAtEnd", func(t *testing.T) { + // A read offset equal to the blob size is valid and + // should yield an empty stream. + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "6538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "6538d378083b9afa5ffad767f7269508", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "6538d378083b9afa5ffad767f7269507", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "6538d378083b9afa5ffad767f7269506", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( + gomock.Any(), + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + + req, err := client.Read(ctx, &bytestream.ReadRequest{ + ResourceName: "debian8/blobs/6538d378083b9afa5ffad767f7269509/22", + ReadOffset: 22, + }) + require.NoError(t, err) + _, err = req.Recv() + require.Equal(t, io.EOF, err) + }) + + t.Run("ReadChunkedZSTDCompressionWithOffset", func(t *testing.T) { + // Test reading with ZSTD compression where the read + // offset references the uncompressed stream. + digest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "7538d378083b9afa5ffad767f7269509", 22) + chunkDigest1 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "7538d378083b9afa5ffad767f7269508", 10) + chunkDigest2 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "7538d378083b9afa5ffad767f7269507", 10) + chunkDigest3 := digest.MustNewDigest("debian8", remoteexecution.DigestFunction_MD5, "7538d378083b9afa5ffad767f7269506", 2) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("debian8")).Return(twoChunksParameters, nil) + contentAddressableStorage.EXPECT().GetManifest( + gomock.Any(), + digest1, + ).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunkDigest1}, + {Offset: 10, Digest: chunkDigest2}, + {Offset: 20, Digest: chunkDigest3}, + }, + nil, + ) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest2, + ).Return([]byte("long messa"), nil) + contentAddressableStorage.EXPECT().FetchChunk( + gomock.Any(), + chunkDigest3, + ).Return([]byte("ge"), nil) + + req, err := client.Read(ctx, &bytestream.ReadRequest{ + ResourceName: "debian8/compressed-blobs/zstd/7538d378083b9afa5ffad767f7269509/22", + ReadOffset: 14, + }) + require.NoError(t, err) + + var compressedData []byte + for { + response, err := req.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + compressedData = append(compressedData, response.Data...) + } + + decoder, err := zstd.NewReader(nil) + require.NoError(t, err) + defer decoder.Close() + + decompressedData, err := decoder.DecodeAll(compressedData, nil) + require.NoError(t, err) + require.Equal(t, []byte(" message"), decompressedData) + }) + + t.Run("ReadNonexistentBlob", func(t *testing.T) { + digest1 := digest.MustNewDigest("fedora28", remoteexecution.DigestFunction_MD5, "09f34d28e9c8bb445ec996388968a9e8", 7) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("fedora28")).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().FetchChunk(gomock.Any(), digest1).Return(nil, status.Error(codes.NotFound, "Blob not found")) req, err := client.Read(ctx, &bytestream.ReadRequest{ ResourceName: "///fedora28//blobs/09f34d28e9c8bb445ec996388968a9e8/////7/", @@ -304,16 +483,9 @@ func TestByteStreamServer(t *testing.T) { t.Run("WriteSuccessEmptyInstance", func(t *testing.T) { // Attempt to write a blob without an instance name. - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "581c1053f832a1c719fb6528a588ccfd", 14), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("LaputanMachine"), data) - return nil - }) + digest1 := digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "581c1053f832a1c719fb6528a588ccfd", 14) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().PutChunk(gomock.Any(), digest1, []byte("LaputanMachine")).Return(nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -345,16 +517,8 @@ func TestByteStreamServer(t *testing.T) { generator.Write(originalData) actualDigest := generator.Sum() - blobAccess.EXPECT().Put( - gomock.Any(), - actualDigest, - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, originalData, data) - return nil - }) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().PutChunk(gomock.Any(), actualDigest, originalData).Return(nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -382,16 +546,8 @@ func TestByteStreamServer(t *testing.T) { generator.Write(originalData) actualDigest := generator.Sum() - blobAccess.EXPECT().Put( - gomock.Any(), - actualDigest, - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, originalData, data) - return nil - }) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().PutChunk(gomock.Any(), actualDigest, originalData).Return(nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -431,16 +587,8 @@ func TestByteStreamServer(t *testing.T) { generator.Write(originalData) actualDigest := generator.Sum() - blobAccess.EXPECT().Put( - gomock.Any(), - actualDigest, - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(100000) - require.NoError(t, err) - require.Equal(t, originalData, data) - return nil - }) + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) + contentAddressableStorage.EXPECT().PutChunk(gomock.Any(), actualDigest, originalData).Return(nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -485,15 +633,9 @@ func TestByteStreamServer(t *testing.T) { // Test writing with invalid ZSTD data. invalidData := []byte("This is not valid ZSTD compressed data") - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_SHA256, "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5", 10), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - _, err := b.ToByteSlice(1000) - require.Error(t, err) - return err - }) + // The decompressor fails while the blob is being chunked, so + // nothing gets stored. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -520,16 +662,9 @@ func TestByteStreamServer(t *testing.T) { generator.Write(originalData) actualDigest := generator.Sum() - blobAccess.EXPECT().Put( - gomock.Any(), - actualDigest, - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, originalData, data) - return nil - }) + // An empty blob decomposes into no chunks at all, so it + // requires no storage. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -552,16 +687,10 @@ func TestByteStreamServer(t *testing.T) { }) t.Run("WriteSuccessWithoutFinish", func(t *testing.T) { - // Attempt to write without finishing properly. - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("", remoteexecution.DigestFunction_SHA1, "f10e562d8825ec2e17e0d9f58646f8084a658cfa", 6), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - _, err := b.ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Client closed stream without finishing write"), err) - return err - }) + // Attempt to write without finishing properly. The chunking of + // the blob aborts while reading the stream, so nothing gets + // stored. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), digest.EmptyInstanceName).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -570,20 +699,13 @@ func TestByteStreamServer(t *testing.T) { Data: []byte("Foo"), })) _, err = stream.CloseAndRecv() - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Client closed stream without finishing write"), err) + testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Failed to chunk write stream: Client closed stream without finishing write"), err) }) t.Run("WriteFailFinishTwice", func(t *testing.T) { - // Attempted to write while finishing twice. - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("fedora28", remoteexecution.DigestFunction_MD5, "cbd8f7984c654c25512e3d9241ae569f", 3), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - _, err := b.ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Client closed stream twice"), err) - return err - }) + // Attempted to write while finishing twice. The chunking of the + // blob aborts while reading the stream, so nothing gets stored. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("fedora28")).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -598,20 +720,14 @@ func TestByteStreamServer(t *testing.T) { FinishWrite: true, })) _, err = stream.CloseAndRecv() - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Client closed stream twice"), err) + testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Failed to chunk write stream: Client closed stream twice"), err) }) t.Run("WriteFailBadOffset", func(t *testing.T) { - // Attempted to write with a bad write offset. - blobAccess.EXPECT().Put( - gomock.Any(), - digest.MustNewDigest("windows10", remoteexecution.DigestFunction_MD5, "68e109f0f40ca72a15e05cc22786f8e6", 10), - gomock.Any(), - ).DoAndReturn(func(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { - _, err := b.ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Attempted to write at offset 4, while 5 was expected"), err) - return err - }) + // Attempted to write with a bad write offset. The chunking of + // the blob aborts while reading the stream, so nothing gets + // stored. + contentAddressableStorage.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("windows10")).Return(singleChunkParameters, nil) stream, err := client.Write(ctx) require.NoError(t, err) @@ -625,7 +741,7 @@ func TestByteStreamServer(t *testing.T) { FinishWrite: true, })) _, err = stream.CloseAndRecv() - testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Attempted to write at offset 4, while 5 was expected"), err) + testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Failed to chunk write stream: Attempted to write at offset 4, while 5 was expected"), err) }) t.Run("QueryWriteStatus", func(t *testing.T) { @@ -635,3 +751,11 @@ func TestByteStreamServer(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.Unimplemented, "This service does not support querying write status"), err) }) } + +func mustNewInstanceName(name string) digest.InstanceName { + instanceName, err := digest.NewInstanceName(name) + if err != nil { + panic(err) + } + return instanceName +} diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server.go b/pkg/blobstore/grpcservers/content_addressable_storage_server.go index bd76a5e4c..9fb747179 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server.go @@ -4,8 +4,8 @@ import ( "context" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" - "github.com/buildbarn/bb-storage/pkg/blobstore" - "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -14,17 +14,15 @@ import ( ) type contentAddressableStorageServer struct { - contentAddressableStorage blobstore.BlobAccess - chunkListStorage blobstore.BlobAccess + contentAddressableStorage cdc.ContentAddressableStorage maximumMessageSizeBytes int64 } // NewContentAddressableStorageServer creates a GRPC service for serving // the contents of a Bazel Content Addressable Storage (CAS) to Bazel. -func NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage blobstore.BlobAccess, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { +func NewContentAddressableStorageServer(contentAddressableStorage cdc.ContentAddressableStorage, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { return &contentAddressableStorageServer{ contentAddressableStorage: contentAddressableStorage, - chunkListStorage: chunkListStorage, maximumMessageSizeBytes: maximumMessageSizeBytes, } } @@ -43,53 +41,26 @@ func (s *contentAddressableStorageServer) FindMissingBlobs(ctx context.Context, } inDigests := digest.NewSetBuilder(len(in.BlobDigests)) - for _, partialDigest := range in.BlobDigests { - digest, err := digestFunction.NewDigestFromProto(partialDigest) + for _, inDigest := range in.BlobDigests { + digest, err := digestFunction.NewDigestFromProto(inDigest) if err != nil { return nil, err } inDigests.Add(digest) } - outDigests, err := s.contentAddressableStorage.FindMissing(ctx, inDigests.Build()) + + missing, err := s.contentAddressableStorage.FindMissing(ctx, inDigests.Build()) if err != nil { return nil, err } - partialDigests := make([]*remoteexecution.Digest, 0, outDigests.Length()) - for _, outDigest := range outDigests.Items() { - partialDigests = append(partialDigests, outDigest.GetProto()) - } - // Server is configured with Chunk List Storage (CLS) so we must - // verify the CLS as well. Note that in this version of bb-storage a - // missing chunk list for a blob does not imply that the blob is - // missing. It is merely required to manage the life time of chunk - // lists. In a future version of bb-storage FMB calls will go to - // either the chunk storage or the chunk list storage. - if s.chunkListStorage != nil { - capabilities, err := s.chunkListStorage.GetCapabilities(ctx, instanceName) - if err != nil { - return nil, err - } - if capabilities.CacheCapabilities.GetRepMaxCdcParams() == nil { - return nil, status.Error(codes.Internal, "This server implementation is only compatible with RepMaxCDC") - } - minChunkSize := capabilities.CacheCapabilities.RepMaxCdcParams.MinChunkSizeBytes - maxChunkSize := 2*minChunkSize - 1 - bigBlobDigests := digest.NewSetBuilder(0) - for _, partialDigest := range in.BlobDigests { - if partialDigest.GetSizeBytes() > int64(maxChunkSize) { - digest, err := digestFunction.NewDigestFromProto(partialDigest) - if err != nil { - return nil, err - } - bigBlobDigests.Add(digest) - } - } - _, _ = s.chunkListStorage.FindMissing(ctx, bigBlobDigests.Build()) + outDigests := make([]*remoteexecution.Digest, 0, missing.Length()) + for _, outDigest := range missing.Items() { + outDigests = append(outDigests, outDigest.GetProto()) } return &remoteexecution.FindMissingBlobsResponse{ - MissingBlobDigests: partialDigests, + MissingBlobDigests: outDigests, }, nil } @@ -106,6 +77,7 @@ func (s *contentAddressableStorageServer) BatchReadBlobs(ctx context.Context, in return nil, err } + // TODO: Compensate for message overhead. bytesRemaining := s.maximumMessageSizeBytes digests := make([]digest.Digest, 0, len(in.Digests)) for _, reqDigest := range in.Digests { @@ -130,10 +102,11 @@ func (s *contentAddressableStorageServer) BatchReadBlobs(ctx context.Context, in Responses: make([]*remoteexecution.BatchReadBlobsResponse_Response, 0, len(in.Digests)), } for i, reqDigest := range in.Digests { - data, err := s.contentAddressableStorage.Get( + data, err := cdc.GetBytes( ctx, + s.contentAddressableStorage, digests[i], - ).ToByteSlice(int(digests[i].GetSizeBytes())) + ) response.Responses = append(response.Responses, &remoteexecution.BatchReadBlobsResponse_Response{ Digest: reqDigest, Data: data, @@ -163,10 +136,11 @@ func (s *contentAddressableStorageServer) BatchUpdateBlobs(ctx context.Context, for _, request := range in.Requests { digest, err := digestFunction.NewDigestFromProto(request.Digest) if err == nil { - err = s.contentAddressableStorage.Put( + err = cdc.PutBytes( ctx, + s.contentAddressableStorage, digest, - buffer.NewCASBufferFromByteSlice(digest, request.Data, buffer.UserProvided), + request.Data, ) } response.Responses = append(response.Responses, @@ -183,11 +157,6 @@ func (contentAddressableStorageServer) GetTree(in *remoteexecution.GetTreeReques } func (s *contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *remoteexecution.SpliceBlobRequest) (*remoteexecution.SpliceBlobResponse, error) { - // TODO: Require that s.chunkListStorage is non-null once we require chunking. - if s.chunkListStorage == nil { - return nil, status.Error(codes.Unimplemented, "This service does not support SpliceBlob") - } - instanceName, err := digest.NewInstanceName(in.InstanceName) if err != nil { return nil, util.StatusWrapf(err, "Invalid instance name %#v", in.InstanceName) @@ -201,13 +170,18 @@ func (s *contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *re return nil, err } - splitBlobResponse := &remoteexecution.SplitBlobResponse{ - ChunkDigests: in.ChunkDigests, - ChunkingFunction: in.ChunkingFunction, + chunkList := make(chunklist.ChunkList, 0, len(in.ChunkDigests)) + offset := uint64(0) + for _, chunkDigestProto := range in.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkDigestProto) + if err != nil { + return nil, err + } + chunkList = append(chunkList, chunklist.Entry{Digest: chunkDigest, Offset: offset}) + offset += uint64(chunkDigest.GetSizeBytes()) } - b := buffer.NewProtoBufferFromProto(splitBlobResponse, buffer.UserProvided) - if err := s.chunkListStorage.Put(ctx, blobDigest, b); err != nil { + if err := s.contentAddressableStorage.PutManifest(ctx, blobDigest, chunkList); err != nil { return nil, err } @@ -217,11 +191,6 @@ func (s *contentAddressableStorageServer) SpliceBlob(ctx context.Context, in *re } func (s *contentAddressableStorageServer) SplitBlob(ctx context.Context, in *remoteexecution.SplitBlobRequest) (*remoteexecution.SplitBlobResponse, error) { - // TODO: Require that s.chunkListStorage is non-null once we require chunking. - if s.chunkListStorage == nil { - return nil, status.Error(codes.Unimplemented, "This service does not support SplitBlob") - } - instanceName, err := digest.NewInstanceName(in.InstanceName) if err != nil { return nil, util.StatusWrapf(err, "Invalid instance name %#v", in.InstanceName) @@ -234,9 +203,39 @@ func (s *contentAddressableStorageServer) SplitBlob(ctx context.Context, in *rem if err != nil { return nil, err } - splitBlobResponse, err := s.chunkListStorage.Get(ctx, blobDigest).ToProto(&remoteexecution.SplitBlobResponse{}, int(s.maximumMessageSizeBytes)) + + // Blobs stored as a single chunk have no chunk list; synthesize + // the trivial answer, provided the blob exists. + params, err := s.contentAddressableStorage.FetchCDCParameters(ctx, instanceName) if err != nil { return nil, err } - return splitBlobResponse.(*remoteexecution.SplitBlobResponse), nil + if cdc.IsSingleChunk(params, blobDigest) { + missing, err := s.contentAddressableStorage.FindMissing(ctx, blobDigest.ToSingletonSet()) + if err != nil { + return nil, util.StatusWrap(err, "Failed to check blob existence") + } + if !missing.Empty() { + return nil, status.Errorf(codes.NotFound, "Blob %s not found", blobDigest) + } + return &remoteexecution.SplitBlobResponse{ + ChunkDigests: []*remoteexecution.Digest{in.BlobDigest}, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + }, nil + } + + storedChunkList, err := s.contentAddressableStorage.GetManifest(ctx, blobDigest) + if err != nil { + return nil, util.StatusWrap(err, "Failed to decode chunk list") + } + + chunkDigests := make([]*remoteexecution.Digest, 0, len(storedChunkList)) + for _, entry := range storedChunkList { + chunkDigests = append(chunkDigests, entry.Digest.GetProto()) + } + + return &remoteexecution.SplitBlobResponse{ + ChunkDigests: chunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + }, nil } diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go b/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go index 7c5ed83b4..12878182c 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server_test.go @@ -6,7 +6,8 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/internal/mock" - "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/testutil" @@ -44,18 +45,18 @@ func TestContentAddressableStorageServerBatchReadBlobsSuccess(t *testing.T) { InstanceName: "ubuntu1804", } - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + cas := mock.NewMockContentAddressableStorage(ctrl) + singleChunkParameters := cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20} + cas.EXPECT().FetchCDCParameters(gomock.Any(), mustNewInstanceName("ubuntu1804")).Return(singleChunkParameters, nil).Times(3) a := make([]byte, 123) - buf := buffer.NewValidatedBufferFromByteSlice(a) - contentAddressableStorage.EXPECT().Get(ctx, digest1).Return(buf) + cas.EXPECT().FetchChunk(ctx, digest1).Return(a, nil) b := make([]byte, 234) - buf2 := buffer.NewValidatedBufferFromByteSlice(b) - contentAddressableStorage.EXPECT().Get(ctx, digest2).Return(buf2) - buf3 := buffer.NewBufferFromError(status.Error(codes.NotFound, "The object you requested could not be found")) - contentAddressableStorage.EXPECT().Get(ctx, digest3).Return(buf3) + cas.EXPECT().FetchChunk(ctx, digest2).Return(b, nil) + cas.EXPECT().FetchChunk(ctx, digest3).Return(nil, status.Error(codes.NotFound, "The object you requested could not be found")) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, nil, 4<<20) + maximumMessageSizeBytes := 4 << 20 + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) response, err := contentAddressableStorageServer.BatchReadBlobs(ctx, request) require.NoError(t, err) @@ -106,14 +107,64 @@ func TestContentAddressableStorageServerBatchReadBlobsFailure(t *testing.T) { InstanceName: "ubuntu1804", } - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + cas := mock.NewMockContentAddressableStorage(ctrl) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, nil, 200) + maximumMessageSizeBytes := 200 + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) _, err := contentAddressableStorageServer.BatchReadBlobs(ctx, request) testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Attempted to read a total of at least 357 bytes, while a maximum of 200 bytes is permitted"), err) } +func TestContentAddressableStorageServerBatchUpdateBlobs(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + digest1 := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", 5) + digest2 := digest.MustNewDigest("ubuntu1804", remoteexecution.DigestFunction_SHA256, "0479688f99e8cbc70291ce272876ff8e0db71a0889daf2752884b0996056b4a0", 5) + + request := &remoteexecution.BatchUpdateBlobsRequest{ + Requests: []*remoteexecution.BatchUpdateBlobsRequest_Request{ + { + Digest: digest1.GetProto(), + Data: []byte("Hello"), + }, + { + Digest: digest2.GetProto(), + Data: []byte("World"), + }, + }, + InstanceName: "ubuntu1804", + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + + cas.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("ubuntu1804"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil).Times(2) + cas.EXPECT().PutChunk(ctx, digest1, []byte("Hello")).Return(nil) + cas.EXPECT().PutChunk(ctx, digest2, []byte("World")).Return(status.Error(codes.Internal, "Hard disk has a case of the Mondays")) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, 4<<20) + + response, err := contentAddressableStorageServer.BatchUpdateBlobs(ctx, request) + require.NoError(t, err) + testutil.RequireEqualProto(t, &remoteexecution.BatchUpdateBlobsResponse{ + Responses: []*remoteexecution.BatchUpdateBlobsResponse_Response{ + { + Digest: digest1.GetProto(), + }, + { + Digest: digest2.GetProto(), + Status: &status_pb.Status{ + Code: int32(codes.Internal), + Message: "Hard disk has a case of the Mondays", + }, + }, + }, + }, response) +} + func TestContentAddressableStorageServerFindMissingBlobs(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) @@ -128,30 +179,16 @@ func TestContentAddressableStorageServerFindMissingBlobs(t *testing.T) { }, } - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) - chunkListStorage := mock.NewMockBlobAccess(ctrl) - setBuilder := digest.NewSetBuilder(2) - digestSet := setBuilder.Add(digest1).Add(digest2).Build() - - // Missing chunk lists is not an error, nor does it imply a missing - // blob at this stage. - contentAddressableStorage.EXPECT().FindMissing(ctx, digestSet).Return(digest.EmptySet, nil) - chunkListStorage.EXPECT().GetCapabilities(ctx, digest1.GetInstanceName()).Return( - &remoteexecution.ServerCapabilities{ - CacheCapabilities: &remoteexecution.CacheCapabilities{ - SplitBlobSupport: true, - SpliceBlobSupport: true, - RepMaxCdcParams: &remoteexecution.RepMaxCdcParams{ - MinChunkSizeBytes: 64, - HorizonSizeBytes: 128, - }, - }, - }, - nil, - ) - chunkListStorage.EXPECT().FindMissing(ctx, digest2.ToSingletonSet()).Return(digest2.ToSingletonSet(), nil) + cas := mock.NewMockContentAddressableStorage(ctrl) + + digests := digest.NewSetBuilder(2) + digests.Add(digest1) + digests.Add(digest2) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) + cas.EXPECT().FindMissing(ctx, digests.Build()).Return(digest.EmptySet, nil) + + maximumMessageSizeBytes := 200 + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) response, err := contentAddressableStorageServer.FindMissingBlobs(ctx, request) require.NoError(t, err) @@ -170,9 +207,6 @@ func TestContentAddressableStorageServerSplitBlob(t *testing.T) { DigestFunction: remoteexecution.DigestFunction_SHA256, } - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) - chunkListStorage := mock.NewMockBlobAccess(ctrl) - instanceName, err := digest.NewInstanceName(request.InstanceName) require.NoError(t, err) digestFunction, err := instanceName.GetDigestFunction(request.DigestFunction, len(request.BlobDigest.Hash)) @@ -180,27 +214,102 @@ func TestContentAddressableStorageServerSplitBlob(t *testing.T) { blobDigest, err := digestFunction.NewDigestFromProto(request.BlobDigest) require.NoError(t, err) - chunkListStorage.EXPECT().Get(ctx, blobDigest).Return( - buffer.NewProtoBufferFromProto( - &remoteexecution.SplitBlobResponse{ - ChunkDigests: []*remoteexecution.Digest{ - { - Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", - SizeBytes: 8, - }, - { - Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", - SizeBytes: 8, - }, - }, - }, - buffer.UserProvided, - ), + chunk1Digest, err := digestFunction.NewDigestFromProto(&remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }) + require.NoError(t, err) + chunk2Digest, err := digestFunction.NewDigestFromProto(&remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 8, + }) + require.NoError(t, err) + + cas := mock.NewMockContentAddressableStorage(ctrl) + cas.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("my_instance_name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 4, HorizonSizeBytes: 8}, nil) + cas.EXPECT().GetManifest(ctx, blobDigest).Return( + chunklist.ChunkList{ + {Offset: 0, Digest: chunk1Digest}, + {Offset: 8, Digest: chunk2Digest}, + }, + nil, ) - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) - _, err = contentAddressableStorageServer.SplitBlob(ctx, request) + maximumMessageSizeBytes := 200 + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) + + response, err := contentAddressableStorageServer.SplitBlob(ctx, request) require.NoError(t, err) + require.Len(t, response.ChunkDigests, 2) + require.Equal(t, remoteexecution.ChunkingFunction_REP_MAX_CDC, response.ChunkingFunction) +} + +func TestContentAddressableStorageServerSplitBlobSingleChunk(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SplitBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + InstanceName: "my_instance_name", + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + + // The blob is small enough to fit in a single chunk. Its chunk + // list is never consulted; the blob itself is the single chunk. + cas.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("my_instance_name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil) + blobDigest, err := digest.MustNewFunction("my_instance_name", remoteexecution.DigestFunction_SHA256).NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + cas.EXPECT().FindMissing(ctx, blobDigest.ToSingletonSet()).Return(digest.EmptySet, nil) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, 200) + + response, err := contentAddressableStorageServer.SplitBlob(ctx, request) + require.NoError(t, err) + require.Len(t, response.ChunkDigests, 1) + testutil.RequireEqualProto(t, &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, response.ChunkDigests[0]) + require.Equal(t, remoteexecution.ChunkingFunction_REP_MAX_CDC, response.ChunkingFunction) +} + +func TestContentAddressableStorageServerSplitBlobNotFound(t *testing.T) { + ctrl, ctx := gomock.WithContext(context.Background(), t) + + request := &remoteexecution.SplitBlobRequest{ + BlobDigest: &remoteexecution.Digest{ + Hash: "409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49", + SizeBytes: 16, + }, + InstanceName: "my_instance_name", + DigestFunction: remoteexecution.DigestFunction_SHA256, + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + + // A single-chunk blob that does not exist. + cas.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("my_instance_name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil) + blobDigest, err := digest.MustNewFunction("my_instance_name", remoteexecution.DigestFunction_SHA256).NewDigestFromProto(request.BlobDigest) + require.NoError(t, err) + cas.EXPECT().FindMissing(ctx, blobDigest.ToSingletonSet()).Return(blobDigest.ToSingletonSet(), nil) + + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, 200) + + _, err = contentAddressableStorageServer.SplitBlob(ctx, request) + testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Blob 1-409a7f83ac6b31dc8c77e3ec18038f209bd2f545e0f4177c2e2381aa4e067b49-16-my_instance_name not found"), err) } func TestContentAddressableStorageServerSpliceBlob(t *testing.T) { @@ -224,9 +333,6 @@ func TestContentAddressableStorageServerSpliceBlob(t *testing.T) { InstanceName: "my_instance_name", } - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) - chunkListStorage := mock.NewMockBlobAccess(ctrl) - instanceName, err := digest.NewInstanceName(request.InstanceName) require.NoError(t, err) digestFunction, err := instanceName.GetDigestFunction(request.DigestFunction, len(request.BlobDigest.Hash)) @@ -234,11 +340,22 @@ func TestContentAddressableStorageServerSpliceBlob(t *testing.T) { blobDigest, err := digestFunction.NewDigestFromProto(request.BlobDigest) require.NoError(t, err) - chunkListStorage.EXPECT().Put(ctx, blobDigest, buffer.NewProtoBufferFromProto(&remoteexecution.SplitBlobResponse{ - ChunkDigests: request.ChunkDigests, - }, buffer.UserProvided)).Return(nil) + chunk1Digest, err := digestFunction.NewDigestFromProto(request.ChunkDigests[0]) + require.NoError(t, err) + chunk2Digest, err := digestFunction.NewDigestFromProto(request.ChunkDigests[1]) + require.NoError(t, err) + + expectedChunkList := chunklist.ChunkList{ + {Offset: 0, Digest: chunk1Digest}, + {Offset: 8, Digest: chunk2Digest}, + } + + cas := mock.NewMockContentAddressableStorage(ctrl) + cas.EXPECT().PutManifest(ctx, blobDigest, expectedChunkList).Return(nil) + + maximumMessageSizeBytes := 200 - contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(contentAddressableStorage, chunkListStorage, 200) + contentAddressableStorageServer := grpcservers.NewContentAddressableStorageServer(cas, int64(maximumMessageSizeBytes)) response, err := contentAddressableStorageServer.SpliceBlob(ctx, request) require.NoError(t, err) require.Equal(t, request.BlobDigest, response.BlobDigest) diff --git a/pkg/blobstore/grpcservers/integration/BUILD.bazel b/pkg/blobstore/grpcservers/integration/BUILD.bazel new file mode 100644 index 000000000..4381b6dbf --- /dev/null +++ b/pkg/blobstore/grpcservers/integration/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "integration_test", + timeout = "short", + srcs = [ + "byte_stream_server_test.go", + "content_addressable_storage_server_test.go", + "utils_test.go", + ], + data = [ + "//cmd/bb_replicator", + "//cmd/bb_storage", + ], + env = { + "BB_STORAGE_RUNFILE_PATH": "$(rlocationpath //cmd/bb_storage:bb_storage)", + "BB_REPLICATOR_RUNFILE_PATH": "$(rlocationpath //cmd/bb_replicator:bb_replicator)", + }, + deps = [ + "//pkg/digest", + "//pkg/util", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_google_uuid//:uuid", + "@com_github_klauspost_compress//zstd", + "@com_github_stretchr_testify//require", + "@org_golang_google_genproto_googleapis_bytestream//:bytestream", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//credentials/insecure", + "@org_golang_google_grpc//status", + "@org_golang_google_protobuf//proto", + "@rules_go//go/runfiles", + ], +) diff --git a/pkg/blobstore/grpcservers/integration/byte_stream_server_test.go b/pkg/blobstore/grpcservers/integration/byte_stream_server_test.go new file mode 100644 index 000000000..e7a944888 --- /dev/null +++ b/pkg/blobstore/grpcservers/integration/byte_stream_server_test.go @@ -0,0 +1,50 @@ +package integration + +import ( + "context" + "testing" + "time" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/stretchr/testify/require" +) + +func TestByteStreamAPI(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + blobData := makeRandomData(t, 2*maximumMessageSizeBytes, 0) + digest := computeDigest(blobData) + + tests := []struct { + name string + compressor remoteexecution.Compressor_Value + data []byte + }{ + {name: "IDENTITY", compressor: remoteexecution.Compressor_IDENTITY, data: blobData}, + {name: "ZSTD", compressor: remoteexecution.Compressor_ZSTD, data: zstdEncode(blobData)}, + } + + t.Run("Write and read back binary data", func(t *testing.T) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, _, bsClient := setupServers(t) + // Write blob. + err := bytestreamWriteBlob(ctx, bsClient, test.data, digest, test.compressor) + require.NoError(t, err, "Could not write blob") + + // Read back uncompressed. + receivedData, err := bytestreamReadBlob(ctx, bsClient, digest, remoteexecution.Compressor_IDENTITY) + require.NoError(t, err, "Could not read back uploaded data") + require.Equal(t, blobData, receivedData, "Downloaded payload does not match uploaded data") + + // Read back compressed. + receivedZstdData, err := bytestreamReadBlob(ctx, bsClient, digest, remoteexecution.Compressor_ZSTD) + require.NoError(t, err, "Could not read back uploaded data") + decompressedData, err := zstdDecode(receivedZstdData) + require.NoError(t, err, "Failed to decompress ZSTD payload") + require.Equal(t, blobData, decompressedData, "Downloaded compressed payload does not match uploaded data") + }) + } + }) +} diff --git a/pkg/blobstore/grpcservers/integration/content_addressable_storage_server_test.go b/pkg/blobstore/grpcservers/integration/content_addressable_storage_server_test.go new file mode 100644 index 000000000..da569c595 --- /dev/null +++ b/pkg/blobstore/grpcservers/integration/content_addressable_storage_server_test.go @@ -0,0 +1,404 @@ +package integration + +import ( + "context" + "testing" + "time" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/stretchr/testify/require" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +func TestContentAddressableStorageAPI(t *testing.T) { + blobData := makeRandomData(t, maximumMessageSizeBytes/2, 0) + blobDigest := computeDigest(blobData) + + t.Run("GetCapabilities", func(t *testing.T) { + capabilitiesClient, _, _, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + capabilities, err := capabilitiesClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ + InstanceName: "allowed_instance", + }) + require.NoError(t, err) + + cacheCaps := capabilities.GetCacheCapabilities() + require.NotNil(t, cacheCaps) + require.Contains(t, cacheCaps.DigestFunctions, remoteexecution.DigestFunction_SHA256) + + require.True(t, cacheCaps.SpliceBlobSupport) + require.True(t, cacheCaps.SplitBlobSupport) + + chunkingParameters := cacheCaps.GetRepMaxCdcParams() + require.NotNil(t, chunkingParameters) + require.Equal(t, minChunkSizeBytes, int(chunkingParameters.MinChunkSizeBytes)) + require.Equal(t, 8*minChunkSizeBytes, int(chunkingParameters.HorizonSizeBytes)) + + _, err = capabilitiesClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ + InstanceName: "forbidden_instance", + }) + require.Error(t, err) + status, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, codes.PermissionDenied, status.Code()) + }) + + t.Run("UploadSplitFMBDownload", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Upload a test blob. + err := batchUploadBlob(ctx, casClient, blobData, blobDigest) + require.NoError(t, err, "Failed to upload test data") + + // Ask server to split. + chunkDigests, err := splitBlob(ctx, casClient, blobDigest) + require.NoError(t, err, "Failed to split uploaded blob") + + // Check that all digests exist in CAS. + allDigests := append(chunkDigests, blobDigest) + missing, err := findMissingBlobs(ctx, casClient, allDigests) + require.NoError(t, err, "Failed to find missing blobs") + require.Empty(t, missing, "Blobs were unexpectedly missing") + + // Read back chunks and stitch them together. + chunks, err := batchDownloadBlobs(ctx, casClient, chunkDigests) + require.NoError(t, err, "Failed to download blobs") + rebuiltBlob := make([]byte, 0, len(blobData)) + for _, chunk := range chunks { + rebuiltBlob = append(rebuiltBlob, chunk...) + } + require.Equal(t, blobData, rebuiltBlob, "Blob did not stitch back into expected result") + + // Read back blob. + blobs, err := batchDownloadBlobs(ctx, casClient, []digest.Digest{blobDigest}) + require.NoError(t, err) + require.Len(t, blobs, 1) + require.Equal(t, blobData, blobs[0]) + }) +} + +func TestRepMaxCDCSplitAndSpliceBehaviors(t *testing.T) { + t.Run("RoundTripSplitThenSplice", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + dataSize := (minChunkSizeBytes * 4) + 128 + data := makeRandomData(t, int(dataSize), 0) + blobDigest := computeDigest(data) + err := batchUploadBlob(ctx, casClient, data, blobDigest) + require.NoError(t, err) + + digests, err := splitBlob(ctx, casClient, blobDigest) + require.NoError(t, err, "Unexpected error when splitting blob.") + + err = spliceBlob(ctx, casClient, blobDigest, digests) + require.NoError(t, err, "Unexpected error when splicing blob.") + }) + + t.Run("SpliceNonStandardChunkingThenSplit", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := makeRandomData(t, int(minChunkSizeBytes*2), 0) + + chunk1 := blobData[:1] + chunk2 := blobData[1:] + + digest1 := computeDigest(chunk1) + digest2 := computeDigest(chunk2) + blobDigest := computeDigest(blobData) + chunkDigests := []digest.Digest{digest1, digest2} + + err := batchUploadBlob(ctx, casClient, chunk1, digest1) + require.NoError(t, err, "Unexpected error when uploading chunk1.") + + err = batchUploadBlob(ctx, casClient, chunk2, digest2) + require.NoError(t, err, "Unexpected error when uploading chunk2.") + + err = spliceBlob(ctx, casClient, blobDigest, chunkDigests) + require.NoError(t, err, "Unexpected error when splicing chunks.") + + digests, err := splitBlob(ctx, casClient, blobDigest) + require.NoError(t, err, "Unexpected error when splitting recently spliced blob.") + require.NotEqual(t, chunkDigests, digests, "Split should not return non standard split result.") + + chunks, err := batchDownloadBlobs(ctx, casClient, digests) + require.NoError(t, err, "Unexpected error when downloading chunks of split blob.") + + rebuiltBlob := make([]byte, 0, len(blobData)) + for _, chunk := range chunks { + rebuiltBlob = append(rebuiltBlob, chunk...) + } + require.Equal(t, blobData, rebuiltBlob, "Blob did not stitch back into expected result") + }) + + t.Run("SpliceAlreadyExistsOrNoop", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := []byte("This blob will be fully uploaded before we try to splice it.") + blobDigest := computeDigest(blobData) + err := batchUploadBlob(ctx, casClient, blobData, blobDigest) + require.NoError(t, err, "Unexpected error when uploading blob.") + + chunk1 := blobData[:1] + chunk2 := blobData[1:] + digest1 := computeDigest(chunk1) + digest2 := computeDigest(chunk2) + chunkDigests := []digest.Digest{digest1, digest2} + + err = batchUploadBlob(ctx, casClient, chunk1, digest1) + require.NoError(t, err, "Unexpected error when uploading chunk1.") + + err = batchUploadBlob(ctx, casClient, chunk2, digest2) + require.NoError(t, err, "Unexpected error when uploading chunk2.") + + err = spliceBlob(ctx, casClient, blobDigest, chunkDigests) + if err != nil { + require.Equal(t, codes.AlreadyExists, status.Code(err), "Expected OK or ALREADY_EXISTS") + } + }) + + t.Run("ValidationSpliceBlobRejections", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + validData := makeRandomData(t, 512, 0) + validDigest := computeDigest(validData) + err := batchUploadBlob(ctx, casClient, validData, validDigest) + require.NoError(t, err, "Unexpected error when uploading blob.") + + ghostDigest := computeDigest([]byte("I do not exist in storage")) + + tests := []struct { + name string + blobDigest digest.Digest + chunkDigests []digest.Digest + expectError codes.Code + }{ + { + name: "Missing Chunk", + blobDigest: ghostDigest, + chunkDigests: []digest.Digest{ghostDigest}, + expectError: codes.NotFound, + }, + { + name: "Digest Mismatch", + blobDigest: computeDigest([]byte("Fake target")), + chunkDigests: []digest.Digest{validDigest}, + expectError: codes.InvalidArgument, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := spliceBlob(ctx, casClient, tc.blobDigest, tc.chunkDigests) + require.Error(t, err) + require.Equal(t, tc.expectError, status.Code(err), "Not the expected error: %s", err.Error()) + }) + } + }) + + t.Run("SpliceSplicedBlob", func(t *testing.T) { + _, casClient, _, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + blobData := makeRandomData(t, 2*minChunkSizeBytes+1, 0) + blobDigest := computeDigest(blobData) + chunk1 := blobData[:1] + chunk2 := blobData[1:] + digest1 := computeDigest(chunk1) + digest2 := computeDigest(chunk2) + chunkDigests := []digest.Digest{digest1, digest2} + + err := batchUploadBlob(ctx, casClient, chunk1, digest1) + require.NoError(t, err, "Unexpected error when uploading chunk1.") + + err = batchUploadBlob(ctx, casClient, chunk2, digest2) + require.NoError(t, err, "Unexpected error when uploading chunk2.") + + err = spliceBlob(ctx, casClient, blobDigest, chunkDigests) + require.NoError(t, err, "Unexpected error when splicing blob.") + }) +} + +func TestActionCacheAPI(t *testing.T) { + smallData := []byte("small file contents") + smallDigest := computeDigest(smallData) + + bigData := makeRandomData(t, int(minChunkSizeBytes*4), 0) + bigDigest := computeDigest(bigData) + + tree := &remoteexecution.Tree{ + Root: &remoteexecution.Directory{ + Files: []*remoteexecution.FileNode{ + {Name: "big.bin", Digest: bigDigest.GetProto(), IsExecutable: true}, + {Name: "small.txt", Digest: smallDigest.GetProto()}, + }, + }, + } + treeData, err := proto.Marshal(tree) + require.NoError(t, err) + treeDigest := computeDigest(treeData) + + actionResult := &remoteexecution.ActionResult{ + OutputDirectories: []*remoteexecution.OutputDirectory{ + {Path: "build_output", TreeDigest: treeDigest.GetProto()}, + }, + ExitCode: 0, + } + + actionDigest := computeDigest(makeRandomData(t, 128, 0)) + + t.Run("CompleteActionResult", func(t *testing.T) { + _, casClient, acClient, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + require.NoError(t, batchUploadBlob(ctx, casClient, smallData, smallDigest)) + require.NoError(t, batchUploadBlob(ctx, casClient, bigData, bigDigest)) + require.NoError(t, batchUploadBlob(ctx, casClient, treeData, treeDigest)) + + err := updateActionResult(ctx, acClient, actionDigest, actionResult) + require.NoError(t, err) + + getResp, err := getActionResult(ctx, acClient, actionDigest) + require.NoError(t, err) + require.Equal(t, actionResult.ExitCode, getResp.ExitCode) + require.Len(t, getResp.OutputDirectories, 1) + require.Equal(t, treeDigest.GetProto().Hash, getResp.OutputDirectories[0].TreeDigest.Hash) + }) + + t.Run("IncompleteActionResult", func(t *testing.T) { + _, casClient, acClient, _ := setupServers(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Big data has not been uploaded in this test + require.NoError(t, batchUploadBlob(ctx, casClient, smallData, smallDigest)) + require.NoError(t, batchUploadBlob(ctx, casClient, treeData, treeDigest)) + + err = updateActionResult(ctx, acClient, actionDigest, actionResult) + require.NoError(t, err, "Unexpected error updating action result") + + _, err := getActionResult(ctx, acClient, actionDigest) + require.Error(t, err, "Incomplete action result should fail") + require.Equal(t, codes.NotFound, status.Code(err)) + }) +} + +func TestAPIAuthorizationRejections(t *testing.T) { + capClient, casClient, acClient, bsClient := setupServers(t) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + forbiddenInstance := "forbidden_instance" + data := []byte("top secret data") + forbiddenDigest := computeDigestWithInstanceName(data, forbiddenInstance) + chunkDigest1 := computeDigestWithInstanceName([]byte("top "), forbiddenInstance) + chunkDigest2 := computeDigestWithInstanceName([]byte("secret data"), forbiddenInstance) + + dummyActionResult := &remoteexecution.ActionResult{ + ExitCode: 0, + } + + tests := []struct { + name string + call func() error + }{ + { + name: "GetCapabilities", + call: func() error { + _, err := capClient.GetCapabilities(ctx, &remoteexecution.GetCapabilitiesRequest{ + InstanceName: forbiddenInstance, + }) + return err + }, + }, + { + name: "FindMissingBlobs", + call: func() error { + _, err := findMissingBlobs(ctx, casClient, []digest.Digest{forbiddenDigest}) + return err + }, + }, + { + name: "BatchUpdateBlobs", + call: func() error { + return batchUploadBlob(ctx, casClient, data, forbiddenDigest) + }, + }, + { + name: "BatchReadBlobs", + call: func() error { + _, err := batchDownloadBlobs(ctx, casClient, []digest.Digest{forbiddenDigest}) + return err + }, + }, + { + name: "SplitBlob", + call: func() error { + _, err := splitBlob(ctx, casClient, forbiddenDigest) + return err + }, + }, + { + name: "SpliceBlob", + call: func() error { + return spliceBlob(ctx, casClient, forbiddenDigest, []digest.Digest{chunkDigest1, chunkDigest2}) + }, + }, + { + name: "ByteStream Write", + call: func() error { + return bytestreamWriteBlob(ctx, bsClient, data, forbiddenDigest, remoteexecution.Compressor_IDENTITY) + }, + }, + { + name: "ByteStream Read", + call: func() error { + _, err := bytestreamReadBlob(ctx, bsClient, forbiddenDigest, remoteexecution.Compressor_IDENTITY) + return err + }, + }, + { + name: "ActionCache UpdateActionResult", + call: func() error { + return updateActionResult(ctx, acClient, forbiddenDigest, dummyActionResult) + }, + }, + { + name: "ActionCache GetActionResult", + call: func() error { + _, err := getActionResult(ctx, acClient, forbiddenDigest) + return err + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.call() + require.Error(t, err, "Expected API call to fail for forbidden instance") + require.Equal(t, codes.PermissionDenied, status.Code(err), "Expected PermissionDenied, got %v", status.Code(err)) + }) + } +} diff --git a/pkg/blobstore/grpcservers/integration/utils_test.go b/pkg/blobstore/grpcservers/integration/utils_test.go new file mode 100644 index 000000000..178c7f547 --- /dev/null +++ b/pkg/blobstore/grpcservers/integration/utils_test.go @@ -0,0 +1,618 @@ +package integration + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math/rand" + "os" + "os/exec" + "strings" + "testing" + "time" + + remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/util" + "github.com/google/uuid" + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" + "google.golang.org/genproto/googleapis/bytestream" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +type storageAddresses struct { + storageShardSocketPaths []string + localCacheSocketPath string +} + +const ( + maximumMessageSizeBytes = 2 << 20 // 2MiB + minChunkSizeBytes = 256 << 10 // 256KiB + bytestreamWriteChunkSizeBytes = 1 << 20 // 1MiB +) + +func storageConfig(listenSocketPath string) string { + return fmt.Sprintf(` +local listenPath = '%s'; +local maximumMessageSizeBytes = %d; +local minChunkSizeBytes = %d; + +local inMemoryStorage = { + keyLocationMapInMemory: { entries: 1024 * 1024 }, + keyLocationMapMaximumGetAttempts: 32, + keyLocationMapMaximumPutAttempts: 64, + oldBlocks: 1, + currentBlocks: 1, + newBlocks: 1, + blocksInMemory: { blockSizeBytes: 32 * 1024 * 1024 }, +}; + +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { allow: {} }, + }], + maximumMessageSizeBytes: maximumMessageSizeBytes, + contentAddressableStorageServer: { + contentAddressableStorage: { + chunkStorage: { + 'local': inMemoryStorage + { chunkingParameters: { minChunkSizeBytes: minChunkSizeBytes, horizonSizeBytes: 8*minChunkSizeBytes } }, + }, + chunkListStorage: { + 'local': inMemoryStorage, + }, + }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + findMissingAuthorizer: { allow: {} }, + }, + actionCache: { + backend: { 'local': inMemoryStorage }, + getAuthorizer: { allow: {} }, + putAuthorizer: { allow: {} }, + }, +} +`, escapeJSON(listenSocketPath), maximumMessageSizeBytes, minChunkSizeBytes) +} + +func replicatorConfig(listenSocketPath string, params storageAddresses) string { + upstreamsJSON, err := json.Marshal(params.storageShardSocketPaths) + if err != nil { + panic(err) + } + return fmt.Sprintf(` +local listenPath = '%s'; +local upstreamAddresses = %s; +local localCacheAddress = "unix:%s"; +local maximumMessageSizeBytes = %d; + +local topology = { + sharding: { + shards: { + ["shard" + i]: { backend: { grpc: { client: { address: "unix:" + upstreamAddresses[i] } } }, weight: 1 } + for i in std.range(0, std.length(upstreamAddresses) - 1) + } + } +}; + +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { allow: {} }, + }], + maximumMessageSizeBytes: maximumMessageSizeBytes, + source: topology, + sink: { grpc: { client: { address: localCacheAddress } } }, + replicator: { + deduplicating: { + concurrencyLimiting: { + base: { 'local': {} }, + maximumConcurrency: 1, + }, + }, + } +} +`, escapeJSON(listenSocketPath), upstreamsJSON, escapeJSON(params.localCacheSocketPath), maximumMessageSizeBytes) +} + +func frontendConfig(listenSocketPath string, params storageAddresses, replicatorSocketPath string) string { + upstreamsJSON, err := json.Marshal(params.storageShardSocketPaths) + if err != nil { + panic(err) + } + return fmt.Sprintf(` +local listenPath = '%s'; +local upstreamAddresses = %s; +local localCacheAddress = "unix:%s"; +local replicatorAddress = "unix:%s"; +local maximumMessageSizeBytes = %d; + +local topology = { + sharding: { + shards: { + ["shard" + i]: { + backend: { + grpc: { + client: { address: "unix:" + upstreamAddresses[i] }, + }, + }, + weight: 1, + } + for i in std.range(0, std.length(upstreamAddresses) - 1) + } + } +}; + +local readCaching(inner, replicator) = { + readCaching: { + fast: { grpc: { client: { address: localCacheAddress } } }, + slow: inner, + replicator: replicator, + }, +}; + +local authorizer = { + jmespathExpression: { + expression: 'contains(authenticationMetadata.private.mayAccess, instanceName)' + }, +}; + +local csReplicator = { remote: { address: replicatorAddress } }; +local simpleReplicator = { 'local': {} }; + +{ + grpcServers: [{ + listenPaths: [listenPath], + authenticationPolicy: { + allow: { + private: { + mayAccess: ['allowed_instance'], + }, + }, + }, + }], + supportedCompressors: ['ZSTD'], + zstdPool: { + maximumEncoders: 16, + maximumDecoders: 16, + encoderWindowSizeBytes: 8 * 1024 * 1024, + decoderWindowSizeBytes: 8 * 1024 * 1024, + }, + maximumMessageSizeBytes: maximumMessageSizeBytes, + contentAddressableStorageServer: { + contentAddressableStorage: { + chunkStorage: readCaching(topology, csReplicator), + chunkListStorage: readCaching({ chunkListValidating: { backend: topology } }, simpleReplicator), + cdcParameterCache: { + cacheSize: 1, + cacheDuration: '60s', + cacheReplacementPolicy: 'LEAST_RECENTLY_USED', + }, + }, + getAuthorizer: authorizer, + putAuthorizer: authorizer, + findMissingAuthorizer: authorizer, + }, + actionCache: { + backend: readCaching( + { + completenessChecking: { + backend: topology, + maximumTotalTreeSizeBytes: 64 * 1024 * 1024, + }, + }, + simpleReplicator, + ), + getAuthorizer: authorizer, + putAuthorizer: authorizer, + }, +} +`, escapeJSON(listenSocketPath), upstreamsJSON, escapeJSON(params.localCacheSocketPath), escapeJSON(replicatorSocketPath), maximumMessageSizeBytes) +} + +func writeConfigFile(name, content string) (file *os.File, err error) { + if file, err = os.CreateTemp("", name); err != nil { + return nil, err + } + if _, err = file.WriteString(content); err != nil { + return nil, err + } + if err = file.Close(); err != nil { + return nil, err + } + return file, nil +} + +func getBinaryPath(envVar string) (string, error) { + rf, err := runfiles.New() + if err != nil { + return "", util.StatusWrap(err, "Failed to initialize runfiles") + } + + runfilePath := os.Getenv(envVar) + if runfilePath == "" { + return "", util.StatusWrapf(err, "'%s' environment variable is not set", envVar) + } + + return rf.Rlocation(runfilePath) +} + +func setupServers(t *testing.T) (remoteexecution.CapabilitiesClient, remoteexecution.ContentAddressableStorageClient, remoteexecution.ActionCacheClient, bytestream.ByteStreamClient) { + storageBinaryPath, err := getBinaryPath("BB_STORAGE_RUNFILE_PATH") + require.NoError(t, err, "Could not get storage binary path") + replicatorBinaryPath, err := getBinaryPath("BB_REPLICATOR_RUNFILE_PATH") + require.NoError(t, err, "Could not get replicator binary path") + + storageSocketPaths := make([]string, 2) + for i := 0; i < 2; i++ { + storageSocketPaths[i] = createSocketPath(t) + setupServer(t, storageBinaryPath, fmt.Sprintf("storage%d", i), storageConfig(storageSocketPaths[i])) + require.Eventually(t, func() bool { + _, err := os.Stat(storageSocketPaths[i]) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "Storage server %d did not start.", i) + } + + localCacheSocketPath := createSocketPath(t) + setupServer(t, storageBinaryPath, "localCache", storageConfig(localCacheSocketPath)) + require.Eventually(t, func() bool { + _, err := os.Stat(localCacheSocketPath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "Local cache storage server did not start.") + + storageAddresses := storageAddresses{ + storageShardSocketPaths: storageSocketPaths, + localCacheSocketPath: localCacheSocketPath, + } + + replicatorSocketPath := createSocketPath(t) + setupServer(t, replicatorBinaryPath, "replicator", replicatorConfig(replicatorSocketPath, storageAddresses)) + require.Eventually(t, func() bool { + _, err := os.Stat(replicatorSocketPath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "Replicator did not start.") + + frontendSocketPath := createSocketPath(t) + setupServer(t, storageBinaryPath, "frontend", frontendConfig(frontendSocketPath, storageAddresses, replicatorSocketPath)) + require.Eventually(t, func() bool { + _, err := os.Stat(frontendSocketPath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "Frontend server did not start.") + + conn, err := grpc.NewClient(fmt.Sprintf("unix:%s", frontendSocketPath), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + + t.Cleanup(func() { + conn.Close() + }) + + return remoteexecution.NewCapabilitiesClient(conn), + remoteexecution.NewContentAddressableStorageClient(conn), + remoteexecution.NewActionCacheClient(conn), + bytestream.NewByteStreamClient(conn) +} + +func setupServer(t *testing.T, binaryPath, name, config string) { + configFile, err := writeConfigFile(name, config) + require.NoError(t, err, "Could not write config file") + + cmd := exec.Command(binaryPath, configFile.Name()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Start() + require.NoError(t, err) + + t.Cleanup(func() { + cmd.Process.Kill() + cmd.Wait() + os.Remove(configFile.Name()) + }) +} + +func createSocketPath(t *testing.T) string { + t.Helper() + socketFile, err := os.CreateTemp("", "bb_*.sock") + require.NoError(t, err) + socketPath := socketFile.Name() + socketFile.Close() + err = os.Remove(socketPath) + require.NoError(t, err) + return socketPath +} + +func computeDigestWithInstanceName(data []byte, instanceName string) digest.Digest { + hash := sha256.Sum256(data) + return digest.MustNewDigest( + instanceName, + remoteexecution.DigestFunction_SHA256, + hex.EncodeToString(hash[:]), + int64(len(data)), + ) +} + +func computeDigest(data []byte) digest.Digest { + return computeDigestWithInstanceName(data, "allowed_instance") +} + +func escapeJSON(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, "\"", "\\\"") + s = strings.ReplaceAll(s, "'", "\\'") + s = strings.ReplaceAll(s, "\n", "\\n") + return s +} + +func makeRandomData(t *testing.T, size int, seed int64) []byte { + t.Helper() + data := make([]byte, size) + r := rand.New(rand.NewSource(seed)) + _, err := r.Read(data) + require.NoError(t, err) + return data +} + +func zstdEncode(data []byte) []byte { + encoder, err := zstd.NewWriter(nil) + if err != nil { + panic(err) + } + defer encoder.Close() + return encoder.EncodeAll(data, nil) +} + +func zstdDecode(data []byte) ([]byte, error) { + decoder, err := zstd.NewReader(nil) + if err != nil { + panic(err) + } + defer decoder.Close() + ret, err := decoder.DecodeAll(data, nil) + if err != nil { + return nil, err + } + return ret, nil +} + +func batchUploadBlob(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, data []byte, d digest.Digest) error { + req := &remoteexecution.BatchUpdateBlobsRequest{ + InstanceName: d.GetInstanceName().String(), + Requests: []*remoteexecution.BatchUpdateBlobsRequest_Request{ + { + Digest: d.GetProto(), + Data: data, + }, + }, + } + + responses, err := client.BatchUpdateBlobs(ctx, req) + if err != nil { + return err + } + for _, response := range responses.Responses { + return status.ErrorProto(response.Status) + } + return err +} + +func batchDownloadBlobs(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, digests []digest.Digest) ([][]byte, error) { + if len(digests) == 0 { + return nil, nil + } + + dataMap := make(map[string][]byte, len(digests)) + + batchStart := 0 + for batchStart < len(digests) { + batchSize := int64(0) + batchEnd := batchStart + + // Slide the window forward until we hit our byte limit. We + // always include at least one digest, even if it exceeds the + // limit. + for batchEnd < len(digests) { + size := digests[batchEnd].GetSizeBytes() + if batchEnd > batchStart && batchSize+size > maximumMessageSizeBytes/2 { + break + } + batchSize += size + batchEnd++ + } + + // Prepare the batch request. + var batchProtos []*remoteexecution.Digest + for _, d := range digests[batchStart:batchEnd] { + batchProtos = append(batchProtos, d.GetProto()) + } + + req := &remoteexecution.BatchReadBlobsRequest{ + InstanceName: digests[0].GetInstanceName().String(), + Digests: batchProtos, + } + + // Execute the RPC for this specific batch. + res, err := client.BatchReadBlobs(ctx, req) + if err != nil { + return nil, err + } + + for _, r := range res.Responses { + if r.Status != nil && r.Status.Code != int32(codes.OK) { + return nil, status.ErrorProto(r.Status) + } + dataMap[r.Digest.Hash] = r.Data + } + + // Move the window forward for the next batch. + batchStart = batchEnd + } + + // Map the responses back to the requested order. + var downloadedData [][]byte + for _, d := range digests { + data, ok := dataMap[d.GetHashString()] + if !ok { + return nil, status.Errorf(codes.NotFound, "Digest %s was not returned in BatchReadBlobs response", d.GetHashString()) + } + downloadedData = append(downloadedData, data) + } + + return downloadedData, nil +} + +func findMissingBlobs(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, digests []digest.Digest) ([]digest.Digest, error) { + if len(digests) == 0 { + return nil, nil + } + + var digestProtos []*remoteexecution.Digest + for _, d := range digests { + digestProtos = append(digestProtos, d.GetProto()) + } + + req := &remoteexecution.FindMissingBlobsRequest{ + InstanceName: digests[0].GetInstanceName().String(), + BlobDigests: digestProtos, + } + + res, err := client.FindMissingBlobs(ctx, req) + if err != nil { + return nil, err + } + + var missingDigests []digest.Digest + digestFunction := digests[0].GetDigestFunction() + for _, p := range res.MissingBlobDigests { + d, err := digestFunction.NewDigestFromProto(p) + if err != nil { + return nil, util.StatusWrap(err, "Failed to parse missing digest from proto") + } + missingDigests = append(missingDigests, d) + } + + return missingDigests, nil +} + +func splitBlob(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, d digest.Digest) ([]digest.Digest, error) { + req := &remoteexecution.SplitBlobRequest{ + InstanceName: d.GetInstanceName().String(), + BlobDigest: d.GetProto(), + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + + res, err := client.SplitBlob(ctx, req) + if err != nil { + return nil, err + } + + chunkDigests := make([]digest.Digest, 0, len(res.ChunkDigests)) + digestFunction := d.GetDigestFunction() + for _, chunkProto := range res.ChunkDigests { + chunkDigest, err := digestFunction.NewDigestFromProto(chunkProto) + if err != nil { + return nil, util.StatusWrap(err, "Failed to parse chunk digest from proto") + } + chunkDigests = append(chunkDigests, chunkDigest) + } + + return chunkDigests, nil +} + +func spliceBlob(ctx context.Context, client remoteexecution.ContentAddressableStorageClient, blob digest.Digest, chunks []digest.Digest) error { + chunkDigests := make([]*remoteexecution.Digest, 0, len(chunks)) + for _, d := range chunks { + chunkDigests = append(chunkDigests, d.GetProto()) + } + + req := &remoteexecution.SpliceBlobRequest{ + InstanceName: blob.GetInstanceName().String(), + BlobDigest: blob.GetProto(), + ChunkDigests: chunkDigests, + ChunkingFunction: remoteexecution.ChunkingFunction_REP_MAX_CDC, + } + + _, err := client.SpliceBlob(ctx, req) + return err +} + +func bytestreamWriteBlob(ctx context.Context, client bytestream.ByteStreamClient, data []byte, digest digest.Digest, compressor remoteexecution.Compressor_Value) error { + writeStream, err := client.Write(ctx) + if err != nil { + return err + } + offset := int64(0) + dataSize := int64(len(data)) + + for offset < dataSize { + end := offset + bytestreamWriteChunkSizeBytes + if end > dataSize { + end = dataSize + } + + chunk := data[offset:end] + isLast := end == dataSize + + err := writeStream.Send(&bytestream.WriteRequest{ + ResourceName: digest.GetByteStreamWritePath(uuid.New(), compressor), + WriteOffset: offset, + FinishWrite: isLast, + Data: chunk, + }) + if err == io.EOF { + _, innerErr := writeStream.CloseAndRecv() + if innerErr != nil { + return util.StatusFromMultiple([]error{innerErr, err}) + } + } else if err != nil { + return err + } + offset = end + } + _, err = writeStream.CloseAndRecv() + return err +} + +func bytestreamReadBlob(ctx context.Context, client bytestream.ByteStreamClient, digest digest.Digest, compressor remoteexecution.Compressor_Value) ([]byte, error) { + readReq := &bytestream.ReadRequest{ + ResourceName: digest.GetByteStreamReadPath(compressor), + } + readStream, err := client.Read(ctx, readReq) + if err != nil { + return nil, err + } + defer readStream.CloseSend() + + receivedData := make([]byte, 0, digest.GetSizeBytes()) + for { + res, err := readStream.Recv() + if err == io.EOF { + return receivedData, nil + } else if err != nil { + return nil, err + } + receivedData = append(receivedData, res.Data...) + } +} + +func updateActionResult(ctx context.Context, acClient remoteexecution.ActionCacheClient, actionDigest digest.Digest, result *remoteexecution.ActionResult) error { + _, err := acClient.UpdateActionResult(ctx, &remoteexecution.UpdateActionResultRequest{ + InstanceName: actionDigest.GetInstanceName().String(), + ActionDigest: actionDigest.GetProto(), + ActionResult: result, + }) + return err +} + +func getActionResult(ctx context.Context, acClient remoteexecution.ActionCacheClient, actionDigest digest.Digest) (*remoteexecution.ActionResult, error) { + return acClient.GetActionResult(ctx, &remoteexecution.GetActionResultRequest{ + InstanceName: actionDigest.GetInstanceName().String(), + ActionDigest: actionDigest.GetProto(), + }) +} diff --git a/pkg/blobstore/hierarchical_instance_names_blob_access.go b/pkg/blobstore/hierarchical_instance_names_blob_access.go index 87fb5149c..87cadafd4 100644 --- a/pkg/blobstore/hierarchical_instance_names_blob_access.go +++ b/pkg/blobstore/hierarchical_instance_names_blob_access.go @@ -4,7 +4,6 @@ import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -42,21 +41,6 @@ func (ba *hierarchicalInstanceNamesBlobAccess) Get(ctx context.Context, digest d ) } -func (ba *hierarchicalInstanceNamesBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - parentDigests := parentDigest.GetDigestsWithParentInstanceNames() - childDigests := childDigest.GetDigestsWithParentInstanceNames() - return buffer.WithErrorHandler( - ba.BlobAccess.GetFromComposite(ctx, parentDigests[len(parentDigests)-1], childDigests[len(childDigests)-1], slicer), - &hierarchicalInstanceNamesGetFromCompositeErrorHandler{ - blobAccess: ba.BlobAccess, - context: ctx, - parentDigests: parentDigests, - childDigests: childDigests, - slicer: slicer, - }, - ) -} - func (ba *hierarchicalInstanceNamesBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { // Skip as much work as possible during the initial phase by // just requesting the original set of digests. This keeps the @@ -156,34 +140,3 @@ func (eh *hierarchicalInstanceNamesGetErrorHandler) OnError(err error) (buffer.B } func (hierarchicalInstanceNamesGetErrorHandler) Done() {} - -type hierarchicalInstanceNamesGetFromCompositeErrorHandler struct { - blobAccess BlobAccess - context context.Context - parentDigests []digest.Digest - childDigests []digest.Digest - slicer slicing.BlobSlicer -} - -func (eh *hierarchicalInstanceNamesGetFromCompositeErrorHandler) OnError(err error) (buffer.Buffer, error) { - if status.Code(err) != codes.NotFound { - // Serious error. Prepend the instance name, so that - // errors can be disambiguated. - return nil, util.StatusWrapf(err, "Instance name %#v", eh.parentDigests[len(eh.parentDigests)-1].GetInstanceName().String()) - } - if len(eh.parentDigests) == 1 { - // The object was found in none of the instance names. - // There is no need to prepend the instance name. - return nil, err - } - eh.parentDigests = eh.parentDigests[:len(eh.parentDigests)-1] - eh.childDigests = eh.childDigests[:len(eh.childDigests)-1] - return eh.blobAccess.GetFromComposite( - eh.context, - eh.parentDigests[len(eh.parentDigests)-1], - eh.childDigests[len(eh.childDigests)-1], - eh.slicer, - ), nil -} - -func (hierarchicalInstanceNamesGetFromCompositeErrorHandler) Done() {} diff --git a/pkg/blobstore/hierarchical_instance_names_blob_access_test.go b/pkg/blobstore/hierarchical_instance_names_blob_access_test.go index 493d359ba..8118d1a4d 100644 --- a/pkg/blobstore/hierarchical_instance_names_blob_access_test.go +++ b/pkg/blobstore/hierarchical_instance_names_blob_access_test.go @@ -73,65 +73,6 @@ func TestHierarchicalInstanceNamesBlobAccessGet(t *testing.T) { }) } -func TestHierarchicalInstanceNamesBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - baseBlobAccess := mock.NewMockBlobAccess(ctrl) - blobAccess := blobstore.NewHierarchicalInstanceNamesBlobAccess(baseBlobAccess) - - helloDigest1 := digest.MustNewDigest("a/b", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - helloDigest2 := digest.MustNewDigest("a", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - helloDigest3 := digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - ellDigest1 := digest.MustNewDigest("a/b", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 5) - ellDigest2 := digest.MustNewDigest("a", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 5) - ellDigest3 := digest.MustNewDigest("", remoteexecution.DigestFunction_MD5, "3123059c1c816471780539f6b6b738dc", 5) - slicer := mock.NewMockBlobSlicer(ctrl) - - t.Run("Failure", func(t *testing.T) { - // Errors from backends should be propagated. The - // instance name should be prepended to the error - // message, to disambiguate. - gomock.InOrder( - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest1, ellDigest1, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest2, ellDigest2, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.Internal, "Disk on fire"))), - ) - - _, err := blobAccess.GetFromComposite(ctx, helloDigest1, ellDigest1, slicer).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Instance name \"a\": Disk on fire"), err) - }) - - t.Run("NotFound", func(t *testing.T) { - // NotFound errors should cause requests to be retried - // against parent instance names. - gomock.InOrder( - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest1, ellDigest1, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest2, ellDigest2, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest3, ellDigest3, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - ) - - _, err := blobAccess.GetFromComposite(ctx, helloDigest1, ellDigest1, slicer).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Object not found"), err) - }) - - t.Run("Success", func(t *testing.T) { - gomock.InOrder( - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest1, ellDigest1, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))), - baseBlobAccess.EXPECT().GetFromComposite(ctx, helloDigest2, ellDigest2, slicer). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("ell"))), - ) - - data, err := blobAccess.GetFromComposite(ctx, helloDigest1, ellDigest1, slicer).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("ell"), data) - }) -} - func TestHierarchicalInstanceNamesBlobAccessFindMissing(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/local/BUILD.bazel b/pkg/blobstore/local/BUILD.bazel index b6f9ae1d7..e9c57aea0 100644 --- a/pkg/blobstore/local/BUILD.bazel +++ b/pkg/blobstore/local/BUILD.bazel @@ -72,7 +72,6 @@ go_test( "//internal/mock", "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/filesystem", "//pkg/filesystem/path", diff --git a/pkg/blobstore/local/flat_blob_access_test.go b/pkg/blobstore/local/flat_blob_access_test.go index a7d9a34d6..7b5f9b4a5 100644 --- a/pkg/blobstore/local/flat_blob_access_test.go +++ b/pkg/blobstore/local/flat_blob_access_test.go @@ -9,7 +9,6 @@ import ( "github.com/buildbarn/bb-storage/internal/mock" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/local" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/testutil" "github.com/stretchr/testify/require" @@ -191,284 +190,6 @@ func TestFlatBlobAccessGet(t *testing.T) { }) } -func TestFlatBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - keyLocationMap := mock.NewMockKeyLocationMap(ctrl) - locationBlobMap := mock.NewMockLocationBlobMap(ctrl) - capabilitiesProvider := mock.NewMockCapabilitiesProvider(ctrl) - blobAccess := local.NewFlatBlobAccess(keyLocationMap, locationBlobMap, digest.KeyWithoutInstance, &sync.RWMutex{}, "cas", capabilitiesProvider) - parentDigest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "3e25960a79dbc69b674cd4ec67a72c62", 11) - parentKey := local.NewKeyFromString("3-3e25960a79dbc69b674cd4ec67a72c62-11") - child1Digest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - child1Key := local.NewKeyFromString("3-8b1a9953c4611296a827abf8c47804d7-5") - child2Digest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "7d793037a0760186574b0282f2f435e7", 5) - child2Key := local.NewKeyFromString("3-7d793037a0760186574b0282f2f435e7-5") - slicer := mock.NewMockBlobSlicer(ctrl) - location1 := local.Location{ - BlockIndex: 7, - OffsetBytes: 42, - SizeBytes: 11, - } - location2 := local.Location{ - BlockIndex: 8, - OffsetBytes: 382, - SizeBytes: 5, - } - - t.Run("NoSlicingParentNotFound", func(t *testing.T) { - // Even if the requested child object were to exist, the - // existence of the parent object is the one that - // counts. If the parent object does not exist, we - // should report it as being absent, so that it may be - // replicated/regenerated. - keyLocationMap.EXPECT().Get(parentKey). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Blob not found"), err) - }) - - t.Run("NoSlicingChildFailure", func(t *testing.T) { - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.Internal, "I/O error")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "I/O error"), err) - }) - - t.Run("NoSlicingSuccess", func(t *testing.T) { - // If both the parent doesn't need to be refreshed and - // the child exists, we can return data immediately. - // Don't bother taking into account whether the child - // needs to be refreshed. We'll just slice the object - // again by the time the child gets lost. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(location2, nil) - childGetter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location2). - Return(childGetter.Call, true) - childGetter.EXPECT().Call(child1Digest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) - - t.Run("SlicingParentNotFound", func(t *testing.T) { - // An initial lookup on the parent returned success, but - // when retrying the lookup while holding an exclusive - // lock, we got NotFound. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - getter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(getter.Call, true) - keyLocationMap.EXPECT().Get(parentKey). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.NotFound, "Blob not found"), err) - }) - - t.Run("SlicingChildFailure", func(t *testing.T) { - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.Internal, "I/O error")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "I/O error"), err) - }) - - t.Run("SlicingButNoLongerNeeded", func(t *testing.T) { - // An initial lookup indicated that the parent needed to - // be sliced, but a second lookup while holding an - // exclusive lock showed that this is no longer needed. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, true) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(location2, nil) - childGetter := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location2). - Return(childGetter.Call, true) - childGetter.EXPECT().Call(child1Digest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) - - t.Run("SlicingPutFailure", func(t *testing.T) { - // Successfully sliced an object, but we failed to - // insert new key-location map entries for the resulting - // slices. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - parentGetter2.EXPECT().Call(parentDigest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - slicer.EXPECT().Slice(gomock.Any(), child1Digest).DoAndReturn(func(b buffer.Buffer, child1Digest digest.Digest) (buffer.Buffer, []slicing.BlobSlice) { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - return buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), []slicing.BlobSlice{ - {Digest: child1Digest, OffsetBytes: 0, SizeBytes: 5}, - {Digest: child2Digest, OffsetBytes: 6, SizeBytes: 5}, - } - }) - keyLocationMap.EXPECT().Put(child1Key, local.Location{ - BlockIndex: 7, - OffsetBytes: 42, - SizeBytes: 5, - }).Return(status.Error(codes.Internal, "I/O error")) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Failed to create child blob \"3-8b1a9953c4611296a827abf8c47804d7-5-example\": I/O error"), err) - }) - - t.Run("SlicingSuccess", func(t *testing.T) { - // Successful instance where an object is sliced, and the - // contents of a single slice is returned. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, false) - keyLocationMap.EXPECT().Get(child1Key). - Return(local.Location{}, status.Error(codes.NotFound, "Blob not found")) - parentGetter2.EXPECT().Call(parentDigest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - slicer.EXPECT().Slice(gomock.Any(), child1Digest).DoAndReturn(func(b buffer.Buffer, child1Digest digest.Digest) (buffer.Buffer, []slicing.BlobSlice) { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - return buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), []slicing.BlobSlice{ - {Digest: child1Digest, OffsetBytes: 0, SizeBytes: 5}, - {Digest: child2Digest, OffsetBytes: 6, SizeBytes: 5}, - } - }) - keyLocationMap.EXPECT().Put(child1Key, local.Location{ - BlockIndex: 7, - OffsetBytes: 42, - SizeBytes: 5, - }) - keyLocationMap.EXPECT().Put(child2Key, local.Location{ - BlockIndex: 7, - OffsetBytes: 48, - SizeBytes: 5, - }) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) - - t.Run("RefreshSuccess", func(t *testing.T) { - // Successful instance where an object is both refreshed and - // sliced, and the contents of a single slice is returned. - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter1 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter1.Call, true) - keyLocationMap.EXPECT().Get(parentKey). - Return(location1, nil) - parentGetter2 := mock.NewMockLocationBlobGetter(ctrl) - locationBlobMap.EXPECT().Get(location1). - Return(parentGetter2.Call, true) - parentGetter2.EXPECT().Call(parentDigest). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - putWriter := mock.NewMockLocationBlobPutWriter(ctrl) - locationBlobMap.EXPECT().Put(int64(11)). - Return(putWriter.Call, nil) - putFinalizer := mock.NewMockLocationBlobPutFinalizer(ctrl) - putWriter.EXPECT().Call(gomock.Any()).DoAndReturn(func(b buffer.Buffer) local.LocationBlobPutFinalizer { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - return putFinalizer.Call - }) - putFinalizer.EXPECT().Call(). - Return(location2, nil) - keyLocationMap.EXPECT().Put(parentKey, location2) - slicer.EXPECT().Slice(gomock.Any(), child1Digest).DoAndReturn(func(b buffer.Buffer, child1Digest digest.Digest) (buffer.Buffer, []slicing.BlobSlice) { - data, err := b.ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - return buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), []slicing.BlobSlice{ - {Digest: child1Digest, OffsetBytes: 0, SizeBytes: 5}, - {Digest: child2Digest, OffsetBytes: 6, SizeBytes: 5}, - } - }) - keyLocationMap.EXPECT().Put(child1Key, local.Location{ - BlockIndex: 8, - OffsetBytes: 382, - SizeBytes: 5, - }) - keyLocationMap.EXPECT().Put(child2Key, local.Location{ - BlockIndex: 8, - OffsetBytes: 388, - SizeBytes: 5, - }) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, child1Digest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) - - // TODO: Some testing coverage may be missing for the cases - // where refreshing fails. -} - func TestFlatBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/metrics_blob_access.go b/pkg/blobstore/metrics_blob_access.go index 0941156bc..dc1f4a2e8 100644 --- a/pkg/blobstore/metrics_blob_access.go +++ b/pkg/blobstore/metrics_blob_access.go @@ -7,7 +7,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -113,23 +112,6 @@ func (ba *metricsBlobAccess) Get(ctx context.Context, digest digest.Digest) buff return b } -func (ba *metricsBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - timeStart := ba.clock.Now() - b := buffer.WithErrorHandler( - ba.blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer), - &metricsErrorHandler{ - blobAccess: ba, - timeStart: timeStart, - errorCode: codes.OK, - durationSeconds: ba.getFromCompositeDurationSeconds, - }, - ) - if sizeBytes, err := b.GetSizeBytes(); err == nil { - ba.getFromCompositeBlobSizeBytes.Observe(float64(sizeBytes)) - } - return b -} - func (ba *metricsBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { // If the Buffer is in a known error state, return the error // here instead of propagating the error to the underlying diff --git a/pkg/blobstore/mirrored/BUILD.bazel b/pkg/blobstore/mirrored/BUILD.bazel index b037f64cc..b9da8b276 100644 --- a/pkg/blobstore/mirrored/BUILD.bazel +++ b/pkg/blobstore/mirrored/BUILD.bazel @@ -9,7 +9,6 @@ go_library( "//pkg/blobstore", "//pkg/blobstore/buffer", "//pkg/blobstore/replication", - "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/util", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/mirrored/mirrored_blob_access.go b/pkg/blobstore/mirrored/mirrored_blob_access.go index 377aa31d7..24a91e5c7 100644 --- a/pkg/blobstore/mirrored/mirrored_blob_access.go +++ b/pkg/blobstore/mirrored/mirrored_blob_access.go @@ -9,7 +9,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/replication" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" "github.com/prometheus/client_golang/prometheus" @@ -107,11 +106,6 @@ func (ba *mirroredBlobAccess) Get(ctx context.Context, digest digest.Digest) buf return replication.GetWithBlobReplicator(ctx, digest, firstBackend, successiveBackends) } -func (ba *mirroredBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - firstBackend, successiveBackends := ba.getBlobReplicatorSelector() - return replication.GetFromCompositeWithBlobReplicator(ctx, parentDigest, childDigest, slicer, firstBackend, successiveBackends) -} - func (ba *mirroredBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { // Store object in both storage backends. b1, b2 := b.CloneStream() diff --git a/pkg/blobstore/mirrored/mirrored_blob_access_test.go b/pkg/blobstore/mirrored/mirrored_blob_access_test.go index db67d60aa..89683b4ed 100644 --- a/pkg/blobstore/mirrored/mirrored_blob_access_test.go +++ b/pkg/blobstore/mirrored/mirrored_blob_access_test.go @@ -88,31 +88,6 @@ func TestMirroredBlobAccessGet(t *testing.T) { }) } -func TestMirroredBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - backendA := mock.NewMockBlobAccess(ctrl) - backendB := mock.NewMockBlobAccess(ctrl) - replicatorAToB := mock.NewMockBlobReplicator(ctrl) - replicatorBToA := mock.NewMockBlobReplicator(ctrl) - parentDigest := digest.MustNewDigest("default", remoteexecution.DigestFunction_SHA256, "834c514174f3a7d5952dfa68d4b657f3c4cf78b3973dcf2721731c3861559828", 100) - childDigest := digest.MustNewDigest("default", remoteexecution.DigestFunction_SHA256, "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c", 11) - slicer := mock.NewMockBlobSlicer(ctrl) - - // We assume that tests for Get() provides coverage for other - // scenarios. - - t.Run("RepairSuccess", func(t *testing.T) { - backendA.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found"))) - replicatorBToA.EXPECT().ReplicateComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - - blobAccess := mirrored.NewMirroredBlobAccess(backendA, backendB, replicatorAToB, replicatorBToA) - data, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - }) -} - func TestMirroredBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/read_canarying_blob_access.go b/pkg/blobstore/read_canarying_blob_access.go index f01361b07..895366d6e 100644 --- a/pkg/blobstore/read_canarying_blob_access.go +++ b/pkg/blobstore/read_canarying_blob_access.go @@ -6,7 +6,6 @@ import ( "time" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/eviction" @@ -139,26 +138,6 @@ func (ba *readCanaryingBlobAccess) Get(ctx context.Context, d digest.Digest) buf ) } -func (ba *readCanaryingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - instanceNameStr := parentDigest.GetInstanceName().String() - if ba.shouldSendToReplica(instanceNameStr) { - return buffer.WithErrorHandler( - ba.replica.GetFromComposite(ctx, parentDigest, childDigest, slicer), - &readCanaryingReplicaGetFromCompositeErrorHandler{ - blobAccess: ba, - context: ctx, - parentDigest: parentDigest, - childDigest: childDigest, - slicer: slicer, - }, - ) - } - return buffer.WithErrorHandler( - ba.BlobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer), - readCanaryingSourceErrorHandler{}, - ) -} - func (ba *readCanaryingBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { // The backend may behave differently based on the REv2 instance // name that is, for example if AuthorizingBlobAccess and @@ -226,39 +205,6 @@ func (eh *readCanaryingReplicaGetErrorHandler) Done() { } } -// readCanaryingReplicaGetFromCompositeErrorHandler is the ErrorHandler -// that is attached to all buffers read from the replica backend through -// the GetFromComposite() operation. -type readCanaryingReplicaGetFromCompositeErrorHandler struct { - blobAccess *readCanaryingBlobAccess - context context.Context - parentDigest digest.Digest - childDigest digest.Digest - slicer slicing.BlobSlicer -} - -func (eh *readCanaryingReplicaGetFromCompositeErrorHandler) OnError(err error) (buffer.Buffer, error) { - ba := eh.blobAccess - if ba == nil { - // Already retried the operation against the source backend. - return nil, util.StatusWrap(err, "Source") - } - eh.blobAccess = nil - if ba.recordReplicaResponse(eh.parentDigest.GetInstanceName().String(), err) { - // Request against the replica failed with an - // infrastructure error. Retry it against the source - // backend. - return ba.BlobAccess.GetFromComposite(eh.context, eh.parentDigest, eh.childDigest, eh.slicer), nil - } - return nil, util.StatusWrap(err, "Replica") -} - -func (eh *readCanaryingReplicaGetFromCompositeErrorHandler) Done() { - if ba := eh.blobAccess; ba != nil { - ba.recordReplicaResponse(eh.parentDigest.GetInstanceName().String(), nil) - } -} - // readCanaryingSourceErrorHandler is the ErrorHandler that is attached // to all buffers read from the source backend. type readCanaryingSourceErrorHandler struct{} diff --git a/pkg/blobstore/read_canarying_blob_access_test.go b/pkg/blobstore/read_canarying_blob_access_test.go index 6e89ac037..9795743e8 100644 --- a/pkg/blobstore/read_canarying_blob_access_test.go +++ b/pkg/blobstore/read_canarying_blob_access_test.go @@ -113,34 +113,6 @@ func TestReadCanaryingBlobAccess(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.Unavailable, "Source: Server is offline"), err) }) - t.Run("GetFromComposite", func(t *testing.T) { - // Don't provide exhaustive testing coverage for - // GetFromComposite(), as most of the logic is shared - // with Get(). Just test that traffic is capable of - // going to both backends. - parentDigest := digest.MustNewDigest("get-from-composite", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 100) - childDigest := digest.MustNewDigest("get-from-composite", remoteexecution.DigestFunction_MD5, "80cd354fb9a929ffad1b059d909b3b69", 10) - slicer := mock.NewMockBlobSlicer(ctrl) - - clock.EXPECT().Now().Return(time.Unix(15000, 0)) - replicaBackend.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.Unavailable, "Server is offline"))) - replicaErrorLogger.EXPECT().Log(status.Error(codes.Unavailable, "Server is offline")) - clock.EXPECT().Now().Return(time.Unix(15000, 100000000)) - sourceBackend.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.Unavailable, "Server is also offline"))) - - _, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.Unavailable, "Source: Server is also offline"), err) - - clock.EXPECT().Now().Return(time.Unix(15001, 0)) - sourceBackend.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.Unavailable, "Server is offline"))) - - _, err = blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - testutil.RequireEqualStatus(t, status.Error(codes.Unavailable, "Source: Server is offline"), err) - }) - t.Run("FindMissing", func(t *testing.T) { // Let a FindMissing() call against the replica fail for a // given REv2 instance name. This should cause it to be diff --git a/pkg/blobstore/readcaching/BUILD.bazel b/pkg/blobstore/readcaching/BUILD.bazel index be443ce6c..b9f07c688 100644 --- a/pkg/blobstore/readcaching/BUILD.bazel +++ b/pkg/blobstore/readcaching/BUILD.bazel @@ -9,7 +9,6 @@ go_library( "//pkg/blobstore", "//pkg/blobstore/buffer", "//pkg/blobstore/replication", - "//pkg/blobstore/slicing", "//pkg/digest", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", diff --git a/pkg/blobstore/readcaching/read_caching_blob_access.go b/pkg/blobstore/readcaching/read_caching_blob_access.go index 7f4ff14bb..1be201e05 100644 --- a/pkg/blobstore/readcaching/read_caching_blob_access.go +++ b/pkg/blobstore/readcaching/read_caching_blob_access.go @@ -6,7 +6,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/replication" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "google.golang.org/grpc/codes" @@ -52,14 +51,3 @@ func (ba *readCachingBlobAccess) Get(ctx context.Context, digest digest.Digest) ba.getBlobReplicatorSelector(), ) } - -func (ba *readCachingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return replication.GetFromCompositeWithBlobReplicator( - ctx, - parentDigest, - childDigest, - slicer, - ba.fast, - ba.getBlobReplicatorSelector(), - ) -} diff --git a/pkg/blobstore/readcaching/read_caching_blob_access_test.go b/pkg/blobstore/readcaching/read_caching_blob_access_test.go index 17f1e2c2c..bd01fd879 100644 --- a/pkg/blobstore/readcaching/read_caching_blob_access_test.go +++ b/pkg/blobstore/readcaching/read_caching_blob_access_test.go @@ -66,30 +66,6 @@ func TestReadCachingBlobAccessGet(t *testing.T) { }) } -func TestReadCachingBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - slowBlobAccess := mock.NewMockBlobAccess(ctrl) - fastBlobAccess := mock.NewMockBlobAccess(ctrl) - blobReplicator := mock.NewMockBlobReplicator(ctrl) - blobAccess := readcaching.NewReadCachingBlobAccess(slowBlobAccess, fastBlobAccess, blobReplicator) - parentDigest := digest.MustNewDigest("default", remoteexecution.DigestFunction_SHA256, "834c514174f3a7d5952dfa68d4b657f3c4cf78b3973dcf2721731c3861559828", 100) - childDigest := digest.MustNewDigest("default", remoteexecution.DigestFunction_SHA256, "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c", 11) - slicer := mock.NewMockBlobSlicer(ctrl) - - // We assume that tests for Get() provides coverage for other - // scenarios. - - t.Run("Slow", func(t *testing.T) { - fastBlobAccess.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Blob not found"))) - blobReplicator.EXPECT().ReplicateComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello world"))) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("Hello world"), data) - }) -} - func TestReadCachingBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/readfallback/BUILD.bazel b/pkg/blobstore/readfallback/BUILD.bazel index 3a111a54d..7ded42327 100644 --- a/pkg/blobstore/readfallback/BUILD.bazel +++ b/pkg/blobstore/readfallback/BUILD.bazel @@ -9,7 +9,6 @@ go_library( "//pkg/blobstore", "//pkg/blobstore/buffer", "//pkg/blobstore/replication", - "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/util", "@org_golang_google_grpc//codes", diff --git a/pkg/blobstore/readfallback/read_fallback_blob_access.go b/pkg/blobstore/readfallback/read_fallback_blob_access.go index 39f4bac89..ff81f793f 100644 --- a/pkg/blobstore/readfallback/read_fallback_blob_access.go +++ b/pkg/blobstore/readfallback/read_fallback_blob_access.go @@ -6,7 +6,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" "github.com/buildbarn/bb-storage/pkg/blobstore/replication" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -67,17 +66,6 @@ func (ba *readFallbackBlobAccess) Get(ctx context.Context, digest digest.Digest) ) } -func (ba *readFallbackBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return replication.GetFromCompositeWithBlobReplicator( - ctx, - parentDigest, - childDigest, - slicer, - ba.BlobAccess, - ba.getBlobReplicatorSelector(), - ) -} - func (ba *readFallbackBlobAccess) FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) { // Call FindMissing() on the backends sequentially, as opposed // to calling them concurrently and merging the results. In the diff --git a/pkg/blobstore/readfallback/read_fallback_blob_access_test.go b/pkg/blobstore/readfallback/read_fallback_blob_access_test.go index 63717a51c..4e7642197 100644 --- a/pkg/blobstore/readfallback/read_fallback_blob_access_test.go +++ b/pkg/blobstore/readfallback/read_fallback_blob_access_test.go @@ -88,32 +88,6 @@ func TestReadFallbackBlobAccessGet(t *testing.T) { }) } -func TestReadFallbackBlobAccessGetFromComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - primary := mock.NewMockBlobAccess(ctrl) - secondary := mock.NewMockBlobAccess(ctrl) - replicator := mock.NewMockBlobReplicator(ctrl) - blobAccess := readfallback.NewReadFallbackBlobAccess(primary, secondary, replicator) - parentDigest := digest.MustNewDigest("instance", remoteexecution.DigestFunction_MD5, "d20fb8dfa347cf895b38649410aeb3f8", 100) - childDigest := digest.MustNewDigest("instance", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - slicer := mock.NewMockBlobSlicer(ctrl) - - // We assume that tests for Get() provides coverage for other - // scenarios. - - t.Run("SecondarySuccess", func(t *testing.T) { - primary.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewBufferFromError(status.Error(codes.NotFound, "Object not found"))) - replicator.EXPECT().ReplicateComposite(ctx, parentDigest, childDigest, slicer). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) - - data, err := blobAccess.GetFromComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(100) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) -} - func TestReadFallbackBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/referenceexpanding/BUILD.bazel b/pkg/blobstore/referenceexpanding/BUILD.bazel new file mode 100644 index 000000000..3fe307b6b --- /dev/null +++ b/pkg/blobstore/referenceexpanding/BUILD.bazel @@ -0,0 +1,49 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "referenceexpanding", + srcs = ["reference_expanding_blob_access.go"], + importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/referenceexpanding", + visibility = ["//visibility:public"], + deps = [ + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/slicing", + "//pkg/cloud/aws", + "//pkg/cloud/gcp", + "//pkg/digest", + "//pkg/proto/icas", + "//pkg/util", + "//pkg/zstd", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_aws_aws_sdk_go_v2//aws", + "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + ], +) + +go_test( + name = "referenceexpanding_test", + srcs = ["reference_expanding_blob_access_test.go"], + deps = [ + ":referenceexpanding", + "//internal/mock", + "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/digest", + "//pkg/proto/icas", + "//pkg/testutil", + "//pkg/zstd", + "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", + "@com_github_aws_aws_sdk_go_v2//aws", + "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", + "@com_github_aws_aws_sdk_go_v2_service_s3//types", + "@com_github_klauspost_compress//zstd", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + "@org_uber_go_mock//gomock", + ], +) diff --git a/pkg/blobstore/reference_expanding_blob_access.go b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go similarity index 91% rename from pkg/blobstore/reference_expanding_blob_access.go rename to pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go index 340073a6b..2ecdb2026 100644 --- a/pkg/blobstore/reference_expanding_blob_access.go +++ b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go @@ -1,4 +1,4 @@ -package blobstore +package referenceexpanding import ( "compress/flate" @@ -11,7 +11,9 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" + "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" cloud_aws "github.com/buildbarn/bb-storage/pkg/cloud/aws" cloud_gcp "github.com/buildbarn/bb-storage/pkg/cloud/gcp" @@ -25,8 +27,8 @@ import ( ) type referenceExpandingBlobAccess struct { - indirectContentAddressableStorage BlobAccess - contentAddressableStorage BlobAccess + indirectContentAddressableStorage blobstore.BlobAccess + contentAddressableStorage cdc.ContentAddressableStorage httpClient *http.Client s3Client cloud_aws.S3Client gcsClient cloud_gcp.StorageClient @@ -48,7 +50,7 @@ func getHTTPRangeHeader(reference *icas.Reference) string { // Storage (CAS) backend. Any object requested through this BlobAccess // will cause its reference to be loaded from the ICAS, followed by // fetching its data from the referenced location. -func NewReferenceExpandingBlobAccess(indirectContentAddressableStorage, contentAddressableStorage BlobAccess, httpClient *http.Client, s3Client cloud_aws.S3Client, gcsClient cloud_gcp.StorageClient, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) BlobAccess { +func NewReferenceExpandingBlobAccess(indirectContentAddressableStorage blobstore.BlobAccess, contentAddressableStorage cdc.ContentAddressableStorage, httpClient *http.Client, s3Client cloud_aws.S3Client, gcsClient cloud_gcp.StorageClient, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) blobstore.BlobAccess { return &referenceExpandingBlobAccess{ indirectContentAddressableStorage: indirectContentAddressableStorage, contentAddressableStorage: contentAddressableStorage, @@ -135,17 +137,10 @@ func (ba *referenceExpandingBlobAccess) Get(ctx context.Context, blobDigest dige return buffer.NewBufferFromError(util.StatusWrapWithCode(err, codes.Internal, "Invalid digest")) } - b := ba.contentAddressableStorage.Get(ctx, referenceDigest) - if reference.Decompressor == remoteexecution.Compressor_IDENTITY { - // Optimize the fast path: if no transformations are - // being performed and the digests are identical, we - // can pass through the underlying buffer directly. - instanceNamePatcher := digest.NewInstanceNamePatcher(referenceDigest.GetInstanceName(), blobDigest.GetInstanceName()) - if blobDigest == instanceNamePatcher.PatchDigest(referenceDigest) { - return b - } + r, err = cdc.GetReadCloser(ctx, ba.contentAddressableStorage, referenceDigest) + if err != nil { + return buffer.NewBufferFromError(err) } - r = b.ToReader() default: return buffer.NewBufferFromError(status.Error(codes.Unimplemented, "Reference uses an unsupported medium")) } diff --git a/pkg/blobstore/reference_expanding_blob_access_test.go b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access_test.go similarity index 93% rename from pkg/blobstore/reference_expanding_blob_access_test.go rename to pkg/blobstore/referenceexpanding/reference_expanding_blob_access_test.go index 7728da6a5..11cc47888 100644 --- a/pkg/blobstore/reference_expanding_blob_access_test.go +++ b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access_test.go @@ -1,4 +1,4 @@ -package blobstore_test +package referenceexpanding_test import ( "bytes" @@ -13,8 +13,9 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/internal/mock" - "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/blobstore/referenceexpanding" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/icas" "github.com/buildbarn/bb-storage/pkg/testutil" @@ -32,11 +33,11 @@ func TestReferenceExpandingBlobAccessGet(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) indirectContentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) roundTripper := mock.NewMockRoundTripper(ctrl) s3Client := mock.NewMockS3Client(ctrl) gcsClient := mock.NewMockStorageClient(ctrl) - blobAccess := blobstore.NewReferenceExpandingBlobAccess( + blobAccess := referenceexpanding.NewReferenceExpandingBlobAccess( indirectContentAddressableStorage, contentAddressableStorage, &http.Client{Transport: roundTripper}, @@ -467,10 +468,14 @@ func TestReferenceExpandingBlobAccessGet(t *testing.T) { buffer.BackendProvided(buffer.Irreparable(helloDigest)), ), ) - contentAddressableStorage.EXPECT().Get( + contentAddressableStorage.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("instance/name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil) + contentAddressableStorage.EXPECT().FetchChunk( ctx, digest.MustNewDigest("instance/name", remoteexecution.DigestFunction_SHA256, "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969", 5), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) + ).Return([]byte("Hello"), nil) data, err := blobAccess.Get(ctx, helloDigest).ToByteSlice(10) require.NoError(t, err) @@ -497,16 +502,21 @@ func TestReferenceExpandingBlobAccessGet(t *testing.T) { buffer.BackendProvided(buffer.Irreparable(aaaDigest)), ), ) - contentAddressableStorage.EXPECT().Get( - ctx, - digest.MustNewDigest("instance/name", remoteexecution.DigestFunction_SHA1, "a6cf72f4c7f42afde230ac461d5c7b9e25838530", 21), - ).Return(buffer.NewValidatedBufferFromByteSlice([]byte{ + data := []byte{ // 21 bytes of Zstandard compressed data // that decompress to fifty 'a' bytes. 0x28, 0xb5, 0x2f, 0xfd, 0x04, 0x58, 0x45, 0x00, 0x00, 0x10, 0x61, 0x61, 0x01, 0x00, 0x45, 0x00, 0x0b, 0x23, 0x9f, 0x0f, 0x9a, - })) + } + contentAddressableStorage.EXPECT().FetchCDCParameters( + gomock.Any(), + mustNewInstanceName("instance/name"), + ).Return(cdc.Parameters{MinChunkSizeBytes: 1 << 20, HorizonSizeBytes: 2 << 20}, nil) + contentAddressableStorage.EXPECT().FetchChunk( + ctx, + digest.MustNewDigest("instance/name", remoteexecution.DigestFunction_SHA1, "a6cf72f4c7f42afde230ac461d5c7b9e25838530", 21), + ).Return(data, nil) data, err := blobAccess.Get(ctx, aaaDigest).ToByteSlice(100) require.NoError(t, err) @@ -518,11 +528,11 @@ func TestReferenceExpandingBlobAccessPut(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) indirectContentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) roundTripper := mock.NewMockRoundTripper(ctrl) s3Client := mock.NewMockS3Client(ctrl) gcsClient := mock.NewMockStorageClient(ctrl) - blobAccess := blobstore.NewReferenceExpandingBlobAccess( + blobAccess := referenceexpanding.NewReferenceExpandingBlobAccess( indirectContentAddressableStorage, contentAddressableStorage, &http.Client{Transport: roundTripper}, @@ -557,11 +567,11 @@ func TestReferenceExpandingBlobAccessFindMissing(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) indirectContentAddressableStorage := mock.NewMockBlobAccess(ctrl) - contentAddressableStorage := mock.NewMockBlobAccess(ctrl) + contentAddressableStorage := mock.NewMockContentAddressableStorage(ctrl) roundTripper := mock.NewMockRoundTripper(ctrl) s3Client := mock.NewMockS3Client(ctrl) gcsClient := mock.NewMockStorageClient(ctrl) - blobAccess := blobstore.NewReferenceExpandingBlobAccess( + blobAccess := referenceexpanding.NewReferenceExpandingBlobAccess( indirectContentAddressableStorage, contentAddressableStorage, &http.Client{Transport: roundTripper}, @@ -598,3 +608,11 @@ func TestReferenceExpandingBlobAccessFindMissing(t *testing.T) { testutil.RequireEqualStatus(t, status.Error(codes.Internal, "Network error"), err) }) } + +func mustNewInstanceName(name string) digest.InstanceName { + instanceName, err := digest.NewInstanceName(name) + if err != nil { + panic(err) + } + return instanceName +} diff --git a/pkg/blobstore/replication/BUILD.bazel b/pkg/blobstore/replication/BUILD.bazel index 6c362b07b..3020382d6 100644 --- a/pkg/blobstore/replication/BUILD.bazel +++ b/pkg/blobstore/replication/BUILD.bazel @@ -20,7 +20,6 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/slicing", "//pkg/clock", "//pkg/digest", "//pkg/proto/replicator", diff --git a/pkg/blobstore/replication/blob_replicator.go b/pkg/blobstore/replication/blob_replicator.go index e302be13e..950f8d03f 100644 --- a/pkg/blobstore/replication/blob_replicator.go +++ b/pkg/blobstore/replication/blob_replicator.go @@ -4,7 +4,6 @@ import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -20,9 +19,6 @@ type BlobReplicator interface { // Replicate a single object between backends, while at the same // time giving a handle back to it. ReplicateSingle(ctx context.Context, digest digest.Digest) buffer.Buffer - // Replicate a single composite object between backends, while - // at the same time giving a handle back to one of its children. - ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer // Replicate a set of objects between backends. ReplicateMultiple(ctx context.Context, digests digest.Set) error } diff --git a/pkg/blobstore/replication/concurrency_limiting_blob_replicator.go b/pkg/blobstore/replication/concurrency_limiting_blob_replicator.go index ce8bfdf96..096da5bc3 100644 --- a/pkg/blobstore/replication/concurrency_limiting_blob_replicator.go +++ b/pkg/blobstore/replication/concurrency_limiting_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -47,16 +46,6 @@ func (br *concurrencyLimitingBlobReplicator) ReplicateSingle(ctx context.Context ) } -func (br *concurrencyLimitingBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - if err := br.ReplicateMultiple(ctx, parentDigest.ToSingletonSet()); err != nil { - return buffer.NewBufferFromError(err) - } - return buffer.WithErrorHandler( - br.sink.GetFromComposite(ctx, parentDigest, childDigest, slicer), - notFoundToInternalErrorHandler{}, - ) -} - func (br *concurrencyLimitingBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { if err := util.AcquireSemaphore(ctx, br.semaphore, 1); err != nil { return err diff --git a/pkg/blobstore/replication/deduplicating_blob_replicator.go b/pkg/blobstore/replication/deduplicating_blob_replicator.go index bd219c114..cdd41b9cc 100644 --- a/pkg/blobstore/replication/deduplicating_blob_replicator.go +++ b/pkg/blobstore/replication/deduplicating_blob_replicator.go @@ -6,7 +6,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -64,16 +63,6 @@ func (br *deduplicatingBlobReplicator) ReplicateSingle(ctx context.Context, blob ) } -func (br *deduplicatingBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - if err := br.ReplicateMultiple(ctx, parentDigest.ToSingletonSet()); err != nil { - return buffer.NewBufferFromError(err) - } - return buffer.WithErrorHandler( - br.sink.GetFromComposite(ctx, parentDigest, childDigest, slicer), - notFoundToInternalErrorHandler{}, - ) -} - func (br *deduplicatingBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { NextDigest: for _, digest := range digests.Items() { diff --git a/pkg/blobstore/replication/deduplicating_blob_replicator_test.go b/pkg/blobstore/replication/deduplicating_blob_replicator_test.go index ca747d602..99806e2d5 100644 --- a/pkg/blobstore/replication/deduplicating_blob_replicator_test.go +++ b/pkg/blobstore/replication/deduplicating_blob_replicator_test.go @@ -146,35 +146,6 @@ func TestDeduplicatingBlobReplicatorReplicateSingle(t *testing.T) { }) } -func TestDeduplicatingBlobReplicatorReplicateComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - base := mock.NewMockBlobReplicator(ctrl) - sink := mock.NewMockBlobAccess(ctrl) - replicator := replication.NewDeduplicatingBlobReplicator(base, sink, digest.KeyWithoutInstance) - - parentDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "3e25960a79dbc69b674cd4ec67a72c62", 11) - parentDigestSet := parentDigest.ToSingletonSet() - childDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - slicer := mock.NewMockBlobSlicer(ctrl) - - // Only a single test for the success case is provided, as the - // tests for ReplicateSingle() provide enough coverage. - - t.Run("SuccessReplication", func(t *testing.T) { - // If the sink reports the parent as absent, we should - // see a replication take place before reading the child - // blob from the sink. - sink.EXPECT().FindMissing(ctx, parentDigestSet).Return(parentDigestSet, nil) - base.EXPECT().ReplicateMultiple(ctx, parentDigestSet) - sink.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return(buffer.NewValidatedBufferFromByteSlice([]byte("Hello"))) - - data, err := replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer).ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) -} - func TestDeduplicatingBlobReplicatorReplicateMultiple(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/replication/local_blob_replicator.go b/pkg/blobstore/replication/local_blob_replicator.go index c7611b6df..7d33b7827 100644 --- a/pkg/blobstore/replication/local_blob_replicator.go +++ b/pkg/blobstore/replication/local_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -42,20 +41,6 @@ func (br *localBlobReplicator) ReplicateSingle(ctx context.Context, digest diges }) } -func (br *localBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - // First replicate the object to the sink, and then read it back - // again. Even though cloning like in ReplicateSingle() is more - // optimal in terms of bandwidth, it would require us to do the - // slicing here. This may cause slicing information to get lost. - if err := br.ReplicateMultiple(ctx, parentDigest.ToSingletonSet()); err != nil { - return buffer.NewBufferFromError(err) - } - return buffer.WithErrorHandler( - br.sink.GetFromComposite(ctx, parentDigest, childDigest, slicer), - notFoundToInternalErrorHandler{}, - ) -} - func (br *localBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { for _, blobDigest := range digests.Items() { if err := br.sink.Put(ctx, blobDigest, br.source.Get(ctx, blobDigest)); err != nil { diff --git a/pkg/blobstore/replication/metrics_blob_replicator.go b/pkg/blobstore/replication/metrics_blob_replicator.go index bd6a9fb27..1f5bac952 100644 --- a/pkg/blobstore/replication/metrics_blob_replicator.go +++ b/pkg/blobstore/replication/metrics_blob_replicator.go @@ -6,7 +6,6 @@ import ( "time" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -120,23 +119,6 @@ func (r *metricsBlobReplicator) ReplicateSingle(ctx context.Context, blobDigest return b } -func (r *metricsBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - timeStart := r.clock.Now() - b := buffer.WithErrorHandler( - r.replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer), - &metricsErrorHandler{ - replicator: r, - timeStart: timeStart, - errorCode: codes.OK, - durationSeconds: r.compositeDurationSeconds, - }, - ) - if sizeBytes, err := b.GetSizeBytes(); err == nil { - r.compositeBlobSizeBytes.Observe(float64(sizeBytes)) - } - return b -} - func (r *metricsBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { if digests.Empty() { return nil diff --git a/pkg/blobstore/replication/noop_blob_replicator.go b/pkg/blobstore/replication/noop_blob_replicator.go index 5e5951da1..11f2df240 100644 --- a/pkg/blobstore/replication/noop_blob_replicator.go +++ b/pkg/blobstore/replication/noop_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -27,10 +26,6 @@ func (br noopBlobReplicator) ReplicateSingle(ctx context.Context, digest digest. return br.source.Get(ctx, digest) } -func (br noopBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return br.source.GetFromComposite(ctx, parentDigest, childDigest, slicer) -} - func (noopBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { return nil } diff --git a/pkg/blobstore/replication/queued_blob_replicator.go b/pkg/blobstore/replication/queued_blob_replicator.go index 4b9431a5b..0e099a67b 100644 --- a/pkg/blobstore/replication/queued_blob_replicator.go +++ b/pkg/blobstore/replication/queued_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" ) @@ -51,15 +50,6 @@ func (br *queuedBlobReplicator) ReplicateSingle(ctx context.Context, blobDigest }) } -func (br *queuedBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return br.source.GetFromComposite(ctx, parentDigest, childDigest, slicer).WithTask(func() error { - if err := br.ReplicateMultiple(ctx, parentDigest.ToSingletonSet()); err != nil { - return util.StatusWrap(err, "Replication failed") - } - return nil - }) -} - func (br *queuedBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { // Don't queue requests for objects that have already been // replicated. diff --git a/pkg/blobstore/replication/queued_blob_replicator_test.go b/pkg/blobstore/replication/queued_blob_replicator_test.go index 0594edfe3..49dc4cb7d 100644 --- a/pkg/blobstore/replication/queued_blob_replicator_test.go +++ b/pkg/blobstore/replication/queued_blob_replicator_test.go @@ -127,68 +127,6 @@ func TestQueuedBlobReplicatorReplicateSingle(t *testing.T) { }) } -func TestQueuedBlobReplicatorReplicateComposite(t *testing.T) { - ctrl, ctx := gomock.WithContext(context.Background(), t) - - source := mock.NewMockBlobAccess(ctrl) - baseReplicator := mock.NewMockBlobReplicator(ctrl) - clock := mock.NewMockClock(ctrl) - replicator := replication.NewQueuedBlobReplicator( - source, - baseReplicator, - digest.NewExistenceCache(clock, digest.KeyWithoutInstance, 10, time.Minute, eviction.NewLRUSet[string]()), - ) - - parentDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "3e25960a79dbc69b674cd4ec67a72c62", 11) - parentDigests := parentDigest.ToSingletonSet() - childDigest := digest.MustNewDigest("hello", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - slicer := mock.NewMockBlobSlicer(ctrl) - - // Only a single test for the success case is provided, as the - // tests for ReplicateSingle() provide enough coverage. - - t.Run("Success", func(t *testing.T) { - // The first time the object is requested, it should be - // replicated in the background. - source.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), - ) - clock.EXPECT().Now().Return(time.Unix(1000, 0)).Times(3) - baseReplicator.EXPECT().ReplicateMultiple(ctx, parentDigests).Return(nil) - - b := replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer) - data, err := b.ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - - // The fact that the object has been replicated should - // be cached. We should only perform the load during the - // second attempt. - source.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), - ) - clock.EXPECT().Now().Return(time.Unix(1060, 0)) - - b = replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer) - data, err = b.ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - - // A third request after the cache entry has expired - // should trigger a replication once again. - source.EXPECT().GetFromComposite(ctx, parentDigest, childDigest, slicer).Return( - buffer.NewValidatedBufferFromByteSlice([]byte("Hello")), - ) - clock.EXPECT().Now().Return(time.Unix(1060, 1)).Times(3) - baseReplicator.EXPECT().ReplicateMultiple(ctx, parentDigests).Return(nil) - - b = replicator.ReplicateComposite(ctx, parentDigest, childDigest, slicer) - data, err = b.ToByteSlice(10) - require.NoError(t, err) - require.Equal(t, []byte("Hello"), data) - }) -} - func TestQueuedBlobReplicatorReplicateMultiple(t *testing.T) { ctrl, ctx := gomock.WithContext(context.Background(), t) diff --git a/pkg/blobstore/replication/remote_blob_replicator.go b/pkg/blobstore/replication/remote_blob_replicator.go index 7ce854e9a..c935cd716 100644 --- a/pkg/blobstore/replication/remote_blob_replicator.go +++ b/pkg/blobstore/replication/remote_blob_replicator.go @@ -6,7 +6,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/proto/replicator" @@ -44,20 +43,6 @@ func (br *remoteBlobReplicator) ReplicateSingle(ctx context.Context, digest dige }) } -func (br *remoteBlobReplicator) ReplicateComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - return br.source.GetFromComposite(ctx, parentDigest, childDigest, slicer).WithTask(func() error { - digestFunction := parentDigest.GetDigestFunction() - _, err := br.replicatorClient.ReplicateBlobs(ctx, &replicator.ReplicateBlobsRequest{ - InstanceName: digestFunction.GetInstanceName().String(), - DigestFunction: digestFunction.GetEnumValue(), - BlobDigests: []*remoteexecution.Digest{ - parentDigest.GetProto(), - }, - }) - return err - }) -} - func (br *remoteBlobReplicator) ReplicateMultiple(ctx context.Context, digests digest.Set) error { // Partition all digests by digest function, as the // ReplicateBlobs() RPC can only process digests for a single diff --git a/pkg/blobstore/replication/with_blob_replicator.go b/pkg/blobstore/replication/with_blob_replicator.go index c92e72268..23365c117 100644 --- a/pkg/blobstore/replication/with_blob_replicator.go +++ b/pkg/blobstore/replication/with_blob_replicator.go @@ -5,7 +5,6 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" ) @@ -45,37 +44,3 @@ func (eh *getReplicatingErrorHandler) OnError(observedErr error) (buffer.Buffer, } func (getReplicatingErrorHandler) Done() {} - -// GetFromCompositeWithBlobReplicator is equivalent to -// GetWithBlobReplicator, except that it's a common implementation of -// BlobAccess.GetFromComposite(). -func GetFromCompositeWithBlobReplicator(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer, initialBackend blobstore.BlobAccess, successiveBackends BlobReplicatorSelector) buffer.Buffer { - return buffer.WithErrorHandler( - initialBackend.GetFromComposite(ctx, parentDigest, childDigest, slicer), - &getFromCompositeReplicatingErrorHandler{ - selector: successiveBackends, - context: ctx, - parentDigest: parentDigest, - childDigest: childDigest, - slicer: slicer, - }, - ) -} - -type getFromCompositeReplicatingErrorHandler struct { - selector BlobReplicatorSelector - context context.Context - parentDigest digest.Digest - childDigest digest.Digest - slicer slicing.BlobSlicer -} - -func (eh *getFromCompositeReplicatingErrorHandler) OnError(observedErr error) (buffer.Buffer, error) { - replicator, err := eh.selector(observedErr) - if err != nil { - return nil, err - } - return replicator.ReplicateComposite(eh.context, eh.parentDigest, eh.childDigest, eh.slicer), nil -} - -func (getFromCompositeReplicatingErrorHandler) Done() {} diff --git a/pkg/blobstore/sharding/BUILD.bazel b/pkg/blobstore/sharding/BUILD.bazel index c5794b2ad..4db190e21 100644 --- a/pkg/blobstore/sharding/BUILD.bazel +++ b/pkg/blobstore/sharding/BUILD.bazel @@ -12,7 +12,6 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/slicing", "//pkg/digest", "//pkg/util", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/sharding/sharding_blob_access.go b/pkg/blobstore/sharding/sharding_blob_access.go index 7d22974ec..62e73bc0b 100644 --- a/pkg/blobstore/sharding/sharding_blob_access.go +++ b/pkg/blobstore/sharding/sharding_blob_access.go @@ -8,7 +8,6 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -53,14 +52,6 @@ func (ba *shardingBlobAccess) Get(ctx context.Context, digest digest.Digest) buf ) } -func (ba *shardingBlobAccess) GetFromComposite(ctx context.Context, parentDigest, childDigest digest.Digest, slicer slicing.BlobSlicer) buffer.Buffer { - index := ba.getBackendIndexByDigest(parentDigest) - return buffer.WithErrorHandler( - ba.backends[index].Backend.GetFromComposite(ctx, parentDigest, childDigest, slicer), - shardKeyAddingErrorHandler{key: ba.backends[index].Key}, - ) -} - func (ba *shardingBlobAccess) Put(ctx context.Context, digest digest.Digest, b buffer.Buffer) error { index := ba.getBackendIndexByDigest(digest) if err := ba.backends[index].Backend.Put(ctx, digest, b); err != nil { diff --git a/pkg/blobstore/sharding/sharding_blob_access_test.go b/pkg/blobstore/sharding/sharding_blob_access_test.go index 62d3d7a0c..a20195e0d 100644 --- a/pkg/blobstore/sharding/sharding_blob_access_test.go +++ b/pkg/blobstore/sharding/sharding_blob_access_test.go @@ -40,7 +40,6 @@ func TestShardingBlobAccess(t *testing.T) { ) helloDigest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "8b1a9953c4611296a827abf8c47804d7", 5) - llDigest := digest.MustNewDigest("example", remoteexecution.DigestFunction_MD5, "5b54c0a045f179bcbbbc9abcb8b5cd4c", 2) t.Run("GetFailure", func(t *testing.T) { // Errors should be prefixed with the shard key. @@ -62,20 +61,6 @@ func TestShardingBlobAccess(t *testing.T) { require.Equal(t, []byte("Hello"), data) }) - t.Run("GetFromCompositeSuccess", func(t *testing.T) { - // For reads from composite objects, the sharding needs - // to be based on the parent digest. That digest was - // used to upload the object to storage. - shardSelector.EXPECT().GetShard(uint64(0x8b1a9953c4611296)).Return(0) - slicer := mock.NewMockBlobSlicer(ctrl) - shard0.EXPECT().GetFromComposite(ctx, helloDigest, llDigest, slicer). - Return(buffer.NewValidatedBufferFromByteSlice([]byte("ll"))) - - data, err := blobAccess.GetFromComposite(ctx, helloDigest, llDigest, slicer).ToByteSlice(1000) - require.NoError(t, err) - require.Equal(t, []byte("ll"), data) - }) - t.Run("PutFailure", func(t *testing.T) { // Errors should be prefixed with a shard key. shardSelector.EXPECT().GetShard(uint64(0x8b1a9953c4611296)).Return(1) diff --git a/pkg/digest/function.go b/pkg/digest/function.go index 5f3149864..29c25bf07 100644 --- a/pkg/digest/function.go +++ b/pkg/digest/function.go @@ -113,6 +113,9 @@ func (f Function) NewDigestFromProto(digest *remoteexecution.Digest) (Digest, er if digest == nil { return BadDigest, status.Error(codes.InvalidArgument, "No digest provided") } + if digest.SizeBytes < 0 { + return BadDigest, status.Error(codes.InvalidArgument, "Negative size digest provided") + } return f.NewDigest(digest.Hash, digest.SizeBytes) } diff --git a/pkg/proto/blobstore/chunklist/BUILD.bazel b/pkg/proto/blobstore/chunklist/BUILD.bazel new file mode 100644 index 000000000..9b741fb25 --- /dev/null +++ b/pkg/proto/blobstore/chunklist/BUILD.bazel @@ -0,0 +1,24 @@ +load("@protobuf//bazel:proto_library.bzl", "proto_library") +load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//proto:def.bzl", "go_proto_library") + +proto_library( + name = "chunklist_proto", + srcs = ["chunk_list.proto"], + import_prefix = "github.com/buildbarn/bb-storage", + visibility = ["//visibility:public"], +) + +go_proto_library( + name = "chunklist_go_proto", + importpath = "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist", + proto = ":chunklist_proto", + visibility = ["//visibility:public"], +) + +go_library( + name = "chunklist", + embed = [":chunklist_go_proto"], + importpath = "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist", + visibility = ["//visibility:public"], +) diff --git a/pkg/proto/blobstore/chunklist/chunk_list.pb.go b/pkg/proto/blobstore/chunklist/chunk_list.pb.go new file mode 100644 index 000000000..5835010a9 --- /dev/null +++ b/pkg/proto/blobstore/chunklist/chunk_list.pb.go @@ -0,0 +1,124 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11-devel +// protoc v7.35.0 +// source: github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist/chunk_list.proto + +package chunklist + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ChunkList struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ChunkList) Reset() { + *x = ChunkList{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChunkList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkList) ProtoMessage() {} + +func (x *ChunkList) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkList.ProtoReflect.Descriptor instead. +func (*ChunkList) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescGZIP(), []int{0} +} + +func (x *ChunkList) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto protoreflect.FileDescriptor + +const file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc = "" + + "\n" + + "Ngithub.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist/chunk_list.proto\x12\x1dbuildbarn.blobstore.chunklist\"\x1f\n" + + "\tChunkList\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04dataB?Z=github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklistb\x06proto3" + +var ( + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescOnce sync.Once + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescData []byte +) + +func file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescGZIP() []byte { + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescOnce.Do(func() { + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc))) + }) + return file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDescData +} + +var file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_goTypes = []any{ + (*ChunkList)(nil), // 0: buildbarn.blobstore.chunklist.ChunkList +} +var file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_init() +} +func file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_init() { + if File_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_goTypes, + DependencyIndexes: file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_depIdxs, + MessageInfos: file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_msgTypes, + }.Build() + File_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto = out.File + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_goTypes = nil + file_github_com_buildbarn_bb_storage_pkg_proto_blobstore_chunklist_chunk_list_proto_depIdxs = nil +} diff --git a/pkg/proto/blobstore/chunklist/chunk_list.proto b/pkg/proto/blobstore/chunklist/chunk_list.proto new file mode 100644 index 000000000..7b8c6fdb4 --- /dev/null +++ b/pkg/proto/blobstore/chunklist/chunk_list.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package buildbarn.blobstore.chunklist; + +option go_package = "github.com/buildbarn/bb-storage/pkg/proto/blobstore/chunklist"; + +// ChunkList is the internal storage format for chunk lists in the +// Chunk List Storage (CLS). The data field contains a sequence of +// compact binary digest encodings as produced by +// digest.Digest.GetCompactBinary(). +message ChunkList { + bytes data = 1; +} diff --git a/pkg/proto/configuration/bb_replicator/bb_replicator.proto b/pkg/proto/configuration/bb_replicator/bb_replicator.proto index 363383881..cda2231e9 100644 --- a/pkg/proto/configuration/bb_replicator/bb_replicator.proto +++ b/pkg/proto/configuration/bb_replicator/bb_replicator.proto @@ -16,10 +16,10 @@ message ApplicationConfiguration { // gRPC servers to spawn to listen for requests from clients. repeated buildbarn.configuration.grpc.ServerConfiguration grpc_servers = 2; - // Content Addressable Storage where data needs to be read. + // Chunk Storage where data needs to be read. buildbarn.configuration.blobstore.BlobAccessConfiguration source = 3; - // Content Addressable Storage where data needs to be written. + // Chunk Storage where data needs to be written. buildbarn.configuration.blobstore.BlobAccessConfiguration sink = 4; // Configuration for replication. diff --git a/pkg/proto/configuration/bb_storage/bb_storage.pb.go b/pkg/proto/configuration/bb_storage/bb_storage.pb.go index fb4668e07..c45e752eb 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.pb.go +++ b/pkg/proto/configuration/bb_storage/bb_storage.pb.go @@ -29,20 +29,19 @@ const ( ) type ApplicationConfiguration struct { - state protoimpl.MessageState `protogen:"open.v1"` - GrpcServers []*grpc.ServerConfiguration `protobuf:"bytes,4,rep,name=grpc_servers,json=grpcServers,proto3" json:"grpc_servers,omitempty"` - Schedulers map[string]*builder.SchedulerConfiguration `protobuf:"bytes,5,rep,name=schedulers,proto3" json:"schedulers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - MaximumMessageSizeBytes int64 `protobuf:"varint,8,opt,name=maximum_message_size_bytes,json=maximumMessageSizeBytes,proto3" json:"maximum_message_size_bytes,omitempty"` - Global *global.Configuration `protobuf:"bytes,9,opt,name=global,proto3" json:"global,omitempty"` - ContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,17,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` - ChunkListStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,22,opt,name=chunk_list_storage,json=chunkListStorage,proto3" json:"chunk_list_storage,omitempty"` - ActionCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,18,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` - IndirectContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,10,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` - InitialSizeClassCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,11,opt,name=initial_size_class_cache,json=initialSizeClassCache,proto3" json:"initial_size_class_cache,omitempty"` - FileSystemAccessCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,19,opt,name=file_system_access_cache,json=fileSystemAccessCache,proto3" json:"file_system_access_cache,omitempty"` - ExecuteAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,16,opt,name=execute_authorizer,json=executeAuthorizer,proto3" json:"execute_authorizer,omitempty"` - SupportedCompressors []v2.Compressor_Value `protobuf:"varint,20,rep,packed,name=supported_compressors,json=supportedCompressors,proto3,enum=build.bazel.remote.execution.v2.Compressor_Value" json:"supported_compressors,omitempty"` - ZstdPool *zstd.PoolConfiguration `protobuf:"bytes,21,opt,name=zstd_pool,json=zstdPool,proto3" json:"zstd_pool,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + GrpcServers []*grpc.ServerConfiguration `protobuf:"bytes,4,rep,name=grpc_servers,json=grpcServers,proto3" json:"grpc_servers,omitempty"` + Schedulers map[string]*builder.SchedulerConfiguration `protobuf:"bytes,5,rep,name=schedulers,proto3" json:"schedulers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + MaximumMessageSizeBytes int64 `protobuf:"varint,8,opt,name=maximum_message_size_bytes,json=maximumMessageSizeBytes,proto3" json:"maximum_message_size_bytes,omitempty"` + Global *global.Configuration `protobuf:"bytes,9,opt,name=global,proto3" json:"global,omitempty"` + ContentAddressableStorageServer *ContentAddressableStorageServerConfiguration `protobuf:"bytes,17,opt,name=content_addressable_storage_server,json=contentAddressableStorageServer,proto3" json:"content_addressable_storage_server,omitempty"` + ActionCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,18,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` + IndirectContentAddressableStorage *ScannableBlobAccessConfiguration `protobuf:"bytes,10,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` + InitialSizeClassCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,11,opt,name=initial_size_class_cache,json=initialSizeClassCache,proto3" json:"initial_size_class_cache,omitempty"` + FileSystemAccessCache *NonScannableBlobAccessConfiguration `protobuf:"bytes,19,opt,name=file_system_access_cache,json=fileSystemAccessCache,proto3" json:"file_system_access_cache,omitempty"` + ExecuteAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,16,opt,name=execute_authorizer,json=executeAuthorizer,proto3" json:"execute_authorizer,omitempty"` + SupportedCompressors []v2.Compressor_Value `protobuf:"varint,20,rep,packed,name=supported_compressors,json=supportedCompressors,proto3,enum=build.bazel.remote.execution.v2.Compressor_Value" json:"supported_compressors,omitempty"` + ZstdPool *zstd.PoolConfiguration `protobuf:"bytes,21,opt,name=zstd_pool,json=zstdPool,proto3" json:"zstd_pool,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -105,16 +104,9 @@ func (x *ApplicationConfiguration) GetGlobal() *global.Configuration { return nil } -func (x *ApplicationConfiguration) GetContentAddressableStorage() *ScannableBlobAccessConfiguration { +func (x *ApplicationConfiguration) GetContentAddressableStorageServer() *ContentAddressableStorageServerConfiguration { if x != nil { - return x.ContentAddressableStorage - } - return nil -} - -func (x *ApplicationConfiguration) GetChunkListStorage() *ScannableBlobAccessConfiguration { - if x != nil { - return x.ChunkListStorage + return x.ContentAddressableStorageServer } return nil } @@ -168,6 +160,74 @@ func (x *ApplicationConfiguration) GetZstdPool() *zstd.PoolConfiguration { return nil } +type ContentAddressableStorageServerConfiguration struct { + state protoimpl.MessageState `protogen:"open.v1"` + ContentAddressableStorage *blobstore.ContentAddressableStorageConfiguration `protobuf:"bytes,1,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` + GetAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,2,opt,name=get_authorizer,json=getAuthorizer,proto3" json:"get_authorizer,omitempty"` + PutAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,3,opt,name=put_authorizer,json=putAuthorizer,proto3" json:"put_authorizer,omitempty"` + FindMissingAuthorizer *auth.AuthorizerConfiguration `protobuf:"bytes,4,opt,name=find_missing_authorizer,json=findMissingAuthorizer,proto3" json:"find_missing_authorizer,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContentAddressableStorageServerConfiguration) Reset() { + *x = ContentAddressableStorageServerConfiguration{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContentAddressableStorageServerConfiguration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContentAddressableStorageServerConfiguration) ProtoMessage() {} + +func (x *ContentAddressableStorageServerConfiguration) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContentAddressableStorageServerConfiguration.ProtoReflect.Descriptor instead. +func (*ContentAddressableStorageServerConfiguration) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{1} +} + +func (x *ContentAddressableStorageServerConfiguration) GetContentAddressableStorage() *blobstore.ContentAddressableStorageConfiguration { + if x != nil { + return x.ContentAddressableStorage + } + return nil +} + +func (x *ContentAddressableStorageServerConfiguration) GetGetAuthorizer() *auth.AuthorizerConfiguration { + if x != nil { + return x.GetAuthorizer + } + return nil +} + +func (x *ContentAddressableStorageServerConfiguration) GetPutAuthorizer() *auth.AuthorizerConfiguration { + if x != nil { + return x.PutAuthorizer + } + return nil +} + +func (x *ContentAddressableStorageServerConfiguration) GetFindMissingAuthorizer() *auth.AuthorizerConfiguration { + if x != nil { + return x.FindMissingAuthorizer + } + return nil +} + type NonScannableBlobAccessConfiguration struct { state protoimpl.MessageState `protogen:"open.v1"` Backend *blobstore.BlobAccessConfiguration `protobuf:"bytes,1,opt,name=backend,proto3" json:"backend,omitempty"` @@ -179,7 +239,7 @@ type NonScannableBlobAccessConfiguration struct { func (x *NonScannableBlobAccessConfiguration) Reset() { *x = NonScannableBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[1] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -191,7 +251,7 @@ func (x *NonScannableBlobAccessConfiguration) String() string { func (*NonScannableBlobAccessConfiguration) ProtoMessage() {} func (x *NonScannableBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[1] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -204,7 +264,7 @@ func (x *NonScannableBlobAccessConfiguration) ProtoReflect() protoreflect.Messag // Deprecated: Use NonScannableBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*NonScannableBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{1} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{2} } func (x *NonScannableBlobAccessConfiguration) GetBackend() *blobstore.BlobAccessConfiguration { @@ -240,7 +300,7 @@ type ScannableBlobAccessConfiguration struct { func (x *ScannableBlobAccessConfiguration) Reset() { *x = ScannableBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[2] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -252,7 +312,7 @@ func (x *ScannableBlobAccessConfiguration) String() string { func (*ScannableBlobAccessConfiguration) ProtoMessage() {} func (x *ScannableBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[2] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -265,7 +325,7 @@ func (x *ScannableBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ScannableBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ScannableBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{2} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescGZIP(), []int{3} } func (x *ScannableBlobAccessConfiguration) GetBackend() *blobstore.BlobAccessConfiguration { @@ -300,16 +360,15 @@ var File_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_s const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDesc = "" + "\n" + - "Sgithub.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage/bb_storage.proto\x12\"buildbarn.configuration.bb_storage\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto\x1aQgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x1aMgithub.com/buildbarn/bb-storage/pkg/proto/configuration/builder/builder.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/zstd/zstd.proto\"\xb1\f\n" + + "Sgithub.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage/bb_storage.proto\x12\"buildbarn.configuration.bb_storage\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto\x1aQgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x1aMgithub.com/buildbarn/bb-storage/pkg/proto/configuration/builder/builder.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/global/global.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/zstd/zstd.proto\"\xd6\v\n" + "\x18ApplicationConfiguration\x12T\n" + "\fgrpc_servers\x18\x04 \x03(\v21.buildbarn.configuration.grpc.ServerConfigurationR\vgrpcServers\x12l\n" + "\n" + "schedulers\x18\x05 \x03(\v2L.buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntryR\n" + "schedulers\x12;\n" + "\x1amaximum_message_size_bytes\x18\b \x01(\x03R\x17maximumMessageSizeBytes\x12E\n" + - "\x06global\x18\t \x01(\v2-.buildbarn.configuration.global.ConfigurationR\x06global\x12\x84\x01\n" + - "\x1bcontent_addressable_storage\x18\x11 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x19contentAddressableStorage\x12r\n" + - "\x12chunk_list_storage\x18\x16 \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR\x10chunkListStorage\x12j\n" + + "\x06global\x18\t \x01(\v2-.buildbarn.configuration.global.ConfigurationR\x06global\x12\x9d\x01\n" + + "\"content_addressable_storage_server\x18\x11 \x01(\v2P.buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfigurationR\x1fcontentAddressableStorageServer\x12j\n" + "\faction_cache\x18\x12 \x01(\v2G.buildbarn.configuration.bb_storage.NonScannableBlobAccessConfigurationR\vactionCache\x12\x95\x01\n" + "$indirect_content_addressable_storage\x18\n" + " \x01(\v2D.buildbarn.configuration.bb_storage.ScannableBlobAccessConfigurationR!indirectContentAddressableStorage\x12\x80\x01\n" + @@ -320,7 +379,12 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb "\tzstd_pool\x18\x15 \x01(\v2/.buildbarn.configuration.zstd.PoolConfigurationR\bzstdPool\x1av\n" + "\x0fSchedulersEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12M\n" + - "\x05value\x18\x02 \x01(\v27.buildbarn.configuration.builder.SchedulerConfigurationR\x05value:\x028\x01J\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x06\x10\aJ\x04\b\a\x10\bJ\x04\b\f\x10\rJ\x04\b\r\x10\x0eJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10\"\xb7\x02\n" + + "\x05value\x18\x02 \x01(\v27.buildbarn.configuration.builder.SchedulerConfigurationR\x05value:\x028\x01J\x04\b\x01\x10\x02J\x04\b\x02\x10\x03J\x04\b\x03\x10\x04J\x04\b\x06\x10\aJ\x04\b\a\x10\bJ\x04\b\f\x10\rJ\x04\b\r\x10\x0eJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10\"\xe5\x03\n" + + ",ContentAddressableStorageServerConfiguration\x12\x89\x01\n" + + "\x1bcontent_addressable_storage\x18\x01 \x01(\v2I.buildbarn.configuration.blobstore.ContentAddressableStorageConfigurationR\x19contentAddressableStorage\x12\\\n" + + "\x0eget_authorizer\x18\x02 \x01(\v25.buildbarn.configuration.auth.AuthorizerConfigurationR\rgetAuthorizer\x12\\\n" + + "\x0eput_authorizer\x18\x03 \x01(\v25.buildbarn.configuration.auth.AuthorizerConfigurationR\rputAuthorizer\x12m\n" + + "\x17find_missing_authorizer\x18\x04 \x01(\v25.buildbarn.configuration.auth.AuthorizerConfigurationR\x15findMissingAuthorizer\"\xb7\x02\n" + "#NonScannableBlobAccessConfiguration\x12T\n" + "\abackend\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\abackend\x12\\\n" + "\x0eget_authorizer\x18\x02 \x01(\v25.buildbarn.configuration.auth.AuthorizerConfigurationR\rgetAuthorizer\x12\\\n" + @@ -343,46 +407,51 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_ return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDescData } -var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_goTypes = []any{ - (*ApplicationConfiguration)(nil), // 0: buildbarn.configuration.bb_storage.ApplicationConfiguration - (*NonScannableBlobAccessConfiguration)(nil), // 1: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - (*ScannableBlobAccessConfiguration)(nil), // 2: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - nil, // 3: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry - (*grpc.ServerConfiguration)(nil), // 4: buildbarn.configuration.grpc.ServerConfiguration - (*global.Configuration)(nil), // 5: buildbarn.configuration.global.Configuration - (*auth.AuthorizerConfiguration)(nil), // 6: buildbarn.configuration.auth.AuthorizerConfiguration - (v2.Compressor_Value)(0), // 7: build.bazel.remote.execution.v2.Compressor.Value - (*zstd.PoolConfiguration)(nil), // 8: buildbarn.configuration.zstd.PoolConfiguration - (*blobstore.BlobAccessConfiguration)(nil), // 9: buildbarn.configuration.blobstore.BlobAccessConfiguration - (*builder.SchedulerConfiguration)(nil), // 10: buildbarn.configuration.builder.SchedulerConfiguration + (*ApplicationConfiguration)(nil), // 0: buildbarn.configuration.bb_storage.ApplicationConfiguration + (*ContentAddressableStorageServerConfiguration)(nil), // 1: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration + (*NonScannableBlobAccessConfiguration)(nil), // 2: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + (*ScannableBlobAccessConfiguration)(nil), // 3: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration + nil, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry + (*grpc.ServerConfiguration)(nil), // 5: buildbarn.configuration.grpc.ServerConfiguration + (*global.Configuration)(nil), // 6: buildbarn.configuration.global.Configuration + (*auth.AuthorizerConfiguration)(nil), // 7: buildbarn.configuration.auth.AuthorizerConfiguration + (v2.Compressor_Value)(0), // 8: build.bazel.remote.execution.v2.Compressor.Value + (*zstd.PoolConfiguration)(nil), // 9: buildbarn.configuration.zstd.PoolConfiguration + (*blobstore.ContentAddressableStorageConfiguration)(nil), // 10: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + (*blobstore.BlobAccessConfiguration)(nil), // 11: buildbarn.configuration.blobstore.BlobAccessConfiguration + (*builder.SchedulerConfiguration)(nil), // 12: buildbarn.configuration.builder.SchedulerConfiguration } var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_depIdxs = []int32{ - 4, // 0: buildbarn.configuration.bb_storage.ApplicationConfiguration.grpc_servers:type_name -> buildbarn.configuration.grpc.ServerConfiguration - 3, // 1: buildbarn.configuration.bb_storage.ApplicationConfiguration.schedulers:type_name -> buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry - 5, // 2: buildbarn.configuration.bb_storage.ApplicationConfiguration.global:type_name -> buildbarn.configuration.global.Configuration - 2, // 3: buildbarn.configuration.bb_storage.ApplicationConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 2, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.chunk_list_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 1, // 5: buildbarn.configuration.bb_storage.ApplicationConfiguration.action_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 2, // 6: buildbarn.configuration.bb_storage.ApplicationConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration - 1, // 7: buildbarn.configuration.bb_storage.ApplicationConfiguration.initial_size_class_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 1, // 8: buildbarn.configuration.bb_storage.ApplicationConfiguration.file_system_access_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration - 6, // 9: buildbarn.configuration.bb_storage.ApplicationConfiguration.execute_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 7, // 10: buildbarn.configuration.bb_storage.ApplicationConfiguration.supported_compressors:type_name -> build.bazel.remote.execution.v2.Compressor.Value - 8, // 11: buildbarn.configuration.bb_storage.ApplicationConfiguration.zstd_pool:type_name -> buildbarn.configuration.zstd.PoolConfiguration - 9, // 12: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 6, // 13: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 14: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 9, // 15: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 6, // 16: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 17: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 6, // 18: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration - 10, // 19: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry.value:type_name -> buildbarn.configuration.builder.SchedulerConfiguration - 20, // [20:20] is the sub-list for method output_type - 20, // [20:20] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 5, // 0: buildbarn.configuration.bb_storage.ApplicationConfiguration.grpc_servers:type_name -> buildbarn.configuration.grpc.ServerConfiguration + 4, // 1: buildbarn.configuration.bb_storage.ApplicationConfiguration.schedulers:type_name -> buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry + 6, // 2: buildbarn.configuration.bb_storage.ApplicationConfiguration.global:type_name -> buildbarn.configuration.global.Configuration + 1, // 3: buildbarn.configuration.bb_storage.ApplicationConfiguration.content_addressable_storage_server:type_name -> buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration + 2, // 4: buildbarn.configuration.bb_storage.ApplicationConfiguration.action_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 3, // 5: buildbarn.configuration.bb_storage.ApplicationConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration + 2, // 6: buildbarn.configuration.bb_storage.ApplicationConfiguration.initial_size_class_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 2, // 7: buildbarn.configuration.bb_storage.ApplicationConfiguration.file_system_access_cache:type_name -> buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration + 7, // 8: buildbarn.configuration.bb_storage.ApplicationConfiguration.execute_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 8, // 9: buildbarn.configuration.bb_storage.ApplicationConfiguration.supported_compressors:type_name -> build.bazel.remote.execution.v2.Compressor.Value + 9, // 10: buildbarn.configuration.bb_storage.ApplicationConfiguration.zstd_pool:type_name -> buildbarn.configuration.zstd.PoolConfiguration + 10, // 11: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + 7, // 12: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 13: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 14: buildbarn.configuration.bb_storage.ContentAddressableStorageServerConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 11, // 15: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 7, // 16: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 17: buildbarn.configuration.bb_storage.NonScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 11, // 18: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 7, // 19: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.get_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 20: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.put_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 7, // 21: buildbarn.configuration.bb_storage.ScannableBlobAccessConfiguration.find_missing_authorizer:type_name -> buildbarn.configuration.auth.AuthorizerConfiguration + 12, // 22: buildbarn.configuration.bb_storage.ApplicationConfiguration.SchedulersEntry.value:type_name -> buildbarn.configuration.builder.SchedulerConfiguration + 23, // [23:23] is the sub-list for method output_type + 23, // [23:23] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name } func init() { @@ -398,7 +467,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_bb_storage_bb_storage_proto_rawDesc)), NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/configuration/bb_storage/bb_storage.proto b/pkg/proto/configuration/bb_storage/bb_storage.proto index 55f431b9b..ee27ee097 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.proto +++ b/pkg/proto/configuration/bb_storage/bb_storage.proto @@ -57,13 +57,10 @@ message ApplicationConfiguration { // Common configuration options that apply to all Buildbarn binaries. buildbarn.configuration.global.Configuration global = 9; - // Optional: Blobstore configuration for the Content Addressable - // Storage (CAS). - ScannableBlobAccessConfiguration content_addressable_storage = 17; - - // Optional: Blobstore configuration for the Content List Storage - // (CLS). - ScannableBlobAccessConfiguration chunk_list_storage = 22; + // Optional: Configurations for the Content Addressable Storage server + // (CAS). + ContentAddressableStorageServerConfiguration + content_addressable_storage_server = 17; // Optional: Blobstore configuration for the Action Cache (AC). NonScannableBlobAccessConfiguration action_cache = 18; @@ -115,6 +112,29 @@ message ApplicationConfiguration { buildbarn.configuration.zstd.PoolConfiguration zstd_pool = 21; } +// Storage configuration for a content addressable storage. +message ContentAddressableStorageServerConfiguration { + // Configurations for the constituent Blob Accesses that make up the + // ContentAddressableStorageServer. + buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + content_addressable_storage = 1; + + // The authorizer for determining whether a client may read from + // storage. Access to this authorizer pertains to the + // ByteStream.Read(), BatchReadBlobs() and SplitBlob() operations. + buildbarn.configuration.auth.AuthorizerConfiguration get_authorizer = 2; + + // The authorizer for determining whether a client may write to + // storage. Access to this configuration pertains to the + // ByteStream.Write(), BatchUpdateBlobs() and SpliceBlob() operations. + buildbarn.configuration.auth.AuthorizerConfiguration put_authorizer = 3; + + // The authorizer for determining whether a client may scan storage + // for the existence of a batch of digests. + buildbarn.configuration.auth.AuthorizerConfiguration find_missing_authorizer = + 4; +} + // Storage configuration for backends which don't allow batch digest // scanning. message NonScannableBlobAccessConfiguration { diff --git a/pkg/proto/configuration/blobstore/blobstore.pb.go b/pkg/proto/configuration/blobstore/blobstore.pb.go index 237805e13..b3e408af5 100644 --- a/pkg/proto/configuration/blobstore/blobstore.pb.go +++ b/pkg/proto/configuration/blobstore/blobstore.pb.go @@ -32,17 +32,85 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type ContentAddressableStorageConfiguration struct { + state protoimpl.MessageState `protogen:"open.v1"` + ChunkStorage *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=chunk_storage,json=chunkStorage,proto3" json:"chunk_storage,omitempty"` + ChunkListStorage *BlobAccessConfiguration `protobuf:"bytes,2,opt,name=chunk_list_storage,json=chunkListStorage,proto3" json:"chunk_list_storage,omitempty"` + ChunkListCache *digest.ExistenceCacheConfiguration `protobuf:"bytes,3,opt,name=chunk_list_cache,json=chunkListCache,proto3" json:"chunk_list_cache,omitempty"` + CdcParameterCache *digest.ExistenceCacheConfiguration `protobuf:"bytes,4,opt,name=cdc_parameter_cache,json=cdcParameterCache,proto3" json:"cdc_parameter_cache,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ContentAddressableStorageConfiguration) Reset() { + *x = ContentAddressableStorageConfiguration{} + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ContentAddressableStorageConfiguration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ContentAddressableStorageConfiguration) ProtoMessage() {} + +func (x *ContentAddressableStorageConfiguration) ProtoReflect() protoreflect.Message { + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ContentAddressableStorageConfiguration.ProtoReflect.Descriptor instead. +func (*ContentAddressableStorageConfiguration) Descriptor() ([]byte, []int) { + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{0} +} + +func (x *ContentAddressableStorageConfiguration) GetChunkStorage() *BlobAccessConfiguration { + if x != nil { + return x.ChunkStorage + } + return nil +} + +func (x *ContentAddressableStorageConfiguration) GetChunkListStorage() *BlobAccessConfiguration { + if x != nil { + return x.ChunkListStorage + } + return nil +} + +func (x *ContentAddressableStorageConfiguration) GetChunkListCache() *digest.ExistenceCacheConfiguration { + if x != nil { + return x.ChunkListCache + } + return nil +} + +func (x *ContentAddressableStorageConfiguration) GetCdcParameterCache() *digest.ExistenceCacheConfiguration { + if x != nil { + return x.CdcParameterCache + } + return nil +} + type BlobstoreConfiguration struct { - state protoimpl.MessageState `protogen:"open.v1"` - ContentAddressableStorage *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` - ActionCache *BlobAccessConfiguration `protobuf:"bytes,2,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + ContentAddressableStorage *ContentAddressableStorageConfiguration `protobuf:"bytes,1,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` + ActionCache *BlobAccessConfiguration `protobuf:"bytes,2,opt,name=action_cache,json=actionCache,proto3" json:"action_cache,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *BlobstoreConfiguration) Reset() { *x = BlobstoreConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[0] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -54,7 +122,7 @@ func (x *BlobstoreConfiguration) String() string { func (*BlobstoreConfiguration) ProtoMessage() {} func (x *BlobstoreConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[0] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -67,10 +135,10 @@ func (x *BlobstoreConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobstoreConfiguration.ProtoReflect.Descriptor instead. func (*BlobstoreConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{0} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{1} } -func (x *BlobstoreConfiguration) GetContentAddressableStorage() *BlobAccessConfiguration { +func (x *BlobstoreConfiguration) GetContentAddressableStorage() *ContentAddressableStorageConfiguration { if x != nil { return x.ContentAddressableStorage } @@ -115,7 +183,7 @@ type BlobAccessConfiguration struct { func (x *BlobAccessConfiguration) Reset() { *x = BlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -127,7 +195,7 @@ func (x *BlobAccessConfiguration) String() string { func (*BlobAccessConfiguration) ProtoMessage() {} func (x *BlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -140,7 +208,7 @@ func (x *BlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*BlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{1} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{2} } func (x *BlobAccessConfiguration) GetBackend() isBlobAccessConfiguration_Backend { @@ -465,7 +533,7 @@ type ReadCachingBlobAccessConfiguration struct { func (x *ReadCachingBlobAccessConfiguration) Reset() { *x = ReadCachingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -477,7 +545,7 @@ func (x *ReadCachingBlobAccessConfiguration) String() string { func (*ReadCachingBlobAccessConfiguration) ProtoMessage() {} func (x *ReadCachingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -490,7 +558,7 @@ func (x *ReadCachingBlobAccessConfiguration) ProtoReflect() protoreflect.Message // Deprecated: Use ReadCachingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ReadCachingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{2} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{3} } func (x *ReadCachingBlobAccessConfiguration) GetSlow() *BlobAccessConfiguration { @@ -523,7 +591,7 @@ type ShardingBlobAccessConfiguration struct { func (x *ShardingBlobAccessConfiguration) Reset() { *x = ShardingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[3] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -535,7 +603,7 @@ func (x *ShardingBlobAccessConfiguration) String() string { func (*ShardingBlobAccessConfiguration) ProtoMessage() {} func (x *ShardingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[3] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -548,7 +616,7 @@ func (x *ShardingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ShardingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{3} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{4} } func (x *ShardingBlobAccessConfiguration) GetShards() map[string]*ShardingBlobAccessConfiguration_Shard { @@ -570,7 +638,7 @@ type MirroredBlobAccessConfiguration struct { func (x *MirroredBlobAccessConfiguration) Reset() { *x = MirroredBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[4] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -582,7 +650,7 @@ func (x *MirroredBlobAccessConfiguration) String() string { func (*MirroredBlobAccessConfiguration) ProtoMessage() {} func (x *MirroredBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[4] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -595,7 +663,7 @@ func (x *MirroredBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use MirroredBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*MirroredBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{4} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5} } func (x *MirroredBlobAccessConfiguration) GetBackendA() *BlobAccessConfiguration { @@ -652,7 +720,7 @@ type LocalBlobAccessConfiguration struct { func (x *LocalBlobAccessConfiguration) Reset() { *x = LocalBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -664,7 +732,7 @@ func (x *LocalBlobAccessConfiguration) String() string { func (*LocalBlobAccessConfiguration) ProtoMessage() {} func (x *LocalBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -677,7 +745,7 @@ func (x *LocalBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6} } func (x *LocalBlobAccessConfiguration) GetKeyLocationMapBackend() isLocalBlobAccessConfiguration_KeyLocationMapBackend { @@ -831,7 +899,7 @@ type ExistenceCachingBlobAccessConfiguration struct { func (x *ExistenceCachingBlobAccessConfiguration) Reset() { *x = ExistenceCachingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -843,7 +911,7 @@ func (x *ExistenceCachingBlobAccessConfiguration) String() string { func (*ExistenceCachingBlobAccessConfiguration) ProtoMessage() {} func (x *ExistenceCachingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -856,7 +924,7 @@ func (x *ExistenceCachingBlobAccessConfiguration) ProtoReflect() protoreflect.Me // Deprecated: Use ExistenceCachingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ExistenceCachingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{7} } func (x *ExistenceCachingBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -883,7 +951,7 @@ type CompletenessCheckingBlobAccessConfiguration struct { func (x *CompletenessCheckingBlobAccessConfiguration) Reset() { *x = CompletenessCheckingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[7] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -895,7 +963,7 @@ func (x *CompletenessCheckingBlobAccessConfiguration) String() string { func (*CompletenessCheckingBlobAccessConfiguration) ProtoMessage() {} func (x *CompletenessCheckingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[7] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -908,7 +976,7 @@ func (x *CompletenessCheckingBlobAccessConfiguration) ProtoReflect() protoreflec // Deprecated: Use CompletenessCheckingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*CompletenessCheckingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{7} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{8} } func (x *CompletenessCheckingBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -936,7 +1004,7 @@ type ReadFallbackBlobAccessConfiguration struct { func (x *ReadFallbackBlobAccessConfiguration) Reset() { *x = ReadFallbackBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[8] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -948,7 +1016,7 @@ func (x *ReadFallbackBlobAccessConfiguration) String() string { func (*ReadFallbackBlobAccessConfiguration) ProtoMessage() {} func (x *ReadFallbackBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[8] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -961,7 +1029,7 @@ func (x *ReadFallbackBlobAccessConfiguration) ProtoReflect() protoreflect.Messag // Deprecated: Use ReadFallbackBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ReadFallbackBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{8} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{9} } func (x *ReadFallbackBlobAccessConfiguration) GetPrimary() *BlobAccessConfiguration { @@ -986,19 +1054,19 @@ func (x *ReadFallbackBlobAccessConfiguration) GetReplicator() *BlobReplicatorCon } type ReferenceExpandingBlobAccessConfiguration struct { - state protoimpl.MessageState `protogen:"open.v1"` - IndirectContentAddressableStorage *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` - AwsSession *aws.SessionConfiguration `protobuf:"bytes,2,opt,name=aws_session,json=awsSession,proto3" json:"aws_session,omitempty"` - HttpClient *client.Configuration `protobuf:"bytes,3,opt,name=http_client,json=httpClient,proto3" json:"http_client,omitempty"` - GcpClientOptions *gcp.ClientOptionsConfiguration `protobuf:"bytes,4,opt,name=gcp_client_options,json=gcpClientOptions,proto3" json:"gcp_client_options,omitempty"` - ContentAddressableStorage *BlobAccessConfiguration `protobuf:"bytes,5,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + IndirectContentAddressableStorage *BlobAccessConfiguration `protobuf:"bytes,1,opt,name=indirect_content_addressable_storage,json=indirectContentAddressableStorage,proto3" json:"indirect_content_addressable_storage,omitempty"` + AwsSession *aws.SessionConfiguration `protobuf:"bytes,2,opt,name=aws_session,json=awsSession,proto3" json:"aws_session,omitempty"` + HttpClient *client.Configuration `protobuf:"bytes,3,opt,name=http_client,json=httpClient,proto3" json:"http_client,omitempty"` + GcpClientOptions *gcp.ClientOptionsConfiguration `protobuf:"bytes,4,opt,name=gcp_client_options,json=gcpClientOptions,proto3" json:"gcp_client_options,omitempty"` + ContentAddressableStorage *ContentAddressableStorageConfiguration `protobuf:"bytes,5,opt,name=content_addressable_storage,json=contentAddressableStorage,proto3" json:"content_addressable_storage,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ReferenceExpandingBlobAccessConfiguration) Reset() { *x = ReferenceExpandingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[9] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1010,7 +1078,7 @@ func (x *ReferenceExpandingBlobAccessConfiguration) String() string { func (*ReferenceExpandingBlobAccessConfiguration) ProtoMessage() {} func (x *ReferenceExpandingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[9] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1023,7 +1091,7 @@ func (x *ReferenceExpandingBlobAccessConfiguration) ProtoReflect() protoreflect. // Deprecated: Use ReferenceExpandingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ReferenceExpandingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{9} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{10} } func (x *ReferenceExpandingBlobAccessConfiguration) GetIndirectContentAddressableStorage() *BlobAccessConfiguration { @@ -1054,7 +1122,7 @@ func (x *ReferenceExpandingBlobAccessConfiguration) GetGcpClientOptions() *gcp.C return nil } -func (x *ReferenceExpandingBlobAccessConfiguration) GetContentAddressableStorage() *BlobAccessConfiguration { +func (x *ReferenceExpandingBlobAccessConfiguration) GetContentAddressableStorage() *ContentAddressableStorageConfiguration { if x != nil { return x.ContentAddressableStorage } @@ -1078,7 +1146,7 @@ type BlobReplicatorConfiguration struct { func (x *BlobReplicatorConfiguration) Reset() { *x = BlobReplicatorConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1090,7 +1158,7 @@ func (x *BlobReplicatorConfiguration) String() string { func (*BlobReplicatorConfiguration) ProtoMessage() {} func (x *BlobReplicatorConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1103,7 +1171,7 @@ func (x *BlobReplicatorConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use BlobReplicatorConfiguration.ProtoReflect.Descriptor instead. func (*BlobReplicatorConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{10} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{11} } func (x *BlobReplicatorConfiguration) GetMode() isBlobReplicatorConfiguration_Mode { @@ -1217,7 +1285,7 @@ type QueuedBlobReplicatorConfiguration struct { func (x *QueuedBlobReplicatorConfiguration) Reset() { *x = QueuedBlobReplicatorConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1229,7 +1297,7 @@ func (x *QueuedBlobReplicatorConfiguration) String() string { func (*QueuedBlobReplicatorConfiguration) ProtoMessage() {} func (x *QueuedBlobReplicatorConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1242,7 +1310,7 @@ func (x *QueuedBlobReplicatorConfiguration) ProtoReflect() protoreflect.Message // Deprecated: Use QueuedBlobReplicatorConfiguration.ProtoReflect.Descriptor instead. func (*QueuedBlobReplicatorConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{11} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{12} } func (x *QueuedBlobReplicatorConfiguration) GetBase() *BlobReplicatorConfiguration { @@ -1269,7 +1337,7 @@ type ConcurrencyLimitingBlobReplicatorConfiguration struct { func (x *ConcurrencyLimitingBlobReplicatorConfiguration) Reset() { *x = ConcurrencyLimitingBlobReplicatorConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[12] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1281,7 +1349,7 @@ func (x *ConcurrencyLimitingBlobReplicatorConfiguration) String() string { func (*ConcurrencyLimitingBlobReplicatorConfiguration) ProtoMessage() {} func (x *ConcurrencyLimitingBlobReplicatorConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[12] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1294,7 +1362,7 @@ func (x *ConcurrencyLimitingBlobReplicatorConfiguration) ProtoReflect() protoref // Deprecated: Use ConcurrencyLimitingBlobReplicatorConfiguration.ProtoReflect.Descriptor instead. func (*ConcurrencyLimitingBlobReplicatorConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{12} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{13} } func (x *ConcurrencyLimitingBlobReplicatorConfiguration) GetBase() *BlobReplicatorConfiguration { @@ -1320,7 +1388,7 @@ type DemultiplexingBlobAccessConfiguration struct { func (x *DemultiplexingBlobAccessConfiguration) Reset() { *x = DemultiplexingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[13] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1332,7 +1400,7 @@ func (x *DemultiplexingBlobAccessConfiguration) String() string { func (*DemultiplexingBlobAccessConfiguration) ProtoMessage() {} func (x *DemultiplexingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[13] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1345,7 +1413,7 @@ func (x *DemultiplexingBlobAccessConfiguration) ProtoReflect() protoreflect.Mess // Deprecated: Use DemultiplexingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*DemultiplexingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{13} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{14} } func (x *DemultiplexingBlobAccessConfiguration) GetInstanceNamePrefixes() map[string]*DemultiplexedBlobAccessConfiguration { @@ -1365,7 +1433,7 @@ type DemultiplexedBlobAccessConfiguration struct { func (x *DemultiplexedBlobAccessConfiguration) Reset() { *x = DemultiplexedBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[14] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1377,7 +1445,7 @@ func (x *DemultiplexedBlobAccessConfiguration) String() string { func (*DemultiplexedBlobAccessConfiguration) ProtoMessage() {} func (x *DemultiplexedBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[14] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1390,7 +1458,7 @@ func (x *DemultiplexedBlobAccessConfiguration) ProtoReflect() protoreflect.Messa // Deprecated: Use DemultiplexedBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*DemultiplexedBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{14} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{15} } func (x *DemultiplexedBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -1419,7 +1487,7 @@ type ActionResultExpiringBlobAccessConfiguration struct { func (x *ActionResultExpiringBlobAccessConfiguration) Reset() { *x = ActionResultExpiringBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[15] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1431,7 +1499,7 @@ func (x *ActionResultExpiringBlobAccessConfiguration) String() string { func (*ActionResultExpiringBlobAccessConfiguration) ProtoMessage() {} func (x *ActionResultExpiringBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[15] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1444,7 +1512,7 @@ func (x *ActionResultExpiringBlobAccessConfiguration) ProtoReflect() protoreflec // Deprecated: Use ActionResultExpiringBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ActionResultExpiringBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{15} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{16} } func (x *ActionResultExpiringBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -1487,7 +1555,7 @@ type ReadCanaryingBlobAccessConfiguration struct { func (x *ReadCanaryingBlobAccessConfiguration) Reset() { *x = ReadCanaryingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[16] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1499,7 +1567,7 @@ func (x *ReadCanaryingBlobAccessConfiguration) String() string { func (*ReadCanaryingBlobAccessConfiguration) ProtoMessage() {} func (x *ReadCanaryingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[16] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1512,7 +1580,7 @@ func (x *ReadCanaryingBlobAccessConfiguration) ProtoReflect() protoreflect.Messa // Deprecated: Use ReadCanaryingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ReadCanaryingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{16} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{17} } func (x *ReadCanaryingBlobAccessConfiguration) GetSource() *BlobAccessConfiguration { @@ -1553,7 +1621,7 @@ type ZIPBlobAccessConfiguration struct { func (x *ZIPBlobAccessConfiguration) Reset() { *x = ZIPBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[17] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1565,7 +1633,7 @@ func (x *ZIPBlobAccessConfiguration) String() string { func (*ZIPBlobAccessConfiguration) ProtoMessage() {} func (x *ZIPBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[17] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1578,7 +1646,7 @@ func (x *ZIPBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ZIPBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ZIPBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{17} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{18} } func (x *ZIPBlobAccessConfiguration) GetPath() string { @@ -1605,7 +1673,7 @@ type WithLabelsBlobAccessConfiguration struct { func (x *WithLabelsBlobAccessConfiguration) Reset() { *x = WithLabelsBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[18] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1617,7 +1685,7 @@ func (x *WithLabelsBlobAccessConfiguration) String() string { func (*WithLabelsBlobAccessConfiguration) ProtoMessage() {} func (x *WithLabelsBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[18] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1630,7 +1698,7 @@ func (x *WithLabelsBlobAccessConfiguration) ProtoReflect() protoreflect.Message // Deprecated: Use WithLabelsBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*WithLabelsBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{18} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{19} } func (x *WithLabelsBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -1657,7 +1725,7 @@ type DeadlineEnforcingBlobAccess struct { func (x *DeadlineEnforcingBlobAccess) Reset() { *x = DeadlineEnforcingBlobAccess{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[19] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1669,7 +1737,7 @@ func (x *DeadlineEnforcingBlobAccess) String() string { func (*DeadlineEnforcingBlobAccess) ProtoMessage() {} func (x *DeadlineEnforcingBlobAccess) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[19] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1682,7 +1750,7 @@ func (x *DeadlineEnforcingBlobAccess) ProtoReflect() protoreflect.Message { // Deprecated: Use DeadlineEnforcingBlobAccess.ProtoReflect.Descriptor instead. func (*DeadlineEnforcingBlobAccess) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{19} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{20} } func (x *DeadlineEnforcingBlobAccess) GetTimeout() *durationpb.Duration { @@ -1709,7 +1777,7 @@ type GrpcBlobAccessConfiguration struct { func (x *GrpcBlobAccessConfiguration) Reset() { *x = GrpcBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[20] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1721,7 +1789,7 @@ func (x *GrpcBlobAccessConfiguration) String() string { func (*GrpcBlobAccessConfiguration) ProtoMessage() {} func (x *GrpcBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[20] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1734,7 +1802,7 @@ func (x *GrpcBlobAccessConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use GrpcBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*GrpcBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{20} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{21} } func (x *GrpcBlobAccessConfiguration) GetClient() *grpc.ClientConfiguration { @@ -1760,7 +1828,7 @@ type ChunkListValidatingBlobAccessConfiguration struct { func (x *ChunkListValidatingBlobAccessConfiguration) Reset() { *x = ChunkListValidatingBlobAccessConfiguration{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1772,7 +1840,7 @@ func (x *ChunkListValidatingBlobAccessConfiguration) String() string { func (*ChunkListValidatingBlobAccessConfiguration) ProtoMessage() {} func (x *ChunkListValidatingBlobAccessConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[21] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1785,7 +1853,7 @@ func (x *ChunkListValidatingBlobAccessConfiguration) ProtoReflect() protoreflect // Deprecated: Use ChunkListValidatingBlobAccessConfiguration.ProtoReflect.Descriptor instead. func (*ChunkListValidatingBlobAccessConfiguration) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{21} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{22} } func (x *ChunkListValidatingBlobAccessConfiguration) GetBackend() *BlobAccessConfiguration { @@ -1805,7 +1873,7 @@ type ShardingBlobAccessConfiguration_Shard struct { func (x *ShardingBlobAccessConfiguration_Shard) Reset() { *x = ShardingBlobAccessConfiguration_Shard{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1817,7 +1885,7 @@ func (x *ShardingBlobAccessConfiguration_Shard) String() string { func (*ShardingBlobAccessConfiguration_Shard) ProtoMessage() {} func (x *ShardingBlobAccessConfiguration_Shard) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[22] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1830,7 +1898,7 @@ func (x *ShardingBlobAccessConfiguration_Shard) ProtoReflect() protoreflect.Mess // Deprecated: Use ShardingBlobAccessConfiguration_Shard.ProtoReflect.Descriptor instead. func (*ShardingBlobAccessConfiguration_Shard) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{3, 0} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{4, 0} } func (x *ShardingBlobAccessConfiguration_Shard) GetBackend() *BlobAccessConfiguration { @@ -1856,7 +1924,7 @@ type LocalBlobAccessConfiguration_KeyLocationMapInMemory struct { func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) Reset() { *x = LocalBlobAccessConfiguration_KeyLocationMapInMemory{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1868,7 +1936,7 @@ func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) String() string { func (*LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[24] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1881,7 +1949,7 @@ func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) ProtoReflect() pro // Deprecated: Use LocalBlobAccessConfiguration_KeyLocationMapInMemory.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration_KeyLocationMapInMemory) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5, 0} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6, 0} } func (x *LocalBlobAccessConfiguration_KeyLocationMapInMemory) GetEntries() int64 { @@ -1900,7 +1968,7 @@ type LocalBlobAccessConfiguration_BlocksInMemory struct { func (x *LocalBlobAccessConfiguration_BlocksInMemory) Reset() { *x = LocalBlobAccessConfiguration_BlocksInMemory{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1912,7 +1980,7 @@ func (x *LocalBlobAccessConfiguration_BlocksInMemory) String() string { func (*LocalBlobAccessConfiguration_BlocksInMemory) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_BlocksInMemory) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[25] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1925,7 +1993,7 @@ func (x *LocalBlobAccessConfiguration_BlocksInMemory) ProtoReflect() protoreflec // Deprecated: Use LocalBlobAccessConfiguration_BlocksInMemory.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration_BlocksInMemory) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5, 1} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6, 1} } func (x *LocalBlobAccessConfiguration_BlocksInMemory) GetBlockSizeBytes() int64 { @@ -1946,7 +2014,7 @@ type LocalBlobAccessConfiguration_BlocksOnBlockDevice struct { func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) Reset() { *x = LocalBlobAccessConfiguration_BlocksOnBlockDevice{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1958,7 +2026,7 @@ func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) String() string { func (*LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[26] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1971,7 +2039,7 @@ func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) ProtoReflect() protor // Deprecated: Use LocalBlobAccessConfiguration_BlocksOnBlockDevice.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration_BlocksOnBlockDevice) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5, 2} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6, 2} } func (x *LocalBlobAccessConfiguration_BlocksOnBlockDevice) GetSource() *blockdevice.Configuration { @@ -2005,7 +2073,7 @@ type LocalBlobAccessConfiguration_Persistent struct { func (x *LocalBlobAccessConfiguration_Persistent) Reset() { *x = LocalBlobAccessConfiguration_Persistent{} - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2017,7 +2085,7 @@ func (x *LocalBlobAccessConfiguration_Persistent) String() string { func (*LocalBlobAccessConfiguration_Persistent) ProtoMessage() {} func (x *LocalBlobAccessConfiguration_Persistent) ProtoReflect() protoreflect.Message { - mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[27] + mi := &file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2030,7 +2098,7 @@ func (x *LocalBlobAccessConfiguration_Persistent) ProtoReflect() protoreflect.Me // Deprecated: Use LocalBlobAccessConfiguration_Persistent.ProtoReflect.Descriptor instead. func (*LocalBlobAccessConfiguration_Persistent) Descriptor() ([]byte, []int) { - return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{5, 3} + return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescGZIP(), []int{6, 3} } func (x *LocalBlobAccessConfiguration_Persistent) GetStateDirectoryPath() string { @@ -2051,9 +2119,14 @@ var File_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobs const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc = "" + "\n" + - "Qgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x12!buildbarn.configuration.blobstore\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aUgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/digest/digest.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aPgithub.com/buildbarn/bb-storage/pkg/proto/configuration/http/client/client.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"\xf3\x01\n" + - "\x16BlobstoreConfiguration\x12z\n" + - "\x1bcontent_addressable_storage\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\x19contentAddressableStorage\x12]\n" + + "Qgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto\x12!buildbarn.configuration.blobstore\x1a6build/bazel/remote/execution/v2/remote_execution.proto\x1aUgithub.com/buildbarn/bb-storage/pkg/proto/configuration/blockdevice/blockdevice.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/aws/aws.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/cloud/gcp/gcp.proto\x1aKgithub.com/buildbarn/bb-storage/pkg/proto/configuration/digest/digest.proto\x1aGgithub.com/buildbarn/bb-storage/pkg/proto/configuration/grpc/grpc.proto\x1aPgithub.com/buildbarn/bb-storage/pkg/proto/configuration/http/client/client.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"\xc7\x03\n" + + "&ContentAddressableStorageConfiguration\x12_\n" + + "\rchunk_storage\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\fchunkStorage\x12h\n" + + "\x12chunk_list_storage\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\x10chunkListStorage\x12e\n" + + "\x10chunk_list_cache\x18\x03 \x01(\v2;.buildbarn.configuration.digest.ExistenceCacheConfigurationR\x0echunkListCache\x12k\n" + + "\x13cdc_parameter_cache\x18\x04 \x01(\v2;.buildbarn.configuration.digest.ExistenceCacheConfigurationR\x11cdcParameterCache\"\x83\x02\n" + + "\x16BlobstoreConfiguration\x12\x89\x01\n" + + "\x1bcontent_addressable_storage\x18\x01 \x01(\v2I.buildbarn.configuration.blobstore.ContentAddressableStorageConfigurationR\x19contentAddressableStorage\x12]\n" + "\faction_cache\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\vactionCache\"\xe9\x10\n" + "\x17BlobAccessConfiguration\x12j\n" + "\fread_caching\x18\x04 \x01(\v2E.buildbarn.configuration.blobstore.ReadCachingBlobAccessConfigurationH\x00R\vreadCaching\x12T\n" + @@ -2143,15 +2216,15 @@ const file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blo "\tsecondary\x18\x02 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\tsecondary\x12^\n" + "\n" + "replicator\x18\x03 \x01(\v2>.buildbarn.configuration.blobstore.BlobReplicatorConfigurationR\n" + - "replicator\"\xd1\x04\n" + + "replicator\"\xe1\x04\n" + ")ReferenceExpandingBlobAccessConfiguration\x12\x8b\x01\n" + "$indirect_content_addressable_storage\x18\x01 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR!indirectContentAddressableStorage\x12X\n" + "\vaws_session\x18\x02 \x01(\v27.buildbarn.configuration.cloud.aws.SessionConfigurationR\n" + "awsSession\x12S\n" + "\vhttp_client\x18\x03 \x01(\v22.buildbarn.configuration.http.client.ConfigurationR\n" + "httpClient\x12k\n" + - "\x12gcp_client_options\x18\x04 \x01(\v2=.buildbarn.configuration.cloud.gcp.ClientOptionsConfigurationR\x10gcpClientOptions\x12z\n" + - "\x1bcontent_addressable_storage\x18\x05 \x01(\v2:.buildbarn.configuration.blobstore.BlobAccessConfigurationR\x19contentAddressableStorage\"\xa1\x04\n" + + "\x12gcp_client_options\x18\x04 \x01(\v2=.buildbarn.configuration.cloud.gcp.ClientOptionsConfigurationR\x10gcpClientOptions\x12\x89\x01\n" + + "\x1bcontent_addressable_storage\x18\x05 \x01(\v2I.buildbarn.configuration.blobstore.ContentAddressableStorageConfigurationR\x19contentAddressableStorage\"\xa1\x04\n" + "\x1bBlobReplicatorConfiguration\x12.\n" + "\x05local\x18\x01 \x01(\v2\x16.google.protobuf.EmptyH\x00R\x05local\x12K\n" + "\x06remote\x18\x02 \x01(\v21.buildbarn.configuration.grpc.ClientConfigurationH\x00R\x06remote\x12^\n" + @@ -2214,134 +2287,139 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob return file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDescData } -var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 30) +var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 31) var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_goTypes = []any{ - (*BlobstoreConfiguration)(nil), // 0: buildbarn.configuration.blobstore.BlobstoreConfiguration - (*BlobAccessConfiguration)(nil), // 1: buildbarn.configuration.blobstore.BlobAccessConfiguration - (*ReadCachingBlobAccessConfiguration)(nil), // 2: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration - (*ShardingBlobAccessConfiguration)(nil), // 3: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration - (*MirroredBlobAccessConfiguration)(nil), // 4: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration - (*LocalBlobAccessConfiguration)(nil), // 5: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration - (*ExistenceCachingBlobAccessConfiguration)(nil), // 6: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration - (*CompletenessCheckingBlobAccessConfiguration)(nil), // 7: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration - (*ReadFallbackBlobAccessConfiguration)(nil), // 8: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration - (*ReferenceExpandingBlobAccessConfiguration)(nil), // 9: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration - (*BlobReplicatorConfiguration)(nil), // 10: buildbarn.configuration.blobstore.BlobReplicatorConfiguration - (*QueuedBlobReplicatorConfiguration)(nil), // 11: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration - (*ConcurrencyLimitingBlobReplicatorConfiguration)(nil), // 12: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration - (*DemultiplexingBlobAccessConfiguration)(nil), // 13: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration - (*DemultiplexedBlobAccessConfiguration)(nil), // 14: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration - (*ActionResultExpiringBlobAccessConfiguration)(nil), // 15: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration - (*ReadCanaryingBlobAccessConfiguration)(nil), // 16: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration - (*ZIPBlobAccessConfiguration)(nil), // 17: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration - (*WithLabelsBlobAccessConfiguration)(nil), // 18: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration - (*DeadlineEnforcingBlobAccess)(nil), // 19: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess - (*GrpcBlobAccessConfiguration)(nil), // 20: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration - (*ChunkListValidatingBlobAccessConfiguration)(nil), // 21: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration - (*ShardingBlobAccessConfiguration_Shard)(nil), // 22: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard - nil, // 23: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry - (*LocalBlobAccessConfiguration_KeyLocationMapInMemory)(nil), // 24: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory - (*LocalBlobAccessConfiguration_BlocksInMemory)(nil), // 25: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory - (*LocalBlobAccessConfiguration_BlocksOnBlockDevice)(nil), // 26: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice - (*LocalBlobAccessConfiguration_Persistent)(nil), // 27: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent - nil, // 28: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry - nil, // 29: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry - (*status.Status)(nil), // 30: google.rpc.Status - (*blockdevice.Configuration)(nil), // 31: buildbarn.configuration.blockdevice.Configuration - (*v2.RepMaxCdcParams)(nil), // 32: build.bazel.remote.execution.v2.RepMaxCdcParams - (*digest.ExistenceCacheConfiguration)(nil), // 33: buildbarn.configuration.digest.ExistenceCacheConfiguration - (*aws.SessionConfiguration)(nil), // 34: buildbarn.configuration.cloud.aws.SessionConfiguration - (*client.Configuration)(nil), // 35: buildbarn.configuration.http.client.Configuration - (*gcp.ClientOptionsConfiguration)(nil), // 36: buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration - (*emptypb.Empty)(nil), // 37: google.protobuf.Empty - (*grpc.ClientConfiguration)(nil), // 38: buildbarn.configuration.grpc.ClientConfiguration - (*durationpb.Duration)(nil), // 39: google.protobuf.Duration - (*timestamppb.Timestamp)(nil), // 40: google.protobuf.Timestamp + (*ContentAddressableStorageConfiguration)(nil), // 0: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + (*BlobstoreConfiguration)(nil), // 1: buildbarn.configuration.blobstore.BlobstoreConfiguration + (*BlobAccessConfiguration)(nil), // 2: buildbarn.configuration.blobstore.BlobAccessConfiguration + (*ReadCachingBlobAccessConfiguration)(nil), // 3: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration + (*ShardingBlobAccessConfiguration)(nil), // 4: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration + (*MirroredBlobAccessConfiguration)(nil), // 5: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration + (*LocalBlobAccessConfiguration)(nil), // 6: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration + (*ExistenceCachingBlobAccessConfiguration)(nil), // 7: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration + (*CompletenessCheckingBlobAccessConfiguration)(nil), // 8: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration + (*ReadFallbackBlobAccessConfiguration)(nil), // 9: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration + (*ReferenceExpandingBlobAccessConfiguration)(nil), // 10: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration + (*BlobReplicatorConfiguration)(nil), // 11: buildbarn.configuration.blobstore.BlobReplicatorConfiguration + (*QueuedBlobReplicatorConfiguration)(nil), // 12: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration + (*ConcurrencyLimitingBlobReplicatorConfiguration)(nil), // 13: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration + (*DemultiplexingBlobAccessConfiguration)(nil), // 14: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration + (*DemultiplexedBlobAccessConfiguration)(nil), // 15: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration + (*ActionResultExpiringBlobAccessConfiguration)(nil), // 16: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration + (*ReadCanaryingBlobAccessConfiguration)(nil), // 17: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration + (*ZIPBlobAccessConfiguration)(nil), // 18: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration + (*WithLabelsBlobAccessConfiguration)(nil), // 19: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration + (*DeadlineEnforcingBlobAccess)(nil), // 20: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess + (*GrpcBlobAccessConfiguration)(nil), // 21: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration + (*ChunkListValidatingBlobAccessConfiguration)(nil), // 22: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration + (*ShardingBlobAccessConfiguration_Shard)(nil), // 23: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard + nil, // 24: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry + (*LocalBlobAccessConfiguration_KeyLocationMapInMemory)(nil), // 25: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory + (*LocalBlobAccessConfiguration_BlocksInMemory)(nil), // 26: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory + (*LocalBlobAccessConfiguration_BlocksOnBlockDevice)(nil), // 27: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice + (*LocalBlobAccessConfiguration_Persistent)(nil), // 28: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent + nil, // 29: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry + nil, // 30: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry + (*digest.ExistenceCacheConfiguration)(nil), // 31: buildbarn.configuration.digest.ExistenceCacheConfiguration + (*status.Status)(nil), // 32: google.rpc.Status + (*blockdevice.Configuration)(nil), // 33: buildbarn.configuration.blockdevice.Configuration + (*v2.RepMaxCdcParams)(nil), // 34: build.bazel.remote.execution.v2.RepMaxCdcParams + (*aws.SessionConfiguration)(nil), // 35: buildbarn.configuration.cloud.aws.SessionConfiguration + (*client.Configuration)(nil), // 36: buildbarn.configuration.http.client.Configuration + (*gcp.ClientOptionsConfiguration)(nil), // 37: buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration + (*emptypb.Empty)(nil), // 38: google.protobuf.Empty + (*grpc.ClientConfiguration)(nil), // 39: buildbarn.configuration.grpc.ClientConfiguration + (*durationpb.Duration)(nil), // 40: google.protobuf.Duration + (*timestamppb.Timestamp)(nil), // 41: google.protobuf.Timestamp } var file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_depIdxs = []int32{ - 1, // 0: buildbarn.configuration.blobstore.BlobstoreConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 1: buildbarn.configuration.blobstore.BlobstoreConfiguration.action_cache:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 2, // 2: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_caching:type_name -> buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration - 20, // 3: buildbarn.configuration.blobstore.BlobAccessConfiguration.grpc:type_name -> buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration - 30, // 4: buildbarn.configuration.blobstore.BlobAccessConfiguration.error:type_name -> google.rpc.Status - 3, // 5: buildbarn.configuration.blobstore.BlobAccessConfiguration.sharding:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration - 4, // 6: buildbarn.configuration.blobstore.BlobAccessConfiguration.mirrored:type_name -> buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration - 5, // 7: buildbarn.configuration.blobstore.BlobAccessConfiguration.local:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration - 6, // 8: buildbarn.configuration.blobstore.BlobAccessConfiguration.existence_caching:type_name -> buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration - 7, // 9: buildbarn.configuration.blobstore.BlobAccessConfiguration.completeness_checking:type_name -> buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration - 8, // 10: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_fallback:type_name -> buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration - 9, // 11: buildbarn.configuration.blobstore.BlobAccessConfiguration.reference_expanding:type_name -> buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration - 13, // 12: buildbarn.configuration.blobstore.BlobAccessConfiguration.demultiplexing:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration - 1, // 13: buildbarn.configuration.blobstore.BlobAccessConfiguration.hierarchical_instance_names:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 15, // 14: buildbarn.configuration.blobstore.BlobAccessConfiguration.action_result_expiring:type_name -> buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration - 16, // 15: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_canarying:type_name -> buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration - 17, // 16: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_reading:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration - 17, // 17: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_writing:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration - 18, // 18: buildbarn.configuration.blobstore.BlobAccessConfiguration.with_labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration - 19, // 19: buildbarn.configuration.blobstore.BlobAccessConfiguration.deadline_enforcing:type_name -> buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess - 21, // 20: buildbarn.configuration.blobstore.BlobAccessConfiguration.chunk_list_validating:type_name -> buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration - 1, // 21: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.slow:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 22: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.fast:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 23: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 23, // 24: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.shards:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry - 1, // 25: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_a:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 26: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_b:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 27: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_a_to_b:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 10, // 28: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_b_to_a:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 24, // 29: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory - 31, // 30: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_on_block_device:type_name -> buildbarn.configuration.blockdevice.Configuration - 25, // 31: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory - 26, // 32: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_on_block_device:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice - 27, // 33: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.persistent:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent - 32, // 34: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.chunking_parameters:type_name -> build.bazel.remote.execution.v2.RepMaxCdcParams - 1, // 35: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 33, // 36: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 1, // 37: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 38: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.primary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 39: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.secondary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 10, // 40: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 1, // 41: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 34, // 42: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.aws_session:type_name -> buildbarn.configuration.cloud.aws.SessionConfiguration - 35, // 43: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.http_client:type_name -> buildbarn.configuration.http.client.Configuration - 36, // 44: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.gcp_client_options:type_name -> buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration - 1, // 45: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 37, // 46: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.local:type_name -> google.protobuf.Empty - 38, // 47: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.remote:type_name -> buildbarn.configuration.grpc.ClientConfiguration - 11, // 48: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.queued:type_name -> buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration - 37, // 49: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.noop:type_name -> google.protobuf.Empty - 10, // 50: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.deduplicating:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 12, // 51: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.concurrency_limiting:type_name -> buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration - 10, // 52: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 33, // 53: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 10, // 54: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration - 28, // 55: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.instance_name_prefixes:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry - 1, // 56: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 57: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 39, // 58: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_validity:type_name -> google.protobuf.Duration - 39, // 59: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.maximum_validity_jitter:type_name -> google.protobuf.Duration - 40, // 60: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_timestamp:type_name -> google.protobuf.Timestamp - 1, // 61: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.source:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 62: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.replica:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 39, // 63: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.maximum_cache_duration:type_name -> google.protobuf.Duration - 33, // 64: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 1, // 65: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 29, // 66: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry - 39, // 67: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.timeout:type_name -> google.protobuf.Duration - 1, // 68: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 38, // 69: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration.client:type_name -> buildbarn.configuration.grpc.ClientConfiguration - 1, // 70: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 1, // 71: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 22, // 72: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry.value:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard - 31, // 73: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.source:type_name -> buildbarn.configuration.blockdevice.Configuration - 33, // 74: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration - 39, // 75: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent.minimum_epoch_interval:type_name -> google.protobuf.Duration - 14, // 76: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry.value:type_name -> buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration - 1, // 77: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry.value:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration - 78, // [78:78] is the sub-list for method output_type - 78, // [78:78] is the sub-list for method input_type - 78, // [78:78] is the sub-list for extension type_name - 78, // [78:78] is the sub-list for extension extendee - 0, // [0:78] is the sub-list for field type_name + 2, // 0: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration.chunk_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 1: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration.chunk_list_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 31, // 2: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration.chunk_list_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 31, // 3: buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration.cdc_parameter_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 0, // 4: buildbarn.configuration.blobstore.BlobstoreConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + 2, // 5: buildbarn.configuration.blobstore.BlobstoreConfiguration.action_cache:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 3, // 6: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_caching:type_name -> buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration + 21, // 7: buildbarn.configuration.blobstore.BlobAccessConfiguration.grpc:type_name -> buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration + 32, // 8: buildbarn.configuration.blobstore.BlobAccessConfiguration.error:type_name -> google.rpc.Status + 4, // 9: buildbarn.configuration.blobstore.BlobAccessConfiguration.sharding:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration + 5, // 10: buildbarn.configuration.blobstore.BlobAccessConfiguration.mirrored:type_name -> buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration + 6, // 11: buildbarn.configuration.blobstore.BlobAccessConfiguration.local:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration + 7, // 12: buildbarn.configuration.blobstore.BlobAccessConfiguration.existence_caching:type_name -> buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration + 8, // 13: buildbarn.configuration.blobstore.BlobAccessConfiguration.completeness_checking:type_name -> buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration + 9, // 14: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_fallback:type_name -> buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration + 10, // 15: buildbarn.configuration.blobstore.BlobAccessConfiguration.reference_expanding:type_name -> buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration + 14, // 16: buildbarn.configuration.blobstore.BlobAccessConfiguration.demultiplexing:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration + 2, // 17: buildbarn.configuration.blobstore.BlobAccessConfiguration.hierarchical_instance_names:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 16, // 18: buildbarn.configuration.blobstore.BlobAccessConfiguration.action_result_expiring:type_name -> buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration + 17, // 19: buildbarn.configuration.blobstore.BlobAccessConfiguration.read_canarying:type_name -> buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration + 18, // 20: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_reading:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration + 18, // 21: buildbarn.configuration.blobstore.BlobAccessConfiguration.zip_writing:type_name -> buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration + 19, // 22: buildbarn.configuration.blobstore.BlobAccessConfiguration.with_labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration + 20, // 23: buildbarn.configuration.blobstore.BlobAccessConfiguration.deadline_enforcing:type_name -> buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess + 22, // 24: buildbarn.configuration.blobstore.BlobAccessConfiguration.chunk_list_validating:type_name -> buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration + 2, // 25: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.slow:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 26: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.fast:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 11, // 27: buildbarn.configuration.blobstore.ReadCachingBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 24, // 28: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.shards:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry + 2, // 29: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_a:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 30: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.backend_b:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 11, // 31: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_a_to_b:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 11, // 32: buildbarn.configuration.blobstore.MirroredBlobAccessConfiguration.replicator_b_to_a:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 25, // 33: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.KeyLocationMapInMemory + 33, // 34: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.key_location_map_on_block_device:type_name -> buildbarn.configuration.blockdevice.Configuration + 26, // 35: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_in_memory:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksInMemory + 27, // 36: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.blocks_on_block_device:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice + 28, // 37: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.persistent:type_name -> buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent + 34, // 38: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.chunking_parameters:type_name -> build.bazel.remote.execution.v2.RepMaxCdcParams + 2, // 39: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 31, // 40: buildbarn.configuration.blobstore.ExistenceCachingBlobAccessConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 2, // 41: buildbarn.configuration.blobstore.CompletenessCheckingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 42: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.primary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 43: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.secondary:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 11, // 44: buildbarn.configuration.blobstore.ReadFallbackBlobAccessConfiguration.replicator:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 2, // 45: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.indirect_content_addressable_storage:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 35, // 46: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.aws_session:type_name -> buildbarn.configuration.cloud.aws.SessionConfiguration + 36, // 47: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.http_client:type_name -> buildbarn.configuration.http.client.Configuration + 37, // 48: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.gcp_client_options:type_name -> buildbarn.configuration.cloud.gcp.ClientOptionsConfiguration + 0, // 49: buildbarn.configuration.blobstore.ReferenceExpandingBlobAccessConfiguration.content_addressable_storage:type_name -> buildbarn.configuration.blobstore.ContentAddressableStorageConfiguration + 38, // 50: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.local:type_name -> google.protobuf.Empty + 39, // 51: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.remote:type_name -> buildbarn.configuration.grpc.ClientConfiguration + 12, // 52: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.queued:type_name -> buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration + 38, // 53: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.noop:type_name -> google.protobuf.Empty + 11, // 54: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.deduplicating:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 13, // 55: buildbarn.configuration.blobstore.BlobReplicatorConfiguration.concurrency_limiting:type_name -> buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration + 11, // 56: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 31, // 57: buildbarn.configuration.blobstore.QueuedBlobReplicatorConfiguration.existence_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 11, // 58: buildbarn.configuration.blobstore.ConcurrencyLimitingBlobReplicatorConfiguration.base:type_name -> buildbarn.configuration.blobstore.BlobReplicatorConfiguration + 29, // 59: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.instance_name_prefixes:type_name -> buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry + 2, // 60: buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 61: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 40, // 62: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_validity:type_name -> google.protobuf.Duration + 40, // 63: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.maximum_validity_jitter:type_name -> google.protobuf.Duration + 41, // 64: buildbarn.configuration.blobstore.ActionResultExpiringBlobAccessConfiguration.minimum_timestamp:type_name -> google.protobuf.Timestamp + 2, // 65: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.source:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 66: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.replica:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 40, // 67: buildbarn.configuration.blobstore.ReadCanaryingBlobAccessConfiguration.maximum_cache_duration:type_name -> google.protobuf.Duration + 31, // 68: buildbarn.configuration.blobstore.ZIPBlobAccessConfiguration.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 2, // 69: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 30, // 70: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.labels:type_name -> buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry + 40, // 71: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.timeout:type_name -> google.protobuf.Duration + 2, // 72: buildbarn.configuration.blobstore.DeadlineEnforcingBlobAccess.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 39, // 73: buildbarn.configuration.blobstore.GrpcBlobAccessConfiguration.client:type_name -> buildbarn.configuration.grpc.ClientConfiguration + 2, // 74: buildbarn.configuration.blobstore.ChunkListValidatingBlobAccessConfiguration.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 2, // 75: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard.backend:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 23, // 76: buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.ShardsEntry.value:type_name -> buildbarn.configuration.blobstore.ShardingBlobAccessConfiguration.Shard + 33, // 77: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.source:type_name -> buildbarn.configuration.blockdevice.Configuration + 31, // 78: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.BlocksOnBlockDevice.data_integrity_validation_cache:type_name -> buildbarn.configuration.digest.ExistenceCacheConfiguration + 40, // 79: buildbarn.configuration.blobstore.LocalBlobAccessConfiguration.Persistent.minimum_epoch_interval:type_name -> google.protobuf.Duration + 15, // 80: buildbarn.configuration.blobstore.DemultiplexingBlobAccessConfiguration.InstanceNamePrefixesEntry.value:type_name -> buildbarn.configuration.blobstore.DemultiplexedBlobAccessConfiguration + 2, // 81: buildbarn.configuration.blobstore.WithLabelsBlobAccessConfiguration.LabelsEntry.value:type_name -> buildbarn.configuration.blobstore.BlobAccessConfiguration + 82, // [82:82] is the sub-list for method output_type + 82, // [82:82] is the sub-list for method input_type + 82, // [82:82] is the sub-list for extension type_name + 82, // [82:82] is the sub-list for extension extendee + 0, // [0:82] is the sub-list for field type_name } func init() { @@ -2351,7 +2429,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob if File_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto != nil { return } - file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[1].OneofWrappers = []any{ + file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[2].OneofWrappers = []any{ (*BlobAccessConfiguration_ReadCaching)(nil), (*BlobAccessConfiguration_Grpc)(nil), (*BlobAccessConfiguration_Error)(nil), @@ -2373,13 +2451,13 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob (*BlobAccessConfiguration_DeadlineEnforcing)(nil), (*BlobAccessConfiguration_ChunkListValidating)(nil), } - file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[5].OneofWrappers = []any{ + file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[6].OneofWrappers = []any{ (*LocalBlobAccessConfiguration_KeyLocationMapInMemory_)(nil), (*LocalBlobAccessConfiguration_KeyLocationMapOnBlockDevice)(nil), (*LocalBlobAccessConfiguration_BlocksInMemory_)(nil), (*LocalBlobAccessConfiguration_BlocksOnBlockDevice_)(nil), } - file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[10].OneofWrappers = []any{ + file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_msgTypes[11].OneofWrappers = []any{ (*BlobReplicatorConfiguration_Local)(nil), (*BlobReplicatorConfiguration_Remote)(nil), (*BlobReplicatorConfiguration_Queued)(nil), @@ -2393,7 +2471,7 @@ func file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blob GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc), len(file_github_com_buildbarn_bb_storage_pkg_proto_configuration_blobstore_blobstore_proto_rawDesc)), NumEnums: 0, - NumMessages: 30, + NumMessages: 31, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/configuration/blobstore/blobstore.proto b/pkg/proto/configuration/blobstore/blobstore.proto index 917df5514..734c05d1e 100644 --- a/pkg/proto/configuration/blobstore/blobstore.proto +++ b/pkg/proto/configuration/blobstore/blobstore.proto @@ -16,10 +16,35 @@ import "google/rpc/status.proto"; option go_package = "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore"; +message ContentAddressableStorageConfiguration { + // Storage configuration for the Chunk Storage (CS). + BlobAccessConfiguration chunk_storage = 1; + + // Storage configuration for the Chunk List Storage (CLS). + BlobAccessConfiguration chunk_list_storage = 2; + + // Optional: In memory cache for parsed object from the CLS. This + // cache is particularly useful for consumers which need to random + // access reads to the same digest often such as bb-clientd and + // bb-worker. + buildbarn.configuration.digest.ExistenceCacheConfiguration chunk_list_cache = + 3; + + // Optional: In memory cache for CDC parameters per instance. When the + // chunk storage is remote this cache prevents each operation that + // interacts with CDC from having to fetch the capabilities from the + // upstream CS for every interaction with the CDC subsystem. It can be + // set to a relatively small cache with short retention as the number + // of instance names is typically small and the cost of a cache miss + // is relatively low. + buildbarn.configuration.digest.ExistenceCacheConfiguration + cdc_parameter_cache = 4; +} + // Storage configuration for Bazel Buildbarn. message BlobstoreConfiguration { // Storage configuration for the Content Addressable Storage (CAS). - BlobAccessConfiguration content_addressable_storage = 1; + ContentAddressableStorageConfiguration content_addressable_storage = 1; // Storage configuration for the Action Cache (AC). BlobAccessConfiguration action_cache = 2; @@ -201,27 +226,26 @@ message BlobAccessConfiguration { // code will be returned. DeadlineEnforcingBlobAccess deadline_enforcing = 28; - // Validate that chunk list requests fulfill the api specification. + // Validate that chunk list requests fulfill the API specification. // Chunk List Storage (CLS) is used to implement the SplitBlob and - // SpliceBlob methods from the REv2 api. The protocol has several + // SpliceBlob methods from the REv2 API. The protocol has several // demands on the state of the Content Addressable Storage (CAS) // after those methods have been called. // - // SplitBlob requires that that the blob as well as all chunks of - // the blob are available in the CAS, they have their lifetime - // renewed and that the resulting chunk list composes into the blob. - // This is validated by the Get path by splitting the Blob on demand - // if required. + // SplitBlob requires that the blob as well as all chunks of the + // blob are available in the CAS, they have their lifetime renewed + // and that the resulting chunk list composes into the blob. This is + // validated by the Get path. // // SpliceBlob requires that the blob as well as all chunks of the // blob are present in storage, they have their lifetime renewed and // that the supplied chunk list composes into the blob. Notably it // does not require the chunks to follow any particular chunking // algorithm but our implementation ensures that after any call a - // proper rep max cdc chunk list is verified even if the caller + // proper RepMaxCDC chunk list is verified even if the caller // supplied a different chunk list. // - // This decorator must be placed on the CLS. + // This decorator is only supported for the CLS. ChunkListValidatingBlobAccessConfiguration chunk_list_validating = 29; } @@ -676,7 +700,7 @@ message LocalBlobAccessConfiguration { // an outer ChunkValidatingBlobAccess that has view of the entire // Content Addressable Storage (CAS). // - // This option is only supported for the CLS. + // This option is only supported for the CS. build.bazel.remote.execution.v2.RepMaxCdcParams chunking_parameters = 15; } @@ -741,7 +765,7 @@ message ReferenceExpandingBlobAccessConfiguration { // Optional: Storage backend to use when Reference objects refer to // objects stored in another Content Addressable Storage. - BlobAccessConfiguration content_addressable_storage = 5; + ContentAddressableStorageConfiguration content_addressable_storage = 5; } message BlobReplicatorConfiguration { diff --git a/pkg/testutil/testutil.go b/pkg/testutil/testutil.go index 6fc3172b0..c5b1a3e16 100644 --- a/pkg/testutil/testutil.go +++ b/pkg/testutil/testutil.go @@ -14,6 +14,24 @@ import ( "go.uber.org/mock/gomock" ) +// MustMarshal is a helper that requires a proto message to succesfully +// marshal to a slice of bytes. +func MustMarshal(t *testing.T, msg proto.Message) []byte { + t.Helper() + b, err := proto.Marshal(msg) + require.NoError(t, err) + return b +} + +// MustUnmarshal is a helper that requires a byte slice to unmarshal +// into a proto message. +func MustUnmarshal[T proto.Message](t *testing.T, data []byte, m T) T { + t.Helper() + err := proto.Unmarshal(data, m) + require.NoError(t, err) + return m +} + // RequireEqualProto asserts that the two passed protocol buffer // messages are equal. // diff --git a/pkg/ttlcache/BUILD.bazel b/pkg/ttlcache/BUILD.bazel new file mode 100644 index 000000000..ce11025da --- /dev/null +++ b/pkg/ttlcache/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "ttlcache", + srcs = ["ttl_cache.go"], + importpath = "github.com/buildbarn/bb-storage/pkg/ttlcache", + visibility = ["//visibility:public"], + deps = [ + "//pkg/clock", + "//pkg/eviction", + "//pkg/proto/configuration/digest", + "//pkg/util", + ], +) diff --git a/pkg/ttlcache/ttl_cache.go b/pkg/ttlcache/ttl_cache.go new file mode 100644 index 000000000..2e2039a49 --- /dev/null +++ b/pkg/ttlcache/ttl_cache.go @@ -0,0 +1,97 @@ +package ttlcache + +import ( + "sync" + "time" + + "github.com/buildbarn/bb-storage/pkg/clock" + "github.com/buildbarn/bb-storage/pkg/eviction" + digest_pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/digest" + "github.com/buildbarn/bb-storage/pkg/util" +) + +type cachedItem[V any] struct { + value V + expiration time.Time +} + +// TTLCache provides a generic, concurrency-safe cache with TTL and +// eviction. +type TTLCache[K comparable, V any] struct { + clock clock.Clock + evictionSet eviction.Set[K] + maxItems int + cacheDuration time.Duration + + lock sync.Mutex + items map[K]cachedItem[V] +} + +// NewTTLCache instantiates a reusable TTLCache for any key-value pair. +func NewTTLCache[K comparable, V any](clock clock.Clock, evictionSet eviction.Set[K], maxItems int, cacheDuration time.Duration) *TTLCache[K, V] { + return &TTLCache[K, V]{ + clock: clock, + evictionSet: evictionSet, + maxItems: maxItems, + cacheDuration: cacheDuration, + items: make(map[K]cachedItem[V]), + } +} + +// NewTTLCacheFromConfiguration wraps NewTTLCache with the parameters +// specified in a configuration message. +func NewTTLCacheFromConfiguration[K comparable, V any](configuration *digest_pb.ExistenceCacheConfiguration, clock clock.Clock, name string) (*TTLCache[K, V], error) { + cacheDuration := configuration.CacheDuration + if err := cacheDuration.CheckValid(); err != nil { + return nil, util.StatusWrap(err, "Invalid cache duration") + } + evictionSet, err := eviction.NewSetFromConfiguration[K](configuration.CacheReplacementPolicy) + if err != nil { + return nil, util.StatusWrap(err, "Failed to create eviction set") + } + return NewTTLCache[K, V]( + clock, + eviction.NewMetricsSet(evictionSet, name), + int(configuration.CacheSize), + cacheDuration.AsDuration(), + ), nil +} + +// Get retrieves an item if it exists and hasn't expired. +func (c *TTLCache[K, V]) Get(key K) (V, bool) { + c.lock.Lock() + defer c.lock.Unlock() + + if cached, ok := c.items[key]; ok { + if !c.clock.Now().After(cached.expiration) { + c.evictionSet.Touch(key) + return cached.value, true + } + } + + var zero V + return zero, false +} + +// Put inserts or updates an item in the cache, handling eviction if at +// capacity. +func (c *TTLCache[K, V]) Put(key K, value V) { + c.lock.Lock() + defer c.lock.Unlock() + + expiration := c.clock.Now().Add(c.cacheDuration) + + if _, ok := c.items[key]; ok { + c.items[key] = cachedItem[V]{value: value, expiration: expiration} + c.evictionSet.Touch(key) + return + } + + if len(c.items) >= c.maxItems { + delete(c.items, c.evictionSet.Peek()) + c.evictionSet.Remove() + } + + c.items[key] = cachedItem[V]{value: value, expiration: expiration} + c.evictionSet.Insert(key) +} From 2f4d4b71544e4b90623c406df739ae6dee758c84 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 24 Aug 2026 08:55:19 +0200 Subject: [PATCH 4/6] Move ContentAddressableStorage to pkg/cas --- cmd/bb_storage/BUILD.bazel | 4 +--- cmd/bb_storage/main.go | 12 ++++++---- internal/mock/BUILD.bazel | 15 +++++++++++- pkg/blobstore/cdc/BUILD.bazel | 8 ------- .../completenesschecking/BUILD.bazel | 2 +- .../completeness_checking_blob_access.go | 10 ++++---- pkg/blobstore/configuration/BUILD.bazel | 1 + .../configuration/ac_blob_access_creator.go | 6 ++--- .../configuration/cs_blob_access_creator.go | 4 ++-- .../configuration/new_blob_access.go | 7 +++--- pkg/blobstore/grpcservers/BUILD.bazel | 2 +- .../grpcservers/byte_stream_server.go | 12 +++++----- .../content_addressable_storage_server.go | 12 +++++----- pkg/blobstore/referenceexpanding/BUILD.bazel | 2 +- .../reference_expanding_blob_access.go | 8 +++---- pkg/cas/BUILD.bazel | 24 +++++++++++++++++++ .../content_addressable_storage.go | 13 +++++----- pkg/{blobstore/cdc => cas}/operations.go | 9 +++---- pkg/{blobstore/cdc => cas}/put_proto.go | 2 +- pkg/{blobstore/cdc => cas}/read_blob_at.go | 2 +- 20 files changed, 94 insertions(+), 61 deletions(-) create mode 100644 pkg/cas/BUILD.bazel rename pkg/{blobstore/cdc => cas}/content_addressable_storage.go (92%) rename pkg/{blobstore/cdc => cas}/operations.go (95%) rename pkg/{blobstore/cdc => cas}/put_proto.go (98%) rename pkg/{blobstore/cdc => cas}/read_blob_at.go (99%) diff --git a/cmd/bb_storage/BUILD.bazel b/cmd/bb_storage/BUILD.bazel index 230e2d907..58b47e035 100644 --- a/cmd/bb_storage/BUILD.bazel +++ b/cmd/bb_storage/BUILD.bazel @@ -15,8 +15,7 @@ go_library( "//pkg/blobstore/grpcservers", "//pkg/builder", "//pkg/capabilities", - "//pkg/clock", - "//pkg/digest", + "//pkg/cas", "//pkg/global", "//pkg/grpc", "//pkg/program", @@ -24,7 +23,6 @@ go_library( "//pkg/proto/fsac", "//pkg/proto/icas", "//pkg/proto/iscc", - "//pkg/ttlcache", "//pkg/util", "//pkg/zstd", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/cmd/bb_storage/main.go b/cmd/bb_storage/main.go index 9a7c7e288..9ef2a3438 100644 --- a/cmd/bb_storage/main.go +++ b/cmd/bb_storage/main.go @@ -14,6 +14,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/grpcservers" "github.com/buildbarn/bb-storage/pkg/builder" "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/global" bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc" "github.com/buildbarn/bb-storage/pkg/program" @@ -55,10 +56,12 @@ func main() { var cacheCapabilitiesAuthorizers []auth.Authorizer // Content Addressable Storage (CAS). - var contentAddressableStorage cdc.ContentAddressableStorage - var authorizedContentAddressableStorage cdc.ContentAddressableStorage + var contentAddressableStorage cas.ContentAddressableStorage + var authorizedContentAddressableStorage cas.ContentAddressableStorage if configuration.ContentAddressableStorageServer != nil { - cas, chunkStorage, chunkListStorage, _, cdcParametersFetcher, err := blobstore_configuration.NewCASFromConfiguration( + var chunkStorage, chunkListStorage blobstore.BlobAccess + var cdcParametersFetcher cdc.ParametersFetcher + contentAddressableStorage, chunkStorage, chunkListStorage, _, cdcParametersFetcher, err = blobstore_configuration.NewCASFromConfiguration( dependenciesGroup, configuration.ContentAddressableStorageServer.ContentAddressableStorage, grpcClientFactory, @@ -68,7 +71,6 @@ func main() { if err != nil { return util.StatusWrap(err, "Failed to create Content Addressable Storage") } - contentAddressableStorage = cas // Create authorizers. getAuthorizer, err := auth_configuration.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(configuration.ContentAddressableStorageServer.GetAuthorizer, dependenciesGroup, grpcClientFactory) @@ -88,7 +90,7 @@ func main() { authorizedChunkStorage := blobstore.NewAuthorizingBlobAccess(chunkStorage, getAuthorizer, putAuthorizer, findMissingAuthorizer) authorizedChunkListStorage := blobstore.NewAuthorizingBlobAccess(chunkListStorage, getAuthorizer, putAuthorizer, findMissingAuthorizer) authorizedChunkListFetcher := blobstore.NewBlobAccessChunkListFetcher(authorizedChunkListStorage, int(configuration.MaximumMessageSizeBytes)) - authorizedContentAddressableStorage = cdc.NewContentAddressableStorage( + authorizedContentAddressableStorage = cas.NewContentAddressableStorage( authorizedChunkStorage, authorizedChunkListStorage, authorizedChunkListFetcher, diff --git a/internal/mock/BUILD.bazel b/internal/mock/BUILD.bazel index 8211c444a..6e130b604 100644 --- a/internal/mock/BUILD.bazel +++ b/internal/mock/BUILD.bazel @@ -152,11 +152,22 @@ gomock( package = "mock", ) +gomock( + name = "cas", + out = "cas.go", + interfaces = [ + "ContentAddressableStorage", + ], + library = "//pkg/cas", + mockgen_model_library = "@org_uber_go_mock//mockgen/model", + mockgen_tool = "@org_uber_go_mock//mockgen", + package = "mock", +) + gomock( name = "cdc", out = "cdc.go", interfaces = [ - "ContentAddressableStorage", "ParametersFetcher", ], library = "//pkg/blobstore/cdc", @@ -390,6 +401,7 @@ go_library( "buffer.go", "builder.go", "capabilities.go", + "cas.go", "cdc.go", "clock.go", "cloud_aws.go", @@ -422,6 +434,7 @@ go_library( "//pkg/blobstore/sharding", "//pkg/blobstore/slicing", "//pkg/builder", + "//pkg/cas", "//pkg/clock", "//pkg/cloud/gcp", "//pkg/digest", diff --git a/pkg/blobstore/cdc/BUILD.bazel b/pkg/blobstore/cdc/BUILD.bazel index 41be1685d..241c99b54 100644 --- a/pkg/blobstore/cdc/BUILD.bazel +++ b/pkg/blobstore/cdc/BUILD.bazel @@ -6,19 +6,12 @@ go_library( "bypass.go", "caching_parameters_fetcher.go", "chunker.go", - "content_addressable_storage.go", - "operations.go", "parameters.go", - "put_proto.go", - "read_blob_at.go", "reader_chunker.go", ], importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/cdc", visibility = ["//visibility:public"], deps = [ - "//pkg/blobstore", - "//pkg/blobstore/buffer", - "//pkg/blobstore/chunklist", "//pkg/capabilities", "//pkg/digest", "//pkg/ttlcache", @@ -26,7 +19,6 @@ go_library( "@com_github_buildbarn_go_cdc//:go-cdc", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", - "@org_golang_google_protobuf//proto", ], ) diff --git a/pkg/blobstore/completenesschecking/BUILD.bazel b/pkg/blobstore/completenesschecking/BUILD.bazel index 33a9d5b79..21028340d 100644 --- a/pkg/blobstore/completenesschecking/BUILD.bazel +++ b/pkg/blobstore/completenesschecking/BUILD.bazel @@ -8,8 +8,8 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/cdc", "//pkg/blobstore/slicing", + "//pkg/cas", "//pkg/digest", "//pkg/util", "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto", diff --git a/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go b/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go index 115aad87b..cd7663a5f 100644 --- a/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go +++ b/pkg/blobstore/completenesschecking/completeness_checking_blob_access.go @@ -7,8 +7,8 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -22,7 +22,7 @@ import ( type findMissingQueue struct { context context.Context digestFunction digest.Function - contentAddressableStorage cdc.ContentAddressableStorage + contentAddressableStorage cas.ContentAddressableStorage batchSize int pending digest.SetBuilder @@ -74,7 +74,7 @@ func (q *findMissingQueue) finalize() error { type completenessCheckingBlobAccess struct { blobstore.BlobAccess - contentAddressableStorage cdc.ContentAddressableStorage + contentAddressableStorage cas.ContentAddressableStorage batchSize int maximumMessageSizeBytes int maximumTotalTreeSizeBytes int64 @@ -94,7 +94,7 @@ type completenessCheckingBlobAccess struct { // needs to be rebuilt. By calling it, Bazel indicates that all // associated output files must remain present during the build for // forward progress to be made. -func NewCompletenessCheckingBlobAccess(actionCache blobstore.BlobAccess, contentAddressableStorage cdc.ContentAddressableStorage, batchSize, maximumMessageSizeBytes int, maximumTotalTreeSizeBytes int64) blobstore.BlobAccess { +func NewCompletenessCheckingBlobAccess(actionCache blobstore.BlobAccess, contentAddressableStorage cas.ContentAddressableStorage, batchSize, maximumMessageSizeBytes int, maximumTotalTreeSizeBytes int64) blobstore.BlobAccess { return &completenessCheckingBlobAccess{ BlobAccess: actionCache, contentAddressableStorage: contentAddressableStorage, @@ -152,7 +152,7 @@ func (ba *completenessCheckingBlobAccess) checkCompleteness(ctx context.Context, return status.Errorf(codes.NotFound, "Combined size of all output directories exceeds maximum limit of %d bytes", ba.maximumTotalTreeSizeBytes) } remainingTreeSizeBytes -= sizeBytes - r, err := cdc.GetReadCloser(ctx, ba.contentAddressableStorage, treeDigest) + r, err := cas.GetReadCloser(ctx, ba.contentAddressableStorage, treeDigest) if err != nil { return err } diff --git a/pkg/blobstore/configuration/BUILD.bazel b/pkg/blobstore/configuration/BUILD.bazel index 79c4cedef..580c67a57 100644 --- a/pkg/blobstore/configuration/BUILD.bazel +++ b/pkg/blobstore/configuration/BUILD.bazel @@ -36,6 +36,7 @@ go_library( "//pkg/blobstore/sharding", "//pkg/blockdevice", "//pkg/capabilities", + "//pkg/cas", "//pkg/clock", "//pkg/cloud/aws", "//pkg/cloud/gcp", diff --git a/pkg/blobstore/configuration/ac_blob_access_creator.go b/pkg/blobstore/configuration/ac_blob_access_creator.go index b521dabb7..b31a95fb3 100644 --- a/pkg/blobstore/configuration/ac_blob_access_creator.go +++ b/pkg/blobstore/configuration/ac_blob_access_creator.go @@ -3,10 +3,10 @@ package configuration import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" - "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/completenesschecking" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/grpc" @@ -32,7 +32,7 @@ type acBlobAccessCreator struct { protoBlobAccessCreator protoBlobReplicatorCreator - contentAddressableStorage cdc.ContentAddressableStorage + contentAddressableStorage cas.ContentAddressableStorage grpcClientFactory grpc.ClientFactory maximumMessageSizeBytes int } @@ -40,7 +40,7 @@ type acBlobAccessCreator struct { // NewACBlobAccessCreator creates a BlobAccessCreator that can be // provided to NewBlobAccessFromConfiguration() to construct a // BlobAccess that is suitable for accessing the Action Cache. -func NewACBlobAccessCreator(contentAddressableStorage cdc.ContentAddressableStorage, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { +func NewACBlobAccessCreator(contentAddressableStorage cas.ContentAddressableStorage, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int) BlobAccessCreator { return &acBlobAccessCreator{ contentAddressableStorage: contentAddressableStorage, grpcClientFactory: grpcClientFactory, diff --git a/pkg/blobstore/configuration/cs_blob_access_creator.go b/pkg/blobstore/configuration/cs_blob_access_creator.go index 87db8201b..043d344bf 100644 --- a/pkg/blobstore/configuration/cs_blob_access_creator.go +++ b/pkg/blobstore/configuration/cs_blob_access_creator.go @@ -8,11 +8,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" - "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/grpcclients" "github.com/buildbarn/bb-storage/pkg/blobstore/local" "github.com/buildbarn/bb-storage/pkg/blobstore/referenceexpanding" "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/cloud/aws" "github.com/buildbarn/bb-storage/pkg/cloud/gcp" "github.com/buildbarn/bb-storage/pkg/digest" @@ -127,7 +127,7 @@ func (bac *csBlobAccessCreator) NewCustomBlobAccess(terminationGroup program.Gro return BlobAccessInfo{}, "", err } - var contentAddressableStorage cdc.ContentAddressableStorage + var contentAddressableStorage cas.ContentAddressableStorage if backend.ReferenceExpanding.ContentAddressableStorage != nil { contentAddressableStorage, _, _, _, _, err = NewCASFromConfiguration(terminationGroup, backend.ReferenceExpanding.ContentAddressableStorage, bac.grpcClientFactory, bac.maximumMessageSizeBytes, bac.zstdPool) } else { diff --git a/pkg/blobstore/configuration/new_blob_access.go b/pkg/blobstore/configuration/new_blob_access.go index fff54563a..39a1e93f7 100644 --- a/pkg/blobstore/configuration/new_blob_access.go +++ b/pkg/blobstore/configuration/new_blob_access.go @@ -18,6 +18,7 @@ import ( "github.com/buildbarn/bb-storage/pkg/blobstore/sharding" "github.com/buildbarn/bb-storage/pkg/blockdevice" "github.com/buildbarn/bb-storage/pkg/capabilities" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/clock" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/eviction" @@ -640,7 +641,7 @@ func NewBlobAccessFromConfiguration(terminationGroup program.Group, configuratio // Content Addressable Storage (CAS) and a BlobAccess for the Action // Cache. Most Buildbarn components tend to require access to both these // data stores. -func NewCASAndACFromConfiguration(terminationGroup program.Group, configuration *pb.BlobstoreConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (cdc.ContentAddressableStorage, blobstore.BlobAccess, error) { +func NewCASAndACFromConfiguration(terminationGroup program.Group, configuration *pb.BlobstoreConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (cas.ContentAddressableStorage, blobstore.BlobAccess, error) { contentAddressableStorage, _, _, _, _, err := NewCASFromConfiguration(terminationGroup, configuration.ContentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes, zstdPool) if err != nil { return nil, nil, util.StatusWrap(err, "Failed to create Content Addressable Storage") @@ -665,7 +666,7 @@ func NewCASAndACFromConfiguration(terminationGroup program.Group, configuration // NewCASFromConfiguration is a convenience function to create a // ContentAddressableStorage (CAS) and its constituent parts from // configuration. -func NewCASFromConfiguration(terminationGroup program.Group, configuration *pb.ContentAddressableStorageConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (cdc.ContentAddressableStorage, blobstore.BlobAccess, blobstore.BlobAccess, chunklist.Fetcher, cdc.ParametersFetcher, error) { +func NewCASFromConfiguration(terminationGroup program.Group, configuration *pb.ContentAddressableStorageConfiguration, grpcClientFactory grpc.ClientFactory, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) (cas.ContentAddressableStorage, blobstore.BlobAccess, blobstore.BlobAccess, chunklist.Fetcher, cdc.ParametersFetcher, error) { chunkStorageInfo, err := NewBlobAccessFromConfiguration( terminationGroup, configuration.GetChunkStorage(), @@ -713,7 +714,7 @@ func NewCASFromConfiguration(terminationGroup program.Group, configuration *pb.C cdcParametersFetcher = cdc.NewCachingParametersFetcher(cdcParametersFetcher, cache) } - contentAddressableStorage := cdc.NewContentAddressableStorage(chunkStorage, chunkListStorage, chunkListFetcher, cdcParametersFetcher, chunkStorageInfo.DigestKeyFormat.Combine(chunkListStorageInfo.DigestKeyFormat)) + contentAddressableStorage := cas.NewContentAddressableStorage(chunkStorage, chunkListStorage, chunkListFetcher, cdcParametersFetcher, chunkStorageInfo.DigestKeyFormat.Combine(chunkListStorageInfo.DigestKeyFormat)) return contentAddressableStorage, chunkStorage, chunkListStorage, chunkListFetcher, cdcParametersFetcher, nil } diff --git a/pkg/blobstore/grpcservers/BUILD.bazel b/pkg/blobstore/grpcservers/BUILD.bazel index d2ea7ebf7..0216ac6c2 100644 --- a/pkg/blobstore/grpcservers/BUILD.bazel +++ b/pkg/blobstore/grpcservers/BUILD.bazel @@ -15,8 +15,8 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/cdc", "//pkg/blobstore/chunklist", + "//pkg/cas", "//pkg/digest", "//pkg/proto/fsac", "//pkg/proto/icas", diff --git a/pkg/blobstore/grpcservers/byte_stream_server.go b/pkg/blobstore/grpcservers/byte_stream_server.go index ffda7b44b..60b196aee 100644 --- a/pkg/blobstore/grpcservers/byte_stream_server.go +++ b/pkg/blobstore/grpcservers/byte_stream_server.go @@ -6,7 +6,7 @@ import ( "io" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" - "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" bb_zstd "github.com/buildbarn/bb-storage/pkg/zstd" @@ -17,14 +17,14 @@ import ( ) type byteStreamServer struct { - contentAddressableStorage cdc.ContentAddressableStorage + contentAddressableStorage cas.ContentAddressableStorage zstdPool bb_zstd.Pool } // NewByteStreamServer creates a GRPC service for reading blobs from and // writing blobs to a BlobAccess. It is used by Bazel to access the // Content Addressable Storage (CAS). -func NewByteStreamServer(contentAddressableStorage cdc.ContentAddressableStorage, zstdPool bb_zstd.Pool) bytestream.ByteStreamServer { +func NewByteStreamServer(contentAddressableStorage cas.ContentAddressableStorage, zstdPool bb_zstd.Pool) bytestream.ByteStreamServer { return &byteStreamServer{ contentAddressableStorage: contentAddressableStorage, zstdPool: zstdPool, @@ -57,11 +57,11 @@ func (s *byteStreamServer) Read(in *bytestream.ReadRequest, out bytestream.ByteS } defer encoder.Close() w = encoder - return cdc.IntoWriter(ctx, s.contentAddressableStorage, d, in.ReadOffset, encoder) + return cas.IntoWriter(ctx, s.contentAddressableStorage, d, in.ReadOffset, encoder) default: return status.Errorf(codes.Unimplemented, "This service does not support downloading compression type: %s", compressor) } - return cdc.IntoWriter(ctx, s.contentAddressableStorage, d, in.ReadOffset, w) + return cas.IntoWriter(ctx, s.contentAddressableStorage, d, in.ReadOffset, w) } // readStreamWriter adapts the ByteStream_ReadServer to an io.Writer. @@ -113,7 +113,7 @@ func (s *byteStreamServer) Write(stream bytestream.ByteStream_WriteServer) error return status.Errorf(codes.Unimplemented, "This service does not support uploading compression type: %s", compressor) } - if err := cdc.PutReader(ctx, s.contentAddressableStorage, d, r); err != nil { + if err := cas.PutReader(ctx, s.contentAddressableStorage, d, r); err != nil { return err } return stream.SendAndClose(&bytestream.WriteResponse{ diff --git a/pkg/blobstore/grpcservers/content_addressable_storage_server.go b/pkg/blobstore/grpcservers/content_addressable_storage_server.go index 9fb747179..4f1631c65 100644 --- a/pkg/blobstore/grpcservers/content_addressable_storage_server.go +++ b/pkg/blobstore/grpcservers/content_addressable_storage_server.go @@ -4,8 +4,8 @@ import ( "context" remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" - "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" + "github.com/buildbarn/bb-storage/pkg/cas" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -14,13 +14,13 @@ import ( ) type contentAddressableStorageServer struct { - contentAddressableStorage cdc.ContentAddressableStorage + contentAddressableStorage cas.ContentAddressableStorage maximumMessageSizeBytes int64 } // NewContentAddressableStorageServer creates a GRPC service for serving // the contents of a Bazel Content Addressable Storage (CAS) to Bazel. -func NewContentAddressableStorageServer(contentAddressableStorage cdc.ContentAddressableStorage, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { +func NewContentAddressableStorageServer(contentAddressableStorage cas.ContentAddressableStorage, maximumMessageSizeBytes int64) remoteexecution.ContentAddressableStorageServer { return &contentAddressableStorageServer{ contentAddressableStorage: contentAddressableStorage, maximumMessageSizeBytes: maximumMessageSizeBytes, @@ -102,7 +102,7 @@ func (s *contentAddressableStorageServer) BatchReadBlobs(ctx context.Context, in Responses: make([]*remoteexecution.BatchReadBlobsResponse_Response, 0, len(in.Digests)), } for i, reqDigest := range in.Digests { - data, err := cdc.GetBytes( + data, err := cas.GetBytes( ctx, s.contentAddressableStorage, digests[i], @@ -136,7 +136,7 @@ func (s *contentAddressableStorageServer) BatchUpdateBlobs(ctx context.Context, for _, request := range in.Requests { digest, err := digestFunction.NewDigestFromProto(request.Digest) if err == nil { - err = cdc.PutBytes( + err = cas.PutBytes( ctx, s.contentAddressableStorage, digest, @@ -210,7 +210,7 @@ func (s *contentAddressableStorageServer) SplitBlob(ctx context.Context, in *rem if err != nil { return nil, err } - if cdc.IsSingleChunk(params, blobDigest) { + if cas.IsSingleChunk(params, blobDigest) { missing, err := s.contentAddressableStorage.FindMissing(ctx, blobDigest.ToSingletonSet()) if err != nil { return nil, util.StatusWrap(err, "Failed to check blob existence") diff --git a/pkg/blobstore/referenceexpanding/BUILD.bazel b/pkg/blobstore/referenceexpanding/BUILD.bazel index 3fe307b6b..1c3c22586 100644 --- a/pkg/blobstore/referenceexpanding/BUILD.bazel +++ b/pkg/blobstore/referenceexpanding/BUILD.bazel @@ -8,8 +8,8 @@ go_library( deps = [ "//pkg/blobstore", "//pkg/blobstore/buffer", - "//pkg/blobstore/cdc", "//pkg/blobstore/slicing", + "//pkg/cas", "//pkg/cloud/aws", "//pkg/cloud/gcp", "//pkg/digest", diff --git a/pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go index 2ecdb2026..c66d0b787 100644 --- a/pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go +++ b/pkg/blobstore/referenceexpanding/reference_expanding_blob_access.go @@ -13,8 +13,8 @@ import ( remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" - "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/slicing" + "github.com/buildbarn/bb-storage/pkg/cas" cloud_aws "github.com/buildbarn/bb-storage/pkg/cloud/aws" cloud_gcp "github.com/buildbarn/bb-storage/pkg/cloud/gcp" "github.com/buildbarn/bb-storage/pkg/digest" @@ -28,7 +28,7 @@ import ( type referenceExpandingBlobAccess struct { indirectContentAddressableStorage blobstore.BlobAccess - contentAddressableStorage cdc.ContentAddressableStorage + contentAddressableStorage cas.ContentAddressableStorage httpClient *http.Client s3Client cloud_aws.S3Client gcsClient cloud_gcp.StorageClient @@ -50,7 +50,7 @@ func getHTTPRangeHeader(reference *icas.Reference) string { // Storage (CAS) backend. Any object requested through this BlobAccess // will cause its reference to be loaded from the ICAS, followed by // fetching its data from the referenced location. -func NewReferenceExpandingBlobAccess(indirectContentAddressableStorage blobstore.BlobAccess, contentAddressableStorage cdc.ContentAddressableStorage, httpClient *http.Client, s3Client cloud_aws.S3Client, gcsClient cloud_gcp.StorageClient, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) blobstore.BlobAccess { +func NewReferenceExpandingBlobAccess(indirectContentAddressableStorage blobstore.BlobAccess, contentAddressableStorage cas.ContentAddressableStorage, httpClient *http.Client, s3Client cloud_aws.S3Client, gcsClient cloud_gcp.StorageClient, maximumMessageSizeBytes int, zstdPool bb_zstd.Pool) blobstore.BlobAccess { return &referenceExpandingBlobAccess{ indirectContentAddressableStorage: indirectContentAddressableStorage, contentAddressableStorage: contentAddressableStorage, @@ -137,7 +137,7 @@ func (ba *referenceExpandingBlobAccess) Get(ctx context.Context, blobDigest dige return buffer.NewBufferFromError(util.StatusWrapWithCode(err, codes.Internal, "Invalid digest")) } - r, err = cdc.GetReadCloser(ctx, ba.contentAddressableStorage, referenceDigest) + r, err = cas.GetReadCloser(ctx, ba.contentAddressableStorage, referenceDigest) if err != nil { return buffer.NewBufferFromError(err) } diff --git a/pkg/cas/BUILD.bazel b/pkg/cas/BUILD.bazel new file mode 100644 index 000000000..9762efda6 --- /dev/null +++ b/pkg/cas/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "cas", + srcs = [ + "content_addressable_storage.go", + "operations.go", + "put_proto.go", + "read_blob_at.go", + ], + importpath = "github.com/buildbarn/bb-storage/pkg/cas", + visibility = ["//visibility:public"], + deps = [ + "//pkg/blobstore", + "//pkg/blobstore/buffer", + "//pkg/blobstore/cdc", + "//pkg/blobstore/chunklist", + "//pkg/digest", + "//pkg/util", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/pkg/blobstore/cdc/content_addressable_storage.go b/pkg/cas/content_addressable_storage.go similarity index 92% rename from pkg/blobstore/cdc/content_addressable_storage.go rename to pkg/cas/content_addressable_storage.go index 4ae510f94..ce5c4d758 100644 --- a/pkg/blobstore/cdc/content_addressable_storage.go +++ b/pkg/cas/content_addressable_storage.go @@ -1,10 +1,11 @@ -package cdc +package cas import ( "context" "github.com/buildbarn/bb-storage/pkg/blobstore" "github.com/buildbarn/bb-storage/pkg/blobstore/buffer" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -13,7 +14,7 @@ import ( type contentAddressableStorage struct { chunkStorage blobstore.BlobAccess chunkListFetcher chunklist.Fetcher - cdcParametersFetcher ParametersFetcher + cdcParametersFetcher cdc.ParametersFetcher chunkListStorage blobstore.BlobAccess digestKeyFormat digest.KeyFormat } @@ -25,7 +26,7 @@ type ContentAddressableStorage interface { // CS and CLS. GetDigestKeyFormat() digest.KeyFormat // FetchCDCParameters fetches the CDC parameters of the CAS. - FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) + FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (cdc.Parameters, error) // FindMissing returns the set of digests that are not present in // the CAS. FindMissing(ctx context.Context, digests digest.Set) (digest.Set, error) @@ -46,12 +47,12 @@ type ContentAddressableStorage interface { var _ chunklist.ChunkFetcher = ContentAddressableStorage(nil) // ContentAddressableStorage implements the ParametersFetcher interface. -var _ ParametersFetcher = ContentAddressableStorage(nil) +var _ cdc.ParametersFetcher = ContentAddressableStorage(nil) // NewContentAddressableStorage creates an interface towards the Content // Addressable Storage (CAS) that allows access to arbitrary sized // objects from the chunks described in the chunk list. -func NewContentAddressableStorage(chunkStorage, chunkListStorage blobstore.BlobAccess, chunkListFetcher chunklist.Fetcher, cdcParametersFetcher ParametersFetcher, digestKeyFormat digest.KeyFormat) ContentAddressableStorage { +func NewContentAddressableStorage(chunkStorage, chunkListStorage blobstore.BlobAccess, chunkListFetcher chunklist.Fetcher, cdcParametersFetcher cdc.ParametersFetcher, digestKeyFormat digest.KeyFormat) ContentAddressableStorage { return &contentAddressableStorage{ chunkStorage: chunkStorage, chunkListStorage: chunkListStorage, @@ -65,7 +66,7 @@ func (cas *contentAddressableStorage) GetDigestKeyFormat() digest.KeyFormat { return cas.digestKeyFormat } -func (cas *contentAddressableStorage) FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (Parameters, error) { +func (cas *contentAddressableStorage) FetchCDCParameters(ctx context.Context, instanceName digest.InstanceName) (cdc.Parameters, error) { return cas.cdcParametersFetcher.FetchCDCParameters(ctx, instanceName) } diff --git a/pkg/blobstore/cdc/operations.go b/pkg/cas/operations.go similarity index 95% rename from pkg/blobstore/cdc/operations.go rename to pkg/cas/operations.go index 214a049a9..5ee932725 100644 --- a/pkg/blobstore/cdc/operations.go +++ b/pkg/cas/operations.go @@ -1,4 +1,4 @@ -package cdc +package cas import ( "bytes" @@ -6,6 +6,7 @@ import ( "io" "runtime/debug" + "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" "github.com/buildbarn/bb-storage/pkg/digest" "github.com/buildbarn/bb-storage/pkg/util" @@ -16,7 +17,7 @@ import ( ) // IsSingleChunk checks if a digest is represented by a single chunk. -func IsSingleChunk(params Parameters, d digest.Digest) bool { +func IsSingleChunk(params cdc.Parameters, d digest.Digest) bool { return d.GetSizeBytes() < 2*params.MinChunkSizeBytes } @@ -107,7 +108,7 @@ func PutReader(ctx context.Context, cas ContentAddressableStorage, d digest.Dige } digestFunction := d.GetDigestFunction() - chunker := NewReaderChunker(digestFunction, r, params.MinChunkSizeBytes, params.HorizonSizeBytes) + chunker := cdc.NewReaderChunker(digestFunction, r, params.MinChunkSizeBytes, params.HorizonSizeBytes) wholeGen := digestFunction.NewGenerator(d.GetSizeBytes()) chunkList := make(chunklist.ChunkList, 0, d.GetSizeBytes()/params.MinChunkSizeBytes+1) @@ -146,7 +147,7 @@ func PutReader(ctx context.Context, cas ContentAddressableStorage, d digest.Dige } // We generated and validated this chunk list ourselves. - ctx = NewContextWithChunkListValidationBypass(ctx) + ctx = cdc.NewContextWithChunkListValidationBypass(ctx) if err := cas.PutManifest(ctx, d, chunkList); err != nil { return util.StatusWrap(err, "Could not save chunk list for blob") } diff --git a/pkg/blobstore/cdc/put_proto.go b/pkg/cas/put_proto.go similarity index 98% rename from pkg/blobstore/cdc/put_proto.go rename to pkg/cas/put_proto.go index 82dd45226..659d52788 100644 --- a/pkg/blobstore/cdc/put_proto.go +++ b/pkg/cas/put_proto.go @@ -1,4 +1,4 @@ -package cdc +package cas import ( "context" diff --git a/pkg/blobstore/cdc/read_blob_at.go b/pkg/cas/read_blob_at.go similarity index 99% rename from pkg/blobstore/cdc/read_blob_at.go rename to pkg/cas/read_blob_at.go index fcfdfd5ec..6f9ab1de5 100644 --- a/pkg/blobstore/cdc/read_blob_at.go +++ b/pkg/cas/read_blob_at.go @@ -1,4 +1,4 @@ -package cdc +package cas import ( "context" From 0e856739b87a5fc942e80f64bc416e1591d2a5de Mon Sep 17 00:00:00 2001 From: Benjamin Date: Wed, 26 Aug 2026 19:57:42 +0200 Subject: [PATCH 5/6] Remove debug print --- pkg/cas/operations.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/cas/operations.go b/pkg/cas/operations.go index 5ee932725..c2978ec89 100644 --- a/pkg/cas/operations.go +++ b/pkg/cas/operations.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "io" - "runtime/debug" "github.com/buildbarn/bb-storage/pkg/blobstore/cdc" "github.com/buildbarn/bb-storage/pkg/blobstore/chunklist" @@ -136,7 +135,6 @@ func PutReader(ctx context.Context, cas ContentAddressableStorage, d digest.Dige // Verify the whole blob against the advertised digest. if actual := wholeGen.Sum(); actual != d { - debug.PrintStack() return status.Errorf(codes.InvalidArgument, "Blob digest mismatch: advertised %s, actual %s", d, actual) } From 004717cc84ef3a9575162762cc4c83007caf2642 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Wed, 26 Aug 2026 21:00:20 +0200 Subject: [PATCH 6/6] Replace testutils --- pkg/testutil/BUILD.bazel | 1 + pkg/testutil/testutil.go | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/testutil/BUILD.bazel b/pkg/testutil/BUILD.bazel index 733a24b27..c3564d7ba 100644 --- a/pkg/testutil/BUILD.bazel +++ b/pkg/testutil/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/buildbarn/bb-storage/pkg/testutil", visibility = ["//visibility:public"], deps = [ + "//pkg/filesystem", "@com_github_stretchr_testify//require", "@org_golang_google_grpc//status", "@org_golang_google_protobuf//encoding/protojson", diff --git a/pkg/testutil/testutil.go b/pkg/testutil/testutil.go index c5b1a3e16..bc9dbe4d8 100644 --- a/pkg/testutil/testutil.go +++ b/pkg/testutil/testutil.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/buildbarn/bb-storage/pkg/filesystem" "github.com/stretchr/testify/require" "google.golang.org/grpc/status" @@ -14,22 +15,20 @@ import ( "go.uber.org/mock/gomock" ) -// MustMarshal is a helper that requires a proto message to succesfully -// marshal to a slice of bytes. -func MustMarshal(t *testing.T, msg proto.Message) []byte { +// FileReaderIsProto is a helper function that requires a +// filesystem.FileReader to contain the binary representation of a +// specific proto object. The resulting content gets stored in the +// actual parameter. +func FileReaderIsProto[T proto.Message](t *testing.T, r filesystem.FileReader, expected, actual T) { t.Helper() - b, err := proto.Marshal(msg) + len, err := r.Len() require.NoError(t, err) - return b -} - -// MustUnmarshal is a helper that requires a byte slice to unmarshal -// into a proto message. -func MustUnmarshal[T proto.Message](t *testing.T, data []byte, m T) T { - t.Helper() - err := proto.Unmarshal(data, m) + bytes := make([]byte, len) + n, err := r.ReadAt(bytes, 0) require.NoError(t, err) - return m + require.Equal(t, len, int64(n)) + require.NoError(t, proto.Unmarshal(bytes, actual)) + RequireEqualProto(t, expected, actual) } // RequireEqualProto asserts that the two passed protocol buffer