Add authorisation on Fetch and Push endpoints

BUILD.bazel

@@ -10,6 +10,7 @@ load("@gazelle//:def.bzl", "gazelle")
 # gazelle:resolve proto go google/bytestream/bytestream.proto @org_golang_google_genproto_googleapis_bytestream//:bytestream
 # gazelle:resolve proto go google/rpc/status.proto @org_golang_google_genproto_googleapis_rpc//status
 # gazelle:resolve proto go opentelemetry/proto/common/v1/common.proto @io_opentelemetry_go_proto_otlp//common/v1:common
+# gazelle:resolve proto go pkg/proto/configuration/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/auth
 # gazelle:resolve proto go pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore
 # gazelle:resolve proto go pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global
 # gazelle:resolve proto go pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc
@@ -18,6 +19,8 @@ load("@gazelle//:def.bzl", "gazelle")
 # gazelle:resolve proto google/protobuf/timestamp.proto @protobuf//:timestamp_proto
 # gazelle:resolve proto google/rpc/status.proto @googleapis//google/rpc:status_proto
 # gazelle:resolve proto opentelemetry/proto/common/v1/common.proto @io_opentelemetry_proto//:common_proto
+# gazelle:resolve proto pkg/proto/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/auth:auth_proto
+# gazelle:resolve proto pkg/proto/configuration/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/auth:auth_proto
 # gazelle:resolve proto pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto
 # gazelle:resolve proto pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto
 # gazelle:resolve proto pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc:grpc_proto

README.md

@@ -46,6 +46,8 @@ $ cat config/bb_remote_asset.jsonnet
   }],
   allowUpdatesForInstances: ['foo'],
   maximumMessageSizeBytes: 16 * 1024 * 1024 * 1024,
+  fetchAuthorizer: { allow: {} },
+  pushAuthorizer: { allow: {} },
 }
 ```
 
@@ -79,7 +81,7 @@ $ cat config/bb_remote_asset.jsonnet
           keyLocationMapMaximumPutAttempts: 32,
           oldBlocks: 8,
           currentBlocks: 24,
-          newBlocks: 3,
+          newBlocks: 1,
           blocksOnBlockDevice: {
             source: {
               file: {
@@ -106,6 +108,8 @@ $ cat config/bb_remote_asset.jsonnet
   }],
   allowUpdatesForInstances: ['foo'],
   maximumMessageSizeBytes: 16 * 1024 * 1024 * 1024,
+  fetchAuthorizer: { allow: {} },
+  pushAuthorizer: { allow: {} },
 }
 ```
 

cmd/bb_remote_asset/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "//pkg/proto/configuration/bb_remote_asset",
         "//pkg/push",
         "@com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:asset",
+        "@com_github_buildbarn_bb_storage//pkg/auth",
         "@com_github_buildbarn_bb_storage//pkg/clock",
         "@com_github_buildbarn_bb_storage//pkg/digest",
         "@com_github_buildbarn_bb_storage//pkg/global",

cmd/bb_remote_asset/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/buildbarn/bb-remote-asset/pkg/fetch"
 	"github.com/buildbarn/bb-remote-asset/pkg/proto/configuration/bb_remote_asset"
 	"github.com/buildbarn/bb-remote-asset/pkg/push"
+	"github.com/buildbarn/bb-storage/pkg/auth"
 	"github.com/buildbarn/bb-storage/pkg/clock"
 	bb_digest "github.com/buildbarn/bb-storage/pkg/digest"
 	"github.com/buildbarn/bb-storage/pkg/global"
@@ -45,11 +46,23 @@ func main() {
 			return util.StatusWrap(err, "Failed to apply global configuration options")
 		}
 
+		fetchAuthorizer, err := auth.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(config.FetchAuthorizer)
+		if err != nil {
+			return util.StatusWrap(err, "Failed to create Fetch Authorizer from Configuration")
+		}
+
+		pushAuthorizer, err := auth.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(config.PushAuthorizer)
+		if err != nil {
+			return util.StatusWrap(err, "Failed to create Push Authorizer from Configuration")
+		}
+
 		assetStore, contentAddressableStorage, err := configuration.NewAssetStoreAndCASFromConfiguration(
 			config.AssetCache,
 			grpcClientFactory,
 			int(config.MaximumMessageSizeBytes),
 			dependenciesGroup,
+			fetchAuthorizer,
+			pushAuthorizer,
 		)
 		if err != nil {
 			return util.StatusWrap(err, "Failed to create asset store and CAS")
@@ -64,7 +77,7 @@ func main() {
 			allowUpdatesForInstances[instanceName] = true
 		}
 
-		fetchServer, err := configuration.NewFetcherFromConfiguration(config.Fetcher, assetStore, contentAddressableStorage, grpcClientFactory, int(config.MaximumMessageSizeBytes))
+		fetchServer, err := configuration.NewFetcherFromConfiguration(config.Fetcher, assetStore, contentAddressableStorage, grpcClientFactory, int(config.MaximumMessageSizeBytes), fetchAuthorizer)
 		if err != nil {
 			return util.StatusWrap(err, "Failed to initialize fetch server from configuration")
 		}

internal/mock/BUILD.bazel

@@ -11,6 +11,16 @@ gomock(
     package = "mock",
 )
 
+gomock(
+    name = "auth",
+    out = "auth.go",
+    interfaces = [
+        "Authorizer",
+    ],
+    library = "@com_github_buildbarn_bb_storage//pkg/auth",
+    package = "mock",
+)
+
 gomock(
     name = "aliases",
     out = "aliases.go",
@@ -30,18 +40,29 @@ gomock(
     package = "mock",
 )
 
+gomock(
+    name = "storage",
+    out = "storage.go",
+    interfaces = ["AssetStore"],
+    library = "//pkg/storage",
+    package = "mock",
+)
+
 go_library(
     name = "mock",
     srcs = [
         "aliases.go",
+        "auth.go",
         "blobstore.go",
         "dummy.go",
         "fetcher.go",
+        "storage.go",
     ],
     importpath = "github.com/buildbarn/bb-remote-asset/internal/mock",
     visibility = ["//:__subpackages__"],
     # keep
     deps = [
+        "//pkg/proto/asset",
         "//pkg/qualifier",
         "@com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:asset",
         "@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",

patches/bazel_gazelle/dont-flatten-srcs.diff

@@ -0,0 +1,47 @@
+diff --git language/go/generate.go language/go/generate.go
+index 53e397a..38855c6 100644
+--- language/go/generate.go
++++ language/go/generate.go
+@@ -483,7 +483,7 @@ func (g *generator) generateLib(pkg *goPackage, embed string) *rule.Rule {
+ 	} else {
+ 		visibility = g.commonVisibility(pkg.importPath)
+ 	}
+-	g.setCommonAttrs(goLibrary, pkg.rel, visibility, pkg.library, embed)
++	g.setCommonAttrs(goLibrary, pkg.rel, visibility, pkg.library, embed, true)
+ 	g.setImportAttrs(goLibrary, pkg.importPath)
+ 	return goLibrary
+ }
+@@ -512,7 +512,7 @@ func (g *generator) generateBin(pkg *goPackage, library string) *rule.Rule {
+ 		return goBinary // empty
+ 	}
+ 	visibility := g.commonVisibility(pkg.importPath)
+-	g.setCommonAttrs(goBinary, pkg.rel, visibility, pkg.binary, library)
++	g.setCommonAttrs(goBinary, pkg.rel, visibility, pkg.binary, library, true)
+ 	return goBinary
+ }
+
+@@ -552,7 +552,7 @@ func (g *generator) generateTests(pkg *goPackage, library string) []*rule.Rule {
+ 		if test.hasInternalTest {
+ 			embed = library
+ 		}
+-		g.setCommonAttrs(goTest, pkg.rel, nil, test, embed)
++		g.setCommonAttrs(goTest, pkg.rel, nil, test, embed, false)
+ 		if pkg.hasTestdata {
+ 			goTest.SetAttr("data", rule.GlobValue{Patterns: []string{"testdata/**"}})
+ 		}
+@@ -629,9 +629,13 @@ func (g *generator) maybeGenerateExtraLib(lib *rule.Rule, pkg *goPackage) *rule.
+ 	return r
+ }
+
+-func (g *generator) setCommonAttrs(r *rule.Rule, pkgRel string, visibility []string, target goTarget, embed string) {
++func (g *generator) setCommonAttrs(r *rule.Rule, pkgRel string, visibility []string, target goTarget, embed string, flattenSrcs bool) {
+ 	if !target.sources.isEmpty() {
+-		r.SetAttr("srcs", target.sources.buildFlat())
++		if flattenSrcs {
++			r.SetAttr("srcs", target.sources.buildFlat())
++		} else {
++			r.SetAttr("srcs", target.sources.build())
++		}
+ 	}
+ 	if !target.embedSrcs.isEmpty() {
+ 		r.SetAttr("embedsrcs", target.embedSrcs.build())

pkg/configuration/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "//pkg/proto/configuration/bb_remote_asset/fetch",
         "//pkg/storage",
         "//pkg/storage/blobstore",
+        "@com_github_buildbarn_bb_storage//pkg/auth",
         "@com_github_buildbarn_bb_storage//pkg/blobstore",
         "@com_github_buildbarn_bb_storage//pkg/blobstore/configuration",
         "@com_github_buildbarn_bb_storage//pkg/digest",

pkg/configuration/new_asset_store.go

@@ -4,6 +4,7 @@ import (
 	pb "github.com/buildbarn/bb-remote-asset/pkg/proto/configuration/bb_remote_asset"
 	"github.com/buildbarn/bb-remote-asset/pkg/storage"
 	asset_configuration "github.com/buildbarn/bb-remote-asset/pkg/storage/blobstore"
+	"github.com/buildbarn/bb-storage/pkg/auth"
 	"github.com/buildbarn/bb-storage/pkg/blobstore"
 	blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration"
 	"github.com/buildbarn/bb-storage/pkg/grpc"
@@ -17,6 +18,8 @@ func NewAssetStoreAndCASFromConfiguration(
 	grpcClientFactory grpc.ClientFactory,
 	maximumMessageSizeBytes int,
 	dependenciesGroup program.Group,
+	fetchAuthorizer auth.Authorizer,
+	pushAuthorizer auth.Authorizer,
 ) (storage.AssetStore, blobstore.BlobAccess, error) {
 	var assetStore storage.AssetStore
 	var contentAddressableStorage blobstore.BlobAccess
@@ -55,5 +58,5 @@ func NewAssetStoreAndCASFromConfiguration(
 		contentAddressableStorage = cas
 		assetStore = storage.NewActionCacheAssetStore(actionCache, contentAddressableStorage, maximumMessageSizeBytes)
 	}
-	return assetStore, contentAddressableStorage, nil
+	return storage.NewAuthorizingAssetStore(assetStore, fetchAuthorizer, pushAuthorizer), contentAddressableStorage, nil
 }

pkg/configuration/new_fetcher.go

@@ -7,6 +7,7 @@ import (
 	"github.com/buildbarn/bb-remote-asset/pkg/fetch"
 	pb "github.com/buildbarn/bb-remote-asset/pkg/proto/configuration/bb_remote_asset/fetch"
 	"github.com/buildbarn/bb-remote-asset/pkg/storage"
+	"github.com/buildbarn/bb-storage/pkg/auth"
 	"github.com/buildbarn/bb-storage/pkg/blobstore"
 	bb_digest "github.com/buildbarn/bb-storage/pkg/digest"
 	"github.com/buildbarn/bb-storage/pkg/grpc"
@@ -23,11 +24,12 @@ func NewFetcherFromConfiguration(configuration *pb.FetcherConfiguration,
 	contentAddressableStorage blobstore.BlobAccess,
 	grpcClientFactory grpc.ClientFactory,
 	maximumMessageSizeBytes int,
+	authorizer auth.Authorizer,
 ) (fetch.Fetcher, error) {
 	var fetcher fetch.Fetcher
 	switch backend := configuration.Backend.(type) {
 	case *pb.FetcherConfiguration_Caching:
-		innerFetcher, err := NewFetcherFromConfiguration(backend.Caching.Fetcher, assetStore, contentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes)
+		innerFetcher, err := NewFetcherFromConfiguration(backend.Caching.Fetcher, assetStore, contentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -64,5 +66,5 @@ func NewFetcherFromConfiguration(configuration *pb.FetcherConfiguration,
 		return nil, status.Errorf(codes.InvalidArgument, "Fetcher configuration is invalid as no supported Fetchers are defined.")
 	}
 
-	return fetcher, nil
+	return fetch.NewAuthorizingFetcher(fetcher, authorizer), nil
 }

pkg/fetch/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "fetch",
     srcs = [
         "auth_headers.go",
+        "authorizing_fetcher.go",
         "caching_fetcher.go",
         "error_fetcher.go",
         "fetcher.go",
@@ -20,6 +21,7 @@ go_library(
         "//pkg/storage",
         "@com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:asset",
         "@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
+        "@com_github_buildbarn_bb_storage//pkg/auth",
         "@com_github_buildbarn_bb_storage//pkg/blobstore",
         "@com_github_buildbarn_bb_storage//pkg/blobstore/buffer",
         "@com_github_buildbarn_bb_storage//pkg/clock",
@@ -40,6 +42,7 @@ go_library(
 go_test(
     name = "fetch_test",
     srcs = [
+        "authorizing_fetcher_test.go",
         "caching_fetcher_test.go",
         "http_fetcher_test.go",
         "validating_fetcher_test.go",

pkg/fetch/authorizing_fetcher.go

@@ -0,0 +1,47 @@
+package fetch
+
+import (
+	"context"
+
+	remoteasset "github.com/bazelbuild/remote-apis/build/bazel/remote/asset/v1"
+	"github.com/buildbarn/bb-storage/pkg/auth"
+	bb_digest "github.com/buildbarn/bb-storage/pkg/digest"
+)
+
+// AuthorizingFetcher decorates Fetcher and validates the requests against an Authorizer
+type AuthorizingFetcher struct {
+	Fetcher
+	authorizer auth.Authorizer
+}
+
+// NewAuthorizingFetcher creates a new AuthorizingFetcher
+func NewAuthorizingFetcher(f Fetcher, authorizer auth.Authorizer) *AuthorizingFetcher {
+	return &AuthorizingFetcher{
+		f,
+		authorizer,
+	}
+}
+
+// FetchBlob wraps FetchBlob requests, validate request against authorizer
+func (af *AuthorizingFetcher) FetchBlob(ctx context.Context, req *remoteasset.FetchBlobRequest) (*remoteasset.FetchBlobResponse, error) {
+	instanceName, err := bb_digest.NewInstanceName(req.InstanceName)
+	if err != nil {
+		return nil, err
+	}
+	if err = auth.AuthorizeSingleInstanceName(ctx, af.authorizer, instanceName); err != nil {
+		return nil, err
+	}
+	return af.Fetcher.FetchBlob(ctx, req)
+}
+
+// FetchDirectory wraps FetchDirectory requests, validate request against authorizer
+func (af *AuthorizingFetcher) FetchDirectory(ctx context.Context, req *remoteasset.FetchDirectoryRequest) (*remoteasset.FetchDirectoryResponse, error) {
+	instanceName, err := bb_digest.NewInstanceName(req.InstanceName)
+	if err != nil {
+		return nil, err
+	}
+	if err = auth.AuthorizeSingleInstanceName(ctx, af.authorizer, instanceName); err != nil {
+		return nil, err
+	}
+	return af.Fetcher.FetchDirectory(ctx, req)
+}