Skip to content

Commit

Permalink
Allow authorisation on Fetch and Push endpoints
Browse files Browse the repository at this point in the history
Currently we support the authentication policies from bb-storage by way of using
the common gRPC server, but we don't allow for granular authorisation on our
endpoints!  This adds the ability to configure authorizers (as in other
Buildbarn components) to the two endpoints we expose.

These are added at the top-level of configuration to simplify use, even though
one could technically expose this as a configuration option to decorate fetchers
and asset stores.  In particular, a single global fetch authorizer is much
simpler than having to configure a policy for remote fetching and fetching from
assets that were explicitly Push'd.
  • Loading branch information
tomcoldrick-ct committed Apr 29, 2024
1 parent d3fcdae commit 544ae68
Show file tree
Hide file tree
Showing 18 changed files with 535 additions and 116 deletions.
3 changes: 3 additions & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ load("@gazelle//:def.bzl", "gazelle")
# gazelle:resolve proto go google/bytestream/bytestream.proto @org_golang_google_genproto_googleapis_bytestream//:bytestream
# gazelle:resolve proto go google/rpc/status.proto @org_golang_google_genproto_googleapis_rpc//status
# gazelle:resolve proto go opentelemetry/proto/common/v1/common.proto @io_opentelemetry_go_proto_otlp//common/v1:common
# gazelle:resolve proto go pkg/proto/configuration/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/auth
# gazelle:resolve proto go pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore
# gazelle:resolve proto go pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global
# gazelle:resolve proto go pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc
Expand All @@ -18,6 +19,8 @@ load("@gazelle//:def.bzl", "gazelle")
# gazelle:resolve proto google/protobuf/timestamp.proto @protobuf//:timestamp_proto
# gazelle:resolve proto google/rpc/status.proto @googleapis//google/rpc:status_proto
# gazelle:resolve proto opentelemetry/proto/common/v1/common.proto @io_opentelemetry_proto//:common_proto
# gazelle:resolve proto pkg/proto/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/auth:auth_proto
# gazelle:resolve proto pkg/proto/configuration/auth/auth.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/auth:auth_proto
# gazelle:resolve proto pkg/proto/configuration/blobstore/blobstore.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto
# gazelle:resolve proto pkg/proto/configuration/global/global.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto
# gazelle:resolve proto pkg/proto/configuration/grpc/grpc.proto @com_github_buildbarn_bb_storage//pkg/proto/configuration/grpc:grpc_proto
Expand Down
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ $ cat config/bb_remote_asset.jsonnet
}],
allowUpdatesForInstances: ['foo'],
maximumMessageSizeBytes: 16 * 1024 * 1024 * 1024,
fetchAuthorizer: { allow: {} },
pushAuthorizer: { allow: {} },
}
```

Expand Down Expand Up @@ -79,7 +81,7 @@ $ cat config/bb_remote_asset.jsonnet
keyLocationMapMaximumPutAttempts: 32,
oldBlocks: 8,
currentBlocks: 24,
newBlocks: 3,
newBlocks: 1,
blocksOnBlockDevice: {
source: {
file: {
Expand All @@ -106,6 +108,8 @@ $ cat config/bb_remote_asset.jsonnet
}],
allowUpdatesForInstances: ['foo'],
maximumMessageSizeBytes: 16 * 1024 * 1024 * 1024,
fetchAuthorizer: { allow: {} },
pushAuthorizer: { allow: {} },
}
```

Expand Down
1 change: 1 addition & 0 deletions cmd/bb_remote_asset/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ go_library(
"//pkg/proto/configuration/bb_remote_asset",
"//pkg/push",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:asset",
"@com_github_buildbarn_bb_storage//pkg/auth",
"@com_github_buildbarn_bb_storage//pkg/clock",
"@com_github_buildbarn_bb_storage//pkg/digest",
"@com_github_buildbarn_bb_storage//pkg/global",
Expand Down
15 changes: 14 additions & 1 deletion cmd/bb_remote_asset/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/buildbarn/bb-remote-asset/pkg/fetch"
"github.com/buildbarn/bb-remote-asset/pkg/proto/configuration/bb_remote_asset"
"github.com/buildbarn/bb-remote-asset/pkg/push"
"github.com/buildbarn/bb-storage/pkg/auth"
"github.com/buildbarn/bb-storage/pkg/clock"
bb_digest "github.com/buildbarn/bb-storage/pkg/digest"
"github.com/buildbarn/bb-storage/pkg/global"
Expand Down Expand Up @@ -45,11 +46,23 @@ func main() {
return util.StatusWrap(err, "Failed to apply global configuration options")
}

fetchAuthorizer, err := auth.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(config.FetchAuthorizer)
if err != nil {
return util.StatusWrap(err, "Failed to create Fetch Authorizer from Configuration")
}

pushAuthorizer, err := auth.DefaultAuthorizerFactory.NewAuthorizerFromConfiguration(config.PushAuthorizer)
if err != nil {
return util.StatusWrap(err, "Failed to create Push Authorizer from Configuration")
}

assetStore, contentAddressableStorage, err := configuration.NewAssetStoreAndCASFromConfiguration(
config.AssetCache,
grpcClientFactory,
int(config.MaximumMessageSizeBytes),
dependenciesGroup,
fetchAuthorizer,
pushAuthorizer,
)
if err != nil {
return util.StatusWrap(err, "Failed to create asset store and CAS")
Expand All @@ -64,7 +77,7 @@ func main() {
allowUpdatesForInstances[instanceName] = true
}

fetchServer, err := configuration.NewFetcherFromConfiguration(config.Fetcher, assetStore, contentAddressableStorage, grpcClientFactory, int(config.MaximumMessageSizeBytes))
fetchServer, err := configuration.NewFetcherFromConfiguration(config.Fetcher, assetStore, contentAddressableStorage, grpcClientFactory, int(config.MaximumMessageSizeBytes), fetchAuthorizer)
if err != nil {
return util.StatusWrap(err, "Failed to initialize fetch server from configuration")
}
Expand Down
21 changes: 21 additions & 0 deletions internal/mock/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,16 @@ gomock(
package = "mock",
)

gomock(
name = "auth",
out = "auth.go",
interfaces = [
"Authorizer",
],
library = "@com_github_buildbarn_bb_storage//pkg/auth",
package = "mock",
)

gomock(
name = "aliases",
out = "aliases.go",
Expand All @@ -30,18 +40,29 @@ gomock(
package = "mock",
)

gomock(
name = "storage",
out = "storage.go",
interfaces = ["AssetStore"],
library = "//pkg/storage",
package = "mock",
)

go_library(
name = "mock",
srcs = [
"aliases.go",
"auth.go",
"blobstore.go",
"dummy.go",
"fetcher.go",
"storage.go",
],
importpath = "github.com/buildbarn/bb-remote-asset/internal/mock",
visibility = ["//:__subpackages__"],
# keep
deps = [
"//pkg/proto/asset",
"//pkg/qualifier",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:asset",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
Expand Down
47 changes: 47 additions & 0 deletions patches/bazel_gazelle/dont-flatten-srcs.diff
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
diff --git language/go/generate.go language/go/generate.go
index 53e397a..38855c6 100644
--- language/go/generate.go
+++ language/go/generate.go
@@ -483,7 +483,7 @@ func (g *generator) generateLib(pkg *goPackage, embed string) *rule.Rule {
} else {
visibility = g.commonVisibility(pkg.importPath)
}
- g.setCommonAttrs(goLibrary, pkg.rel, visibility, pkg.library, embed)
+ g.setCommonAttrs(goLibrary, pkg.rel, visibility, pkg.library, embed, true)
g.setImportAttrs(goLibrary, pkg.importPath)
return goLibrary
}
@@ -512,7 +512,7 @@ func (g *generator) generateBin(pkg *goPackage, library string) *rule.Rule {
return goBinary // empty
}
visibility := g.commonVisibility(pkg.importPath)
- g.setCommonAttrs(goBinary, pkg.rel, visibility, pkg.binary, library)
+ g.setCommonAttrs(goBinary, pkg.rel, visibility, pkg.binary, library, true)
return goBinary
}

@@ -552,7 +552,7 @@ func (g *generator) generateTests(pkg *goPackage, library string) []*rule.Rule {
if test.hasInternalTest {
embed = library
}
- g.setCommonAttrs(goTest, pkg.rel, nil, test, embed)
+ g.setCommonAttrs(goTest, pkg.rel, nil, test, embed, false)
if pkg.hasTestdata {
goTest.SetAttr("data", rule.GlobValue{Patterns: []string{"testdata/**"}})
}
@@ -629,9 +629,13 @@ func (g *generator) maybeGenerateExtraLib(lib *rule.Rule, pkg *goPackage) *rule.
return r
}

-func (g *generator) setCommonAttrs(r *rule.Rule, pkgRel string, visibility []string, target goTarget, embed string) {
+func (g *generator) setCommonAttrs(r *rule.Rule, pkgRel string, visibility []string, target goTarget, embed string, flattenSrcs bool) {
if !target.sources.isEmpty() {
- r.SetAttr("srcs", target.sources.buildFlat())
+ if flattenSrcs {
+ r.SetAttr("srcs", target.sources.buildFlat())
+ } else {
+ r.SetAttr("srcs", target.sources.build())
+ }
}
if !target.embedSrcs.isEmpty() {
r.SetAttr("embedsrcs", target.embedSrcs.build())
1 change: 1 addition & 0 deletions pkg/configuration/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ go_library(
"//pkg/proto/configuration/bb_remote_asset/fetch",
"//pkg/storage",
"//pkg/storage/blobstore",
"@com_github_buildbarn_bb_storage//pkg/auth",
"@com_github_buildbarn_bb_storage//pkg/blobstore",
"@com_github_buildbarn_bb_storage//pkg/blobstore/configuration",
"@com_github_buildbarn_bb_storage//pkg/digest",
Expand Down
5 changes: 4 additions & 1 deletion pkg/configuration/new_asset_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
pb "github.com/buildbarn/bb-remote-asset/pkg/proto/configuration/bb_remote_asset"
"github.com/buildbarn/bb-remote-asset/pkg/storage"
asset_configuration "github.com/buildbarn/bb-remote-asset/pkg/storage/blobstore"
"github.com/buildbarn/bb-storage/pkg/auth"
"github.com/buildbarn/bb-storage/pkg/blobstore"
blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration"
"github.com/buildbarn/bb-storage/pkg/grpc"
Expand All @@ -17,6 +18,8 @@ func NewAssetStoreAndCASFromConfiguration(
grpcClientFactory grpc.ClientFactory,
maximumMessageSizeBytes int,
dependenciesGroup program.Group,
fetchAuthorizer auth.Authorizer,
pushAuthorizer auth.Authorizer,
) (storage.AssetStore, blobstore.BlobAccess, error) {
var assetStore storage.AssetStore
var contentAddressableStorage blobstore.BlobAccess
Expand Down Expand Up @@ -55,5 +58,5 @@ func NewAssetStoreAndCASFromConfiguration(
contentAddressableStorage = cas
assetStore = storage.NewActionCacheAssetStore(actionCache, contentAddressableStorage, maximumMessageSizeBytes)
}
return assetStore, contentAddressableStorage, nil
return storage.NewAuthorizingAssetStore(assetStore, fetchAuthorizer, pushAuthorizer), contentAddressableStorage, nil
}
6 changes: 4 additions & 2 deletions pkg/configuration/new_fetcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/buildbarn/bb-remote-asset/pkg/fetch"
pb "github.com/buildbarn/bb-remote-asset/pkg/proto/configuration/bb_remote_asset/fetch"
"github.com/buildbarn/bb-remote-asset/pkg/storage"
"github.com/buildbarn/bb-storage/pkg/auth"
"github.com/buildbarn/bb-storage/pkg/blobstore"
bb_digest "github.com/buildbarn/bb-storage/pkg/digest"
"github.com/buildbarn/bb-storage/pkg/grpc"
Expand All @@ -23,11 +24,12 @@ func NewFetcherFromConfiguration(configuration *pb.FetcherConfiguration,
contentAddressableStorage blobstore.BlobAccess,
grpcClientFactory grpc.ClientFactory,
maximumMessageSizeBytes int,
authorizer auth.Authorizer,
) (fetch.Fetcher, error) {
var fetcher fetch.Fetcher
switch backend := configuration.Backend.(type) {
case *pb.FetcherConfiguration_Caching:
innerFetcher, err := NewFetcherFromConfiguration(backend.Caching.Fetcher, assetStore, contentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes)
innerFetcher, err := NewFetcherFromConfiguration(backend.Caching.Fetcher, assetStore, contentAddressableStorage, grpcClientFactory, maximumMessageSizeBytes, nil)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -64,5 +66,5 @@ func NewFetcherFromConfiguration(configuration *pb.FetcherConfiguration,
return nil, status.Errorf(codes.InvalidArgument, "Fetcher configuration is invalid as no supported Fetchers are defined.")
}

return fetcher, nil
return fetch.NewAuthorizingFetcher(fetcher, authorizer), nil
}
3 changes: 3 additions & 0 deletions pkg/fetch/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go_library(
name = "fetch",
srcs = [
"auth_headers.go",
"authorizing_fetcher.go",
"caching_fetcher.go",
"error_fetcher.go",
"fetcher.go",
Expand All @@ -20,6 +21,7 @@ go_library(
"//pkg/storage",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/asset/v1:asset",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
"@com_github_buildbarn_bb_storage//pkg/auth",
"@com_github_buildbarn_bb_storage//pkg/blobstore",
"@com_github_buildbarn_bb_storage//pkg/blobstore/buffer",
"@com_github_buildbarn_bb_storage//pkg/clock",
Expand All @@ -40,6 +42,7 @@ go_library(
go_test(
name = "fetch_test",
srcs = [
"authorizing_fetcher_test.go",
"caching_fetcher_test.go",
"http_fetcher_test.go",
"validating_fetcher_test.go",
Expand Down
47 changes: 47 additions & 0 deletions pkg/fetch/authorizing_fetcher.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package fetch

import (
"context"

remoteasset "github.com/bazelbuild/remote-apis/build/bazel/remote/asset/v1"
"github.com/buildbarn/bb-storage/pkg/auth"
bb_digest "github.com/buildbarn/bb-storage/pkg/digest"
)

// AuthorizingFetcher decorates Fetcher and validates the requests against an Authorizer
type AuthorizingFetcher struct {
Fetcher
authorizer auth.Authorizer
}

// NewAuthorizingFetcher creates a new AuthorizingFetcher
func NewAuthorizingFetcher(f Fetcher, authorizer auth.Authorizer) *AuthorizingFetcher {
return &AuthorizingFetcher{
f,
authorizer,
}
}

// FetchBlob wraps FetchBlob requests, validate request against authorizer
func (af *AuthorizingFetcher) FetchBlob(ctx context.Context, req *remoteasset.FetchBlobRequest) (*remoteasset.FetchBlobResponse, error) {
instanceName, err := bb_digest.NewInstanceName(req.InstanceName)
if err != nil {
return nil, err
}
if err = auth.AuthorizeSingleInstanceName(ctx, af.authorizer, instanceName); err != nil {
return nil, err
}
return af.Fetcher.FetchBlob(ctx, req)
}

// FetchDirectory wraps FetchDirectory requests, validate request against authorizer
func (af *AuthorizingFetcher) FetchDirectory(ctx context.Context, req *remoteasset.FetchDirectoryRequest) (*remoteasset.FetchDirectoryResponse, error) {
instanceName, err := bb_digest.NewInstanceName(req.InstanceName)
if err != nil {
return nil, err
}
if err = auth.AuthorizeSingleInstanceName(ctx, af.authorizer, instanceName); err != nil {
return nil, err
}
return af.Fetcher.FetchDirectory(ctx, req)
}
Loading

0 comments on commit 544ae68

Please sign in to comment.