From 4f654a033a5315fc3a002e0dd4d078bbdbbcc9d0 Mon Sep 17 00:00:00 2001 From: STeve Huang Date: Thu, 28 Nov 2024 17:25:52 -0500 Subject: [PATCH] GitHub proxy part 2.5: git_server cache --- api/client/client.go | 7 +- api/client/events.go | 14 +- api/client/gitserver/gitserver.go | 125 ++++++++++++++++++ api/client/proto/event.pb.go | 58 +++++--- .../teleport/legacy/client/proto/event.proto | 2 + lib/auth/accesspoint/accesspoint.go | 2 + lib/auth/authclient/clt.go | 4 +- lib/auth/helpers.go | 1 + lib/cache/cache.go | 14 ++ lib/cache/cache_test.go | 12 ++ lib/cache/collections.go | 14 ++ lib/cache/git_server.go | 95 +++++++++++++ lib/cache/git_server_test.go | 65 +++++++++ lib/service/service.go | 2 + lib/services/local/events.go | 2 + lib/services/local/git_server.go | 43 ++++++ tool/tctl/common/resource_command.go | 33 ++--- 17 files changed, 445 insertions(+), 48 deletions(-) create mode 100644 api/client/gitserver/gitserver.go create mode 100644 lib/cache/git_server.go create mode 100644 lib/cache/git_server_test.go diff --git a/api/client/client.go b/api/client/client.go index 297e8bb9ed273..ef8316dc2f34f 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -55,6 +55,7 @@ import ( "github.com/gravitational/teleport/api/client/discoveryconfig" "github.com/gravitational/teleport/api/client/dynamicwindows" "github.com/gravitational/teleport/api/client/externalauditstorage" + gitserverclient "github.com/gravitational/teleport/api/client/gitserver" kubewaitingcontainerclient "github.com/gravitational/teleport/api/client/kubewaitingcontainer" "github.com/gravitational/teleport/api/client/okta" "github.com/gravitational/teleport/api/client/proto" @@ -77,7 +78,7 @@ import ( discoveryconfigv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/discoveryconfig/v1" dynamicwindowsv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/dynamicwindows/v1" externalauditstoragev1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/externalauditstorage/v1" - gitserverv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/gitserver/v1" + gitserverpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/gitserver/v1" identitycenterv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/identitycenter/v1" integrationpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/integration/v1" kubeproto "github.com/gravitational/teleport/api/gen/proto/go/teleport/kube/v1" @@ -4876,8 +4877,8 @@ func (c *Client) UserTasksServiceClient() *usertaskapi.Client { } // GitServerClient returns a client for managing git servers -func (c *Client) GitServerClient() gitserverv1.GitServerServiceClient { - return gitserverv1.NewGitServerServiceClient(c.conn) +func (c *Client) GitServerClient() *gitserverclient.Client { + return gitserverclient.NewClient(gitserverpb.NewGitServerServiceClient(c.conn)) } // GetCertAuthority retrieves a CA by type and domain. diff --git a/api/client/events.go b/api/client/events.go index 89e5b5ecc3157..4b8a0688bc71a 100644 --- a/api/client/events.go +++ b/api/client/events.go @@ -171,8 +171,15 @@ func EventToGRPC(in types.Event) (*proto.Event, error) { Namespace: r, } case *types.ServerV2: - out.Resource = &proto.Event_Server{ - Server: r, + switch r.GetKind() { + case types.KindGitServer: + out.Resource = &proto.Event_GitServer{ + GitServer: r, + } + default: + out.Resource = &proto.Event_Server{ + Server: r, + } } case *types.ReverseTunnelV2: out.Resource = &proto.Event_ReverseTunnel{ @@ -609,6 +616,9 @@ func EventFromGRPC(in *proto.Event) (*types.Event, error) { } else if r := in.GetIdentityCenterAccountAssignment(); r != nil { out.Resource = types.Resource153ToLegacy(r) return &out, nil + } else if r := in.GetGitServer(); r != nil { + out.Resource = r + return &out, nil } else { return nil, trace.BadParameter("received unsupported resource %T", in.Resource) } diff --git a/api/client/gitserver/gitserver.go b/api/client/gitserver/gitserver.go new file mode 100644 index 0000000000000..b8388da24e750 --- /dev/null +++ b/api/client/gitserver/gitserver.go @@ -0,0 +1,125 @@ +// Copyright 2024 Gravitational, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package gitserver + +import ( + "context" + + "github.com/gravitational/trace" + + gitserverv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/gitserver/v1" + "github.com/gravitational/teleport/api/types" +) + +// Client is an Git servers client. +type Client struct { + grpcClient gitserverv1.GitServerServiceClient +} + +// NewClient creates a new Git servers client. +func NewClient(grpcClient gitserverv1.GitServerServiceClient) *Client { + return &Client{ + grpcClient: grpcClient, + } +} + +// GetGitServer returns Git servers by name. +func (c *Client) GetGitServer(ctx context.Context, name string) (types.Server, error) { + server, err := c.grpcClient.GetGitServer(ctx, &gitserverv1.GetGitServerRequest{Name: name}) + if err != nil { + return nil, trace.Wrap(err) + } + return server, nil +} + +// ListGitServers returns all Git servers matching filter. +func (c *Client) ListGitServers(ctx context.Context, pageSize int, pageToken string) ([]types.Server, string, error) { + resp, err := c.grpcClient.ListGitServers(ctx, &gitserverv1.ListGitServersRequest{ + PageSize: int32(pageSize), + PageToken: pageToken, + }) + if err != nil { + return nil, "", trace.Wrap(err) + } + + servers := make([]types.Server, 0, len(resp.Servers)) + for _, server := range resp.Servers { + servers = append(servers, server) + } + return servers, resp.NextPageToken, nil +} + +func toServerV2(server types.Server) (*types.ServerV2, error) { + serverV2, ok := server.(*types.ServerV2) + if !ok { + return nil, trace.Errorf("encountered unexpected server type: %T", serverV2) + } + return serverV2, nil +} + +// CreateGitServer creates a Git server resource. +func (c *Client) CreateGitServer(ctx context.Context, item types.Server) (types.Server, error) { + serverV2, err := toServerV2(item) + if err != nil { + return nil, trace.Wrap(err) + } + resp, err := c.grpcClient.CreateGitServer(ctx, &gitserverv1.CreateGitServerRequest{ + Server: serverV2, + }) + if err != nil { + return nil, trace.Wrap(err) + } + return resp, nil +} + +// UpdateGitServer updates a Git server resource. +func (c *Client) UpdateGitServer(ctx context.Context, item types.Server) (types.Server, error) { + serverV2, err := toServerV2(item) + if err != nil { + return nil, trace.Wrap(err) + } + resp, err := c.grpcClient.UpdateGitServer(ctx, &gitserverv1.UpdateGitServerRequest{ + Server: serverV2, + }) + if err != nil { + return nil, trace.Wrap(err) + } + return resp, nil +} + +// UpsertGitServer updates a Git server resource, creating it if it doesn't exist. +func (c *Client) UpsertGitServer(ctx context.Context, item types.Server) (types.Server, error) { + serverV2, err := toServerV2(item) + if err != nil { + return nil, trace.Wrap(err) + } + resp, err := c.grpcClient.UpsertGitServer(ctx, &gitserverv1.UpsertGitServerRequest{ + Server: serverV2, + }) + if err != nil { + return nil, trace.Wrap(err) + } + return resp, nil +} + +// DeleteGitServer removes the specified Git server resource. +func (c *Client) DeleteGitServer(ctx context.Context, name string) error { + _, err := c.grpcClient.DeleteGitServer(ctx, &gitserverv1.DeleteGitServerRequest{Name: name}) + return trace.Wrap(err) +} + +// DeleteAllGitServers removes all Git server resources. +func (c *Client) DeleteAllGitServers(ctx context.Context) error { + return trace.NotImplemented("delete all git servers not implemented") +} diff --git a/api/client/proto/event.pb.go b/api/client/proto/event.pb.go index 8cb8416436e6f..6c44cbaa867dc 100644 --- a/api/client/proto/event.pb.go +++ b/api/client/proto/event.pb.go @@ -186,6 +186,7 @@ type Event struct { // *Event_IdentityCenterAccount // *Event_IdentityCenterPrincipalAssignment // *Event_IdentityCenterAccountAssignment + // *Event_GitServer Resource isEvent_Resource `protobuf_oneof:"Resource"` } @@ -716,6 +717,13 @@ func (x *Event) GetIdentityCenterAccountAssignment() *v114.AccountAssignment { return nil } +func (x *Event) GetGitServer() *types.ServerV2 { + if x, ok := x.GetResource().(*Event_GitServer); ok { + return x.GitServer + } + return nil +} + type isEvent_Resource interface { isEvent_Resource() } @@ -1071,6 +1079,11 @@ type Event_IdentityCenterAccountAssignment struct { IdentityCenterAccountAssignment *v114.AccountAssignment `protobuf:"bytes,74,opt,name=IdentityCenterAccountAssignment,proto3,oneof"` } +type Event_GitServer struct { + // GitServer is a resource for Git proxy server. + GitServer *types.ServerV2 `protobuf:"bytes,75,opt,name=GitServer,proto3,oneof"` +} + func (*Event_ResourceHeader) isEvent_Resource() {} func (*Event_CertAuthority) isEvent_Resource() {} @@ -1209,6 +1222,8 @@ func (*Event_IdentityCenterPrincipalAssignment) isEvent_Resource() {} func (*Event_IdentityCenterAccountAssignment) isEvent_Resource() {} +func (*Event_GitServer) isEvent_Resource() {} + var File_teleport_legacy_client_proto_event_proto protoreflect.FileDescriptor var file_teleport_legacy_client_proto_event_proto_rawDesc = []byte{ @@ -1267,7 +1282,7 @@ var file_teleport_legacy_client_proto_event_proto_rawDesc = []byte{ 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x26, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x75, 0x73, 0x65, 0x72, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0xcf, 0x28, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x54, 0x79, + 0x22, 0x80, 0x29, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3f, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x65, 0x61, 0x64, @@ -1586,19 +1601,22 @@ var file_teleport_legacy_client_proto_event_proto_rawDesc = []byte{ 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x1f, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x41, 0x73, 0x73, 0x69, 0x67, - 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x31, 0x10, 0x32, 0x4a, 0x04, 0x08, - 0x3f, 0x10, 0x40, 0x4a, 0x04, 0x08, 0x44, 0x10, 0x45, 0x52, 0x12, 0x45, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x0e, 0x53, - 0x74, 0x61, 0x74, 0x69, 0x63, 0x48, 0x6f, 0x73, 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, 0x13, 0x41, - 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6c, - 0x61, 0x6e, 0x2a, 0x2a, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x49, 0x54, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x55, 0x54, - 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x34, - 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, - 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x09, 0x47, 0x69, 0x74, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x18, 0x4b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, + 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x56, 0x32, 0x48, 0x00, 0x52, 0x09, 0x47, 0x69, 0x74, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x31, 0x10, 0x32, 0x4a, 0x04, + 0x08, 0x3f, 0x10, 0x40, 0x4a, 0x04, 0x08, 0x44, 0x10, 0x45, 0x52, 0x12, 0x45, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x0e, + 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x48, 0x6f, 0x73, 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, 0x13, + 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, + 0x6c, 0x61, 0x6e, 0x2a, 0x2a, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x49, 0x54, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x55, + 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, + 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, + 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, + 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1756,11 +1774,12 @@ var file_teleport_legacy_client_proto_event_proto_depIdxs = []int32{ 65, // 67: proto.Event.IdentityCenterAccount:type_name -> teleport.identitycenter.v1.Account 66, // 68: proto.Event.IdentityCenterPrincipalAssignment:type_name -> teleport.identitycenter.v1.PrincipalAssignment 67, // 69: proto.Event.IdentityCenterAccountAssignment:type_name -> teleport.identitycenter.v1.AccountAssignment - 70, // [70:70] is the sub-list for method output_type - 70, // [70:70] is the sub-list for method input_type - 70, // [70:70] is the sub-list for extension type_name - 70, // [70:70] is the sub-list for extension extendee - 0, // [0:70] is the sub-list for field type_name + 10, // 70: proto.Event.GitServer:type_name -> types.ServerV2 + 71, // [71:71] is the sub-list for method output_type + 71, // [71:71] is the sub-list for method input_type + 71, // [71:71] is the sub-list for extension type_name + 71, // [71:71] is the sub-list for extension extendee + 0, // [0:71] is the sub-list for field type_name } func init() { file_teleport_legacy_client_proto_event_proto_init() } @@ -1838,6 +1857,7 @@ func file_teleport_legacy_client_proto_event_proto_init() { (*Event_IdentityCenterAccount)(nil), (*Event_IdentityCenterPrincipalAssignment)(nil), (*Event_IdentityCenterAccountAssignment)(nil), + (*Event_GitServer)(nil), } type x struct{} out := protoimpl.TypeBuilder{ diff --git a/api/proto/teleport/legacy/client/proto/event.proto b/api/proto/teleport/legacy/client/proto/event.proto index 7c0cd043eb13d..f408df09f2123 100644 --- a/api/proto/teleport/legacy/client/proto/event.proto +++ b/api/proto/teleport/legacy/client/proto/event.proto @@ -206,5 +206,7 @@ message Event { // IdentityCenterAccountlAssignment is a resource representing a potential // Permission Set grant on a specific AWS account. teleport.identitycenter.v1.AccountAssignment IdentityCenterAccountAssignment = 74; + // GitServer is a resource for Git proxy server. + types.ServerV2 GitServer = 75; } } diff --git a/lib/auth/accesspoint/accesspoint.go b/lib/auth/accesspoint/accesspoint.go index d9ac852bba65b..db4662fa85cba 100644 --- a/lib/auth/accesspoint/accesspoint.go +++ b/lib/auth/accesspoint/accesspoint.go @@ -108,6 +108,7 @@ type Config struct { AutoUpdateService services.AutoUpdateServiceGetter ProvisioningStates services.ProvisioningStates IdentityCenter services.IdentityCenter + GitServers services.GitServers } func (c *Config) CheckAndSetDefaults() error { @@ -205,6 +206,7 @@ func NewCache(cfg Config) (*cache.Cache, error) { DynamicWindowsDesktops: cfg.DynamicWindowsDesktops, ProvisioningStates: cfg.ProvisioningStates, IdentityCenter: cfg.IdentityCenter, + GitServers: cfg.GitServers, } return cache.New(cfg.Setup(cacheCfg)) diff --git a/lib/auth/authclient/clt.go b/lib/auth/authclient/clt.go index 5376d6161baa3..4c098b748e260 100644 --- a/lib/auth/authclient/clt.go +++ b/lib/auth/authclient/clt.go @@ -35,6 +35,7 @@ import ( "github.com/gravitational/teleport/api/client/databaseobject" "github.com/gravitational/teleport/api/client/dynamicwindows" "github.com/gravitational/teleport/api/client/externalauditstorage" + "github.com/gravitational/teleport/api/client/gitserver" "github.com/gravitational/teleport/api/client/proto" "github.com/gravitational/teleport/api/client/secreport" "github.com/gravitational/teleport/api/client/usertask" @@ -43,7 +44,6 @@ import ( clusterconfigpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/clusterconfig/v1" dbobjectimportrulev1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/dbobjectimportrule/v1" devicepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/devicetrust/v1" - gitserverv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/gitserver/v1" identitycenterv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/identitycenter/v1" integrationv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/integration/v1" loginrulepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/loginrule/v1" @@ -1893,5 +1893,5 @@ type ClientI interface { ProvisioningServiceClient() provisioningv1.ProvisioningServiceClient // GitServerClient returns git server client. - GitServerClient() gitserverv1.GitServerServiceClient + GitServerClient() *gitserver.Client } diff --git a/lib/auth/helpers.go b/lib/auth/helpers.go index dd1b2fcf7e414..0c5b0e07ea188 100644 --- a/lib/auth/helpers.go +++ b/lib/auth/helpers.go @@ -370,6 +370,7 @@ func NewTestAuthServer(cfg TestAuthServerConfig) (*TestAuthServer, error) { WebToken: svces.WebTokens(), WindowsDesktops: svces.WindowsDesktops, DynamicWindowsDesktops: svces.DynamicWindowsDesktops, + GitServers: svces.GitServers, }) if err != nil { return nil, trace.Wrap(err) diff --git a/lib/cache/cache.go b/lib/cache/cache.go index 2ba3dbebfbbf5..c997edaf31d69 100644 --- a/lib/cache/cache.go +++ b/lib/cache/cache.go @@ -103,6 +103,7 @@ var highVolumeResources = map[string]struct{}{ types.KindWindowsDesktopService: {}, types.KindKubeServer: {}, types.KindDatabaseObject: {}, + types.KindGitServer: {}, } func isHighVolumeResource(kind string) bool { @@ -198,6 +199,7 @@ func ForAuth(cfg Config) Config { {Kind: types.KindIdentityCenterAccount}, {Kind: types.KindIdentityCenterPrincipalAssignment}, {Kind: types.KindIdentityCenterAccountAssignment}, + {Kind: types.KindGitServer}, } cfg.QueueSize = defaults.AuthQueueSize // We don't want to enable partial health for auth cache because auth uses an event stream @@ -255,6 +257,7 @@ func ForProxy(cfg Config) Config { {Kind: types.KindAutoUpdateVersion}, {Kind: types.KindAutoUpdateAgentRollout}, {Kind: types.KindUserTask}, + {Kind: types.KindGitServer}, } cfg.QueueSize = defaults.ProxyQueueSize return cfg @@ -550,6 +553,7 @@ type Cache struct { staticHostUsersCache *local.StaticHostUserService provisioningStatesCache *local.ProvisioningStateService identityCenterCache *local.IdentityCenterService + gitServersCache *local.GitServerService // closed indicates that the cache has been closed closed atomic.Bool @@ -784,6 +788,9 @@ type Config struct { // IdentityCenter is the upstream Identity Center service that we're caching IdentityCenter services.IdentityCenter + + // GitServers is the Git serever service. + GitServers services.GitServerGetter } // CheckAndSetDefaults checks parameters and sets default values @@ -1023,6 +1030,12 @@ func New(config Config) (*Cache, error) { return nil, trace.Wrap(err) } + gitServersCache, err := local.NewGitServerService(config.Backend) + if err != nil { + cancel() + return nil, trace.Wrap(err) + } + cs := &Cache{ ctx: ctx, cancel: cancel, @@ -1070,6 +1083,7 @@ func New(config Config) (*Cache, error) { staticHostUsersCache: staticHostUserCache, provisioningStatesCache: provisioningStatesCache, identityCenterCache: identityCenterCache, + gitServersCache: gitServersCache, Logger: log.WithFields(log.Fields{ teleport.ComponentKey: config.Component, }), diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go index 2bfb56ae2916e..9a305d85e4b22 100644 --- a/lib/cache/cache_test.go +++ b/lib/cache/cache_test.go @@ -141,6 +141,7 @@ type testPack struct { autoUpdateService services.AutoUpdateService provisioningStates services.ProvisioningStates identityCenter services.IdentityCenter + gitServers services.GitServers } // testFuncs are functions to support testing an object in a cache. @@ -402,6 +403,11 @@ func newPackWithoutCache(dir string, opts ...packOption) (*testPack, error) { return nil, trace.Wrap(err) } + p.gitServers, err = local.NewGitServerService(p.backend) + if err != nil { + return nil, trace.Wrap(err) + } + return p, nil } @@ -455,6 +461,7 @@ func newPack(dir string, setupConfig func(c Config) Config, opts ...packOption) AutoUpdateService: p.autoUpdateService, ProvisioningStates: p.provisioningStates, IdentityCenter: p.identityCenter, + GitServers: p.gitServers, MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, })) @@ -869,6 +876,7 @@ func TestCompletenessInit(t *testing.T) { MaxRetryPeriod: 200 * time.Millisecond, IdentityCenter: p.identityCenter, EventsC: p.eventsC, + GitServers: p.gitServers, })) require.NoError(t, err) @@ -953,6 +961,7 @@ func TestCompletenessReset(t *testing.T) { IdentityCenter: p.identityCenter, MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, + GitServers: p.gitServers, })) require.NoError(t, err) @@ -1164,6 +1173,7 @@ func TestListResources_NodesTTLVariant(t *testing.T) { MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, neverOK: true, // ensure reads are never healthy + GitServers: p.gitServers, })) require.NoError(t, err) @@ -1258,6 +1268,7 @@ func initStrategy(t *testing.T) { IdentityCenter: p.identityCenter, MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, + GitServers: p.gitServers, })) require.NoError(t, err) @@ -3521,6 +3532,7 @@ func TestCacheWatchKindExistsInEvents(t *testing.T) { types.KindIdentityCenterAccount: types.Resource153ToLegacy(newIdentityCenterAccount("some_account")), types.KindIdentityCenterAccountAssignment: types.Resource153ToLegacy(newIdentityCenterAccountAssignment("some_account_assignment")), types.KindIdentityCenterPrincipalAssignment: types.Resource153ToLegacy(newIdentityCenterPrincipalAssignment("some_principal_assignment")), + types.KindGitServer: &types.ServerV2{}, } for name, cfg := range cases { diff --git a/lib/cache/collections.go b/lib/cache/collections.go index c6aa7c73898a3..27c91d6714bea 100644 --- a/lib/cache/collections.go +++ b/lib/cache/collections.go @@ -176,6 +176,7 @@ type cacheCollections struct { identityCenterAccounts collectionReader[identityCenterAccountGetter] identityCenterPrincipalAssignments collectionReader[identityCenterPrincipalAssignmentGetter] identityCenterAccountAssignments collectionReader[identityCenterAccountAssignmentGetter] + gitServers collectionReader[services.GitServerGetter] } // setupCollections returns a registry of collections. @@ -784,6 +785,19 @@ func setupCollections(c *Cache, watches []types.WatchKind) (*cacheCollections, e } collections.byKind[resourceKind] = collections.identityCenterAccountAssignments + case types.KindGitServer: + if c.GitServers == nil { + return nil, trace.BadParameter("missing parameter GitServers") + } + collections.gitServers = &genericCollection[ + types.Server, + services.GitServerGetter, + gitServerExecutor, + ]{ + cache: c, + watch: watch, + } + collections.byKind[resourceKind] = collections.gitServers default: return nil, trace.BadParameter("resource %q is not supported", watch.Kind) } diff --git a/lib/cache/git_server.go b/lib/cache/git_server.go new file mode 100644 index 0000000000000..495bb64d28921 --- /dev/null +++ b/lib/cache/git_server.go @@ -0,0 +1,95 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package cache + +import ( + "context" + + "github.com/gravitational/trace" + + apidefaults "github.com/gravitational/teleport/api/defaults" + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/lib/services" +) + +func (c *Cache) GetGitServer(ctx context.Context, name string) (types.Server, error) { + ctx, span := c.Tracer.Start(ctx, "cache/GetGitServer") + defer span.End() + + rg, err := readCollectionCache(c, c.collections.gitServers) + if err != nil { + return nil, trace.Wrap(err) + } + defer rg.Release() + return rg.reader.GetGitServer(ctx, name) +} + +func (c *Cache) ListGitServers(ctx context.Context, pageSize int, pageToken string) ([]types.Server, string, error) { + ctx, span := c.Tracer.Start(ctx, "cache/GetGitServers") + defer span.End() + + rg, err := readCollectionCache(c, c.collections.gitServers) + if err != nil { + return nil, "", trace.Wrap(err) + } + defer rg.Release() + return rg.reader.ListGitServers(ctx, pageSize, pageToken) +} + +type gitServerExecutor struct{} + +func (gitServerExecutor) getAll(ctx context.Context, cache *Cache, loadSecrets bool) (all []types.Server, err error) { + var page []types.Server + var nextToken string + for { + page, nextToken, err = cache.gitServersCache.ListGitServers(ctx, apidefaults.DefaultChunkSize, nextToken) + if err != nil { + return nil, trace.Wrap(err) + } + all = append(all, page...) + if nextToken == "" { + break + } + } + return all, nil +} + +func (gitServerExecutor) upsert(ctx context.Context, cache *Cache, resource types.Server) error { + _, err := cache.gitServersCache.UpsertGitServer(ctx, resource) + return trace.Wrap(err) +} + +func (gitServerExecutor) deleteAll(ctx context.Context, cache *Cache) error { + return cache.gitServersCache.DeleteAllGitServers(ctx) +} + +func (gitServerExecutor) delete(ctx context.Context, cache *Cache, resource types.Resource) error { + return cache.gitServersCache.DeleteGitServer(ctx, resource.GetName()) +} + +func (gitServerExecutor) isSingleton() bool { return false } + +func (gitServerExecutor) getReader(cache *Cache, cacheOK bool) services.GitServerGetter { + if cacheOK { + return cache.gitServersCache + } + return cache.Config.GitServers +} + +var _ executor[types.Server, services.GitServerGetter] = gitServerExecutor{} diff --git a/lib/cache/git_server_test.go b/lib/cache/git_server_test.go new file mode 100644 index 0000000000000..35ce40941a53b --- /dev/null +++ b/lib/cache/git_server_test.go @@ -0,0 +1,65 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package cache + +import ( + "context" + "testing" + + "github.com/gravitational/trace" + "github.com/stretchr/testify/require" + + "github.com/gravitational/teleport/api/types" +) + +func TestGitServers(t *testing.T) { + t.Parallel() + + p, err := newPack(t.TempDir(), ForAuth) + require.NoError(t, err) + t.Cleanup(p.Close) + + testResources(t, p, testFuncs[types.Server]{ + newResource: func(name string) (types.Server, error) { + return types.NewGitHubServer( + types.GitHubServerMetadata{ + Integration: name, + Organization: name, + }) + }, + create: func(ctx context.Context, server types.Server) error { + _, err := p.gitServers.CreateGitServer(ctx, server) + return trace.Wrap(err) + }, + list: func(ctx context.Context) ([]types.Server, error) { + servers, _, err := p.gitServers.ListGitServers(ctx, 0, "") + return servers, trace.Wrap(err) + }, + update: func(ctx context.Context, server types.Server) error { + _, err := p.gitServers.UpdateGitServer(ctx, server) + return trace.Wrap(err) + }, + deleteAll: p.gitServers.DeleteAllGitServers, + cacheList: func(ctx context.Context) ([]types.Server, error) { + servers, _, err := p.cache.ListGitServers(ctx, 0, "") + return servers, trace.Wrap(err) + }, + cacheGet: p.cache.GetGitServer, + }) +} diff --git a/lib/service/service.go b/lib/service/service.go index ef30805a0fa7d..f57faa560be95 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -2544,6 +2544,7 @@ func (process *TeleportProcess) newAccessCacheForServices(cfg accesspoint.Config cfg.AutoUpdateService = services.AutoUpdateService cfg.ProvisioningStates = services.ProvisioningStates cfg.IdentityCenter = services.IdentityCenter + cfg.GitServers = services.GitServers return accesspoint.NewCache(cfg) } @@ -2590,6 +2591,7 @@ func (process *TeleportProcess) newAccessCacheForClient(cfg accesspoint.Config, cfg.WindowsDesktops = client cfg.DynamicWindowsDesktops = client.DynamicDesktopClient() cfg.AutoUpdateService = client + cfg.GitServers = client.GitServerClient() return accesspoint.NewCache(cfg) } diff --git a/lib/services/local/events.go b/lib/services/local/events.go index 2f1c9854da9cd..8725f29699a9d 100644 --- a/lib/services/local/events.go +++ b/lib/services/local/events.go @@ -248,6 +248,8 @@ func (e *EventsService) NewWatcher(ctx context.Context, watch types.Watch) (type parser = newIdentityCenterPrincipalAssignmentParser() case types.KindIdentityCenterAccountAssignment: parser = newIdentityCenterAccountAssignmentParser() + case types.KindGitServer: + parser = newGitServerParser() default: if watch.AllowPartialSuccess { continue diff --git a/lib/services/local/git_server.go b/lib/services/local/git_server.go index 2d7d7c58b3263..4d0885da38e66 100644 --- a/lib/services/local/git_server.go +++ b/lib/services/local/git_server.go @@ -20,9 +20,11 @@ package local import ( "context" + "strings" "github.com/gravitational/trace" + apidefaults "github.com/gravitational/teleport/api/defaults" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/backend" "github.com/gravitational/teleport/lib/defaults" @@ -128,3 +130,44 @@ func (s *GitServerService) ListGitServers(ctx context.Context, pageSize int, pag } return items, next, nil } + +func newGitServerParser() *gitServerParser { + return &gitServerParser{ + baseParser: newBaseParser(backend.NewKey(gitServerPrefix)), + } +} + +type gitServerParser struct { + baseParser +} + +func (p *gitServerParser) parse(event backend.Event) (types.Resource, error) { + switch event.Type { + case types.OpDelete: + name := event.Item.Key.TrimPrefix(backend.NewKey(gitServerPrefix)).String() + if name == "" { + return nil, trace.NotFound("failed parsing %v", event.Item.Key.String()) + } + + return &types.ResourceHeader{ + Kind: types.KindGitServer, + Version: types.V2, + Metadata: types.Metadata{ + Name: strings.TrimPrefix(name, backend.SeparatorString), + Namespace: apidefaults.Namespace, + }, + }, nil + case types.OpPut: + resource, err := services.UnmarshalServer(event.Item.Value, + types.KindGitServer, + services.WithExpires(event.Item.Expires), + services.WithRevision(event.Item.Revision), + ) + if err != nil { + return nil, trace.Wrap(err) + } + return resource, nil + default: + return nil, trace.BadParameter("event %v is not supported", event.Type) + } +} diff --git a/tool/tctl/common/resource_command.go b/tool/tctl/common/resource_command.go index 5dd610fb90819..5632e9fadda1b 100644 --- a/tool/tctl/common/resource_command.go +++ b/tool/tctl/common/resource_command.go @@ -49,7 +49,6 @@ import ( dbobjectv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/dbobject/v1" dbobjectimportrulev1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/dbobjectimportrule/v1" devicepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/devicetrust/v1" - gitserverv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/gitserver/v1" loginrulepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/loginrule/v1" machineidv1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/machineid/v1" pluginsv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/plugins/v1" @@ -1978,7 +1977,7 @@ func (rc *ResourceCommand) Delete(ctx context.Context, client *authclient.Client } fmt.Printf("static host user %q has been deleted\n", rc.ref.Name) case types.KindGitServer: - if _, err := client.GitServerClient().DeleteGitServer(ctx, &gitserverv1.DeleteGitServerRequest{Name: rc.ref.Name}); err != nil { + if err := client.GitServerClient().DeleteGitServer(ctx, rc.ref.Name); err != nil { return trace.Wrap(err) } fmt.Printf("git_server %q has been deleted\n", rc.ref.Name) @@ -3220,29 +3219,27 @@ func (rc *ResourceCommand) getCollection(ctx context.Context, client *authclient } return &accessMonitoringRuleCollection{items: rules}, nil case types.KindGitServer: - var servers []types.Server + var page, servers []types.Server // TODO(greedy52) use unified resource request once available. if rc.ref.Name != "" { - server, err := client.GitServerClient().GetGitServer(ctx, &gitserverv1.GetGitServerRequest{Name: rc.ref.Name}) + server, err := client.GitServerClient().GetGitServer(ctx, rc.ref.Name) if err != nil { return nil, trace.Wrap(err) } return &serverCollection{servers: append(servers, server)}, nil } - req := &gitserverv1.ListGitServersRequest{} + var err error + var token string for { - resp, err := client.GitServerClient().ListGitServers(ctx, req) + page, token, err = client.GitServerClient().ListGitServers(ctx, 0, token) if err != nil { return nil, trace.Wrap(err) } - for _, server := range resp.Servers { - servers = append(servers, server) - } - if resp.NextPageToken == "" { + servers = append(servers, page...) + if token == "" { break } - req.PageToken = resp.NextPageToken } // TODO(greedy52) consider making dedicated git server collection. return &serverCollection{servers: servers}, nil @@ -3664,14 +3661,10 @@ func (rc *ResourceCommand) createGitServer(ctx context.Context, client *authclie if err != nil { return trace.Wrap(err) } - serverV2, ok := server.(*types.ServerV2) - if !ok { - return trace.CompareFailed("expecting types.ServerV2 but got %T", server) - } if rc.IsForced() { - _, err = client.GitServerClient().UpsertGitServer(ctx, &gitserverv1.UpsertGitServerRequest{Server: serverV2}) + _, err = client.GitServerClient().UpsertGitServer(ctx, server) } else { - _, err = client.GitServerClient().CreateGitServer(ctx, &gitserverv1.CreateGitServerRequest{Server: serverV2}) + _, err = client.GitServerClient().CreateGitServer(ctx, server) } if err != nil { return trace.Wrap(err) @@ -3684,11 +3677,7 @@ func (rc *ResourceCommand) updateGitServer(ctx context.Context, client *authclie if err != nil { return trace.Wrap(err) } - serverV2, ok := server.(*types.ServerV2) - if !ok { - return trace.CompareFailed("expecting types.ServerV2 but got %T", server) - } - _, err = client.GitServerClient().UpdateGitServer(ctx, &gitserverv1.UpdateGitServerRequest{Server: serverV2}) + _, err = client.GitServerClient().UpdateGitServer(ctx, server) if err != nil { return trace.Wrap(err) }