Skip to content

Commit

Permalink
Merge pull request #2683 from Tharsanan1/go-enforcer-new
Browse files Browse the repository at this point in the history 
Set clustername through external processing
  • Loading branch information
@Tharsanan1
Tharsanan1 authored Jan 10, 2025
2 parents f4b53c5 + a851373 commit dba5fa4
Show file tree
Hide file tree
Showing 20 changed files with 456 additions and 290 deletions.
20 changes: 11 additions & 9 deletions adapter/internal/oasparser/envoyconf/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const (
httpConManagerStartPrefix string = "ingress_http"
extAuthzPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute"
extProcPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute"
ratelimitPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimitPerRoute"
ratelimitPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimitPerRoute"
luaPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute"
corsFilterName string = "type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors"
localRateLimitPerRouteName string = "type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit"
Expand All @@ -52,14 +52,16 @@ const (
// These values are shared between the adapter and enforcer, hence if it is required to change
// these values, modifications should be done in the both adapter and enforcer.
const (
pathContextExtension string = "path"
vHostContextExtension string = "vHost"
basePathContextExtension string = "basePath"
methodContextExtension string = "method"
apiVersionContextExtension string = "version"
apiNameContextExtension string = "name"
clusterNameContextExtension string = "clusterName"
retryPolicyRetriableStatusCodes string = "retriable-status-codes"
pathAttribute string = "path"
vHostAttribute string = "vHost"
basePathAttribute string = "basePath"
methodAttribute string = "method"
apiVersionAttribute string = "version"
apiNameAttribute string = "name"
clusterNameAttribute string = "clusterName"
enableBackendBasedAIRatelimitAttribute string = "enableBackendBasedAIRatelimit"
backendBasedAIRatelimitDescriptorValueAttribute string = "backendBasedAIRatelimitDescriptorValue"
retryPolicyRetriableStatusCodes string = "retriable-status-codes"
)

const (
Expand Down
8 changes: 5 additions & 3 deletions adapter/internal/oasparser/envoyconf/http_filters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,10 +241,12 @@ func getExtProcessHTTPFilter() *hcmv3.HttpFilter {
},
FailureModeAllow: true,
ProcessingMode: &ext_process.ProcessingMode{
ResponseBodyMode: ext_process.ProcessingMode_BUFFERED,
// ResponseBodyMode: ext_process.ProcessingMode_BUFFERED,
RequestHeaderMode: ext_process.ProcessingMode_SEND,
ResponseHeaderMode: ext_process.ProcessingMode_SEND,
RequestBodyMode: ext_process.ProcessingMode_BUFFERED,
// RequestHeaderMode: ext_process.ProcessingMode_SKIP,
// ResponseHeaderMode: ext_process.ProcessingMode_SKIP,
// RequestBodyMode: ext_process.ProcessingMode_BUFFERED,
},
MetadataOptions: &ext_process.MetadataOptions{
ForwardingNamespaces: &ext_process.MetadataOptions_MetadataNamespaces{
Expand All @@ -256,7 +258,7 @@ func getExtProcessHTTPFilter() *hcmv3.HttpFilter {
},
RequestAttributes: []string{"xds.route_metadata"},
ResponseAttributes: []string{"xds.route_metadata"},
MessageTimeout: durationpb.New(conf.Envoy.EnforcerResponseTimeoutInSeconds * time.Second),
MessageTimeout: durationpb.New(conf.Envoy.EnforcerResponseTimeoutInSeconds * time.Second),
}
ext, err2 := anypb.New(externalProcessor)
if err2 != nil {
Expand Down
106 changes: 73 additions & 33 deletions adapter/internal/oasparser/envoyconf/routes_with_clusters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -831,21 +831,21 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error
pathMatchType := resource.GetPathMatchType()

contextExtensions := make(map[string]string)
contextExtensions[pathContextExtension] = resourcePath
contextExtensions[vHostContextExtension] = vHost
contextExtensions[pathAttribute] = resourcePath
contextExtensions[vHostAttribute] = vHost
if xWso2Basepath != "" {
contextExtensions[basePathContextExtension] = xWso2Basepath
contextExtensions[basePathAttribute] = xWso2Basepath
} else {
contextExtensions[basePathContextExtension] = endpointBasepath
contextExtensions[basePathAttribute] = endpointBasepath
}
contextExtensions[methodContextExtension] = strings.Join(resourceMethods, " ")
contextExtensions[apiVersionContextExtension] = version
contextExtensions[apiNameContextExtension] = title
contextExtensions[methodAttribute] = strings.Join(resourceMethods, " ")
contextExtensions[apiVersionAttribute] = version
contextExtensions[apiNameAttribute] = title
// One of these values will be selected and added as the cluster-header http header
// from enhancer
// Even if the routing is based on direct cluster, these properties needs to be populated
// to validate the key type component in the token.
contextExtensions[clusterNameContextExtension] = clusterName
contextExtensions[clusterNameAttribute] = clusterName

// extAuthPerFilterConfig := extAuthService.ExtAuthzPerRoute{
// Override: &extAuthService.ExtAuthzPerRoute_CheckSettings{
Expand Down Expand Up @@ -904,13 +904,13 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error
// so, no need to change two places
iInvCtx := &interceptor.InvocationContext{
OrganizationID: params.organizationID,
BasePath: contextExtensions[basePathContextExtension],
SupportedMethods: contextExtensions[methodContextExtension],
APIName: contextExtensions[apiNameContextExtension],
APIVersion: contextExtensions[apiVersionContextExtension],
PathTemplate: contextExtensions[pathContextExtension],
Vhost: contextExtensions[vHostContextExtension],
ClusterName: contextExtensions[clusterNameContextExtension],
BasePath: contextExtensions[basePathAttribute],
SupportedMethods: contextExtensions[methodAttribute],
APIName: contextExtensions[apiNameAttribute],
APIVersion: contextExtensions[apiVersionAttribute],
PathTemplate: contextExtensions[pathAttribute],
Vhost: contextExtensions[vHostAttribute],
ClusterName: contextExtensions[clusterNameAttribute],
APIProperties: getAPIProperties(params.apiProperties),
Environment: params.environment,
}
Expand All @@ -935,8 +935,8 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error
corsFilter, _ := anypb.New(corsPolicy)
perRouteFilterConfigs := map[string]*any.Any{
// wellknown.HTTPExternalAuthorization: extAuthzFilter,
LuaLocal: luaFilter,
wellknown.CORS: corsFilter,
LuaLocal: luaFilter,
wellknown.CORS: corsFilter,
}
// if !params.isAiAPI {
// perFilterConfigExtProc := extProcessorv3.ExtProcPerRoute{
Expand Down Expand Up @@ -1038,28 +1038,68 @@ func createRoutes(params *routeCreateParams) (routes []*routev3.Route, err error
}
routeConfig := resource.GetEndpoints().Config
metaData := &corev3.Metadata{}
if params.isAiAPI {
metaData = &corev3.Metadata{
FilterMetadata: map[string]*structpb.Struct{
"envoy.filters.http.ext_proc": &structpb.Struct{
Fields: map[string]*structpb.Value{
"EnableBackendBasedAIRatelimit": &structpb.Value{
Kind: &structpb.Value_StringValue{
StringValue: fmt.Sprintf("%t", resource.GetEnableBackendBasedAIRatelimit()),
},
// if params.isAiAPI {
metaData = &corev3.Metadata{
FilterMetadata: map[string]*structpb.Struct{
"envoy.filters.http.ext_proc": &structpb.Struct{
Fields: map[string]*structpb.Value{
enableBackendBasedAIRatelimitAttribute: &structpb.Value{
Kind: &structpb.Value_StringValue{
StringValue: fmt.Sprintf("%t", resource.GetEnableBackendBasedAIRatelimit()),
},
},
backendBasedAIRatelimitDescriptorValueAttribute: &structpb.Value{
Kind: &structpb.Value_StringValue{
StringValue: resource.GetBackendBasedAIRatelimitDescriptorValue(),
},
"BackendBasedAIRatelimitDescriptorValue": &structpb.Value{
Kind: &structpb.Value_StringValue{
StringValue: resource.GetBackendBasedAIRatelimitDescriptorValue(),
},
},
pathAttribute: &structpb.Value{ // Use the variable here
Kind: &structpb.Value_StringValue{
StringValue: resourcePath,
},
},
vHostAttribute: &structpb.Value{ // Use the variable here
Kind: &structpb.Value_StringValue{
StringValue: vHost,
},
},
basePathAttribute: &structpb.Value{ // Use the variable here
Kind: &structpb.Value_StringValue{
StringValue: func() string {
if xWso2Basepath != "" {
return xWso2Basepath
}
return endpointBasepath
}(),
},
},
methodAttribute: &structpb.Value{ // Use the variable here
Kind: &structpb.Value_StringValue{
StringValue: strings.Join(resourceMethods, " "),
},
},
apiVersionAttribute: &structpb.Value{ // Use the variable here
Kind: &structpb.Value_StringValue{
StringValue: version,
},
},
apiNameAttribute: &structpb.Value{ // Use the variable here
Kind: &structpb.Value_StringValue{
StringValue: title,
},
},
clusterNameAttribute: &structpb.Value{ // Use the variable here
Kind: &structpb.Value_StringValue{
StringValue: clusterName,
},
},
},
},
}
} else {
metaData = nil
},
}
// } else {
// metaData = nil
// }
if resource.HasPolicies() {
logger.LoggerOasparser.Debug("Start creating routes for resource with policies")
operations := resource.GetOperations()
Expand Down
2 changes: 1 addition & 1 deletion gateway/enforcer/cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func main() {

//Create the TLS configuration
tlsConfig := util.CreateTLSConfig(clientCert, certPool)
client := grpc.NewEventingGRPCClient(host, port, cfg.XdsMaxRetries, time.Duration(cfg.XdsRetryPeriod)*time.Second, tlsConfig, cfg, nil)
client := grpc.NewEventingGRPCClient(host, port, cfg.XdsMaxRetries, time.Duration(cfg.XdsRetryPeriod)*time.Millisecond, tlsConfig, cfg, nil)
// Start the connection
client.InitiateEventingGRPCConnection()

Expand Down
84 changes: 42 additions & 42 deletions gateway/enforcer/internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
* limitations under the License.
*
*/
package config

package config

import (
"sync"
Expand All @@ -27,46 +27,46 @@ import (

// Server holds the configuration parameters for the application.
type Server struct {
TrustedAdapterCertsPath string `envconfig:"TRUSTED_CA_CERTS_PATH" default:"/home/wso2/security/truststore"`
TrustDefaultCerts string `envconfig:"TRUST_DEFAULT_CERTS" default:"true"`
EnforcerPrivateKeyPath string `envconfig:"ENFORCER_PRIVATE_KEY_PATH" default:"/home/wso2/security/keystore/mg.key"`
EnforcerPublicKeyPath string `envconfig:"ENFORCER_PUBLIC_CERT_PATH" default:"/home/wso2/security/keystore/mg.pem"`
OpaClientPrivateKeyPath string `envconfig:"OPA_CLIENT_PRIVATE_KEY_PATH" default:"/home/wso2/security/keystore/mg.key"`
OpaClientPublicKeyPath string `envconfig:"OPA_CLIENT_PUBLIC_CERT_PATH" default:"/home/wso2/security/keystore/mg.pem"`
AdapterHost string `envconfig:"ADAPTER_HOST" default:"adapter"`
RatelimiterHost string `envconfig:"RATELIMITER_HOST" default:"apk-test-wso2-apk-ratelimiter-service.apk.svc"`
RatelimiterPort string `envconfig:"RATELIMITER_PORT" default:"8091"`
AdapterHostname string `envconfig:"ADAPTER_HOST_NAME" default:"adapter"`
AdapterXdsPort string `envconfig:"ADAPTER_XDS_PORT" default:"18000"`
CommonControllerHost string `envconfig:"COMMON_CONTROLLER_HOST" default:"common-controller"`
CommonControllerHostname string `envconfig:"COMMON_CONTROLLER_HOST_NAME" default:"common-controller"`
CommonControllerXdsPort string `envconfig:"COMMON_CONTROLLER_XDS_PORT" default:"18002"`
CommonControllerRestPort string `envconfig:"COMMON_CONTROLLER_REST_PORT" default:"18003"`
XdsMaxMsgSize int `envconfig:"XDS_MAX_MSG_SIZE" default:"4194304"`
EnforcerLabel string `envconfig:"ENFORCER_LABEL" default:"enforcer"`
EnforcerRegionID string `envconfig:"ENFORCER_REGION" default:"UNKNOWN"`
XdsMaxRetries int `envconfig:"XDS_MAX_RETRIES" default:"3"`
XdsRetryPeriod int `envconfig:"XDS_RETRY_PERIOD" default:"5000"` // milliseconds
InstanceIdentifier string `envconfig:"HOSTNAME" default:"Unassigned"`
RedisUsername string `envconfig:"REDIS_USERNAME" default:""`
RedisPassword string `envconfig:"REDIS_PASSWORD" default:""`
RedisHost string `envconfig:"REDIS_HOST" default:"redis-master"`
RedisPort int `envconfig:"REDIS_PORT" default:"6379"`
IsRedisTLSEnabled bool `envconfig:"IS_REDIS_TLS_ENABLED" default:"false"`
RevokedTokensRedisChannel string `envconfig:"REDIS_REVOKED_TOKENS_CHANNEL" default:"wso2-apk-revoked-tokens-channel"`
RedisKeyFile string `envconfig:"REDIS_KEY_FILE" default:"/home/wso2/security/redis/redis.key"`
RedisCertFile string `envconfig:"REDIS_CERT_FILE" default:"/home/wso2/security/redis/redis.crt"`
RedisCaCertFile string `envconfig:"REDIS_CA_CERT_FILE" default:"/home/wso2/security/redis/ca.crt"`
RevokedTokenCleanupInterval int `envconfig:"REVOKED_TOKEN_CLEANUP_INTERVAL" default:"3600"` // seconds
ChoreoAnalyticsAuthToken string `envconfig:"CHOREO_ANALYTICS_AUTH_TOKEN" default:""`
ChoreoAnalyticsAuthURL string `envconfig:"CHOREO_ANALYTICS_AUTH_URL" default:""`
MoesifToken string `envconfig:"MOESIF_TOKEN" default:""`
LogLevel string `envconfig:"LOG_LEVEL" default:"INFO"`
ExternalProcessingPort string `envconfig:"EXTERNAL_PROCESSING_PORT" default:"8081"`
ExternalProcessingKeepAliveTime int `envconfig:"EXTERNAL_PROCESSING_KEEP_ALIVE_TIME" default:"600"`
ExternalProcessingMaxMessageSize int `envconfig:"EXTERNAL_PROCESSING_MAX_MESSAGE_SIZE" default:"1000000000"`
ExternalProcessingMaxHeaderLimit int `envconfig:"EXTERNAL_PROCESSING_MAX_HEADER_LIMIT" default:"8192"`
Logger logging.Logger
TrustedAdapterCertsPath string `envconfig:"TRUSTED_CA_CERTS_PATH" default:"/home/wso2/security/truststore"`
TrustDefaultCerts string `envconfig:"TRUST_DEFAULT_CERTS" default:"true"`
EnforcerPrivateKeyPath string `envconfig:"ENFORCER_PRIVATE_KEY_PATH" default:"/home/wso2/security/keystore/mg.key"`
EnforcerPublicKeyPath string `envconfig:"ENFORCER_PUBLIC_CERT_PATH" default:"/home/wso2/security/keystore/mg.pem"`
OpaClientPrivateKeyPath string `envconfig:"OPA_CLIENT_PRIVATE_KEY_PATH" default:"/home/wso2/security/keystore/mg.key"`
OpaClientPublicKeyPath string `envconfig:"OPA_CLIENT_PUBLIC_CERT_PATH" default:"/home/wso2/security/keystore/mg.pem"`
AdapterHost string `envconfig:"ADAPTER_HOST" default:"adapter"`
RatelimiterHost string `envconfig:"RATELIMITER_HOST" default:"apk-test-wso2-apk-ratelimiter-service.apk.svc"`
RatelimiterPort string `envconfig:"RATELIMITER_PORT" default:"8091"`
AdapterHostname string `envconfig:"ADAPTER_HOST_NAME" default:"adapter"`
AdapterXdsPort string `envconfig:"ADAPTER_XDS_PORT" default:"18000"`
CommonControllerHost string `envconfig:"COMMON_CONTROLLER_HOST" default:"common-controller"`
CommonControllerHostname string `envconfig:"COMMON_CONTROLLER_HOST_NAME" default:"common-controller"`
CommonControllerXdsPort string `envconfig:"COMMON_CONTROLLER_XDS_PORT" default:"18002"`
CommonControllerRestPort string `envconfig:"COMMON_CONTROLLER_REST_PORT" default:"18003"`
XdsMaxMsgSize int `envconfig:"XDS_MAX_MSG_SIZE" default:"4194304"`
EnforcerLabel string `envconfig:"ENFORCER_LABEL" default:"enforcer"`
EnforcerRegionID string `envconfig:"ENFORCER_REGION" default:"UNKNOWN"`
XdsMaxRetries int `envconfig:"XDS_MAX_RETRIES" default:"3"`
XdsRetryPeriod int `envconfig:"XDS_RETRY_PERIOD" default:"5000"` // milliseconds
InstanceIdentifier string `envconfig:"HOSTNAME" default:"Unassigned"`
RedisUsername string `envconfig:"REDIS_USERNAME" default:""`
RedisPassword string `envconfig:"REDIS_PASSWORD" default:""`
RedisHost string `envconfig:"REDIS_HOST" default:"redis-master"`
RedisPort int `envconfig:"REDIS_PORT" default:"6379"`
IsRedisTLSEnabled bool `envconfig:"IS_REDIS_TLS_ENABLED" default:"false"`
RevokedTokensRedisChannel string `envconfig:"REDIS_REVOKED_TOKENS_CHANNEL" default:"wso2-apk-revoked-tokens-channel"`
RedisKeyFile string `envconfig:"REDIS_KEY_FILE" default:"/home/wso2/security/redis/redis.key"`
RedisCertFile string `envconfig:"REDIS_CERT_FILE" default:"/home/wso2/security/redis/redis.crt"`
RedisCaCertFile string `envconfig:"REDIS_CA_CERT_FILE" default:"/home/wso2/security/redis/ca.crt"`
RevokedTokenCleanupInterval int `envconfig:"REVOKED_TOKEN_CLEANUP_INTERVAL" default:"3600"` // seconds
ChoreoAnalyticsAuthToken string `envconfig:"CHOREO_ANALYTICS_AUTH_TOKEN" default:""`
ChoreoAnalyticsAuthURL string `envconfig:"CHOREO_ANALYTICS_AUTH_URL" default:""`
MoesifToken string `envconfig:"MOESIF_TOKEN" default:""`
LogLevel string `envconfig:"LOG_LEVEL" default:"INFO"`
ExternalProcessingPort string `envconfig:"EXTERNAL_PROCESSING_PORT" default:"8081"`
ExternalProcessingKeepAliveTime int `envconfig:"EXTERNAL_PROCESSING_KEEP_ALIVE_TIME" default:"600"`
ExternalProcessingMaxMessageSize int `envconfig:"EXTERNAL_PROCESSING_MAX_MESSAGE_SIZE" default:"1000000000"`
ExternalProcessingMaxHeaderLimit int `envconfig:"EXTERNAL_PROCESSING_MAX_HEADER_LIMIT" default:"8192"`
Logger logging.Logger
}

// package-level variable and mutex for thread safety
Expand Down
5 changes: 3 additions & 2 deletions gateway/enforcer/internal/datastore/api_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@
package datastore

import (
api "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/api"
"sync"

api "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/api"
)

// APIStore is a thread-safe store for APIs.
Expand Down Expand Up @@ -49,4 +50,4 @@ func (s *APIStore) GetAPIs() []*api.Api {
s.mu.RLock()
defer s.mu.RUnlock()
return s.apis
}
}
13 changes: 7 additions & 6 deletions gateway/enforcer/internal/datastore/config_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,19 @@
* limitations under the License.
*
*/
package datastore

package datastore

import (
config_from_adapter "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/config/enforcer"
"sync"

config_from_adapter "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/config/enforcer"
)

// ConfigStore is a thread-safe store for APIs.
type ConfigStore struct {
configs []*config_from_adapter.Config
mu sync.RWMutex
mu sync.RWMutex
}

// NewConfigStore creates a new instance of ConfigStore.
Expand All @@ -44,9 +45,9 @@ func (s *ConfigStore) AddConfigs(apis []*config_from_adapter.Config) {
}

// GetConfigs retrieves the list of Config from the store.
// This method is thread-safe.
// This method is thread-safe.
func (s *ConfigStore) GetConfigs() []*config_from_adapter.Config {
s.mu.RLock()
defer s.mu.RUnlock()
return s.configs
}
}
11 changes: 6 additions & 5 deletions gateway/enforcer/internal/datastore/jwt_issuer_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,19 @@
* limitations under the License.
*
*/
package datastore

package datastore

import (
subscription "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/subscription"
"sync"

subscription "github.com/wso2/apk/adapter/pkg/discovery/api/wso2/discovery/subscription"
)

// JWTIssuerStore is a thread-safe store for APIs.
type JWTIssuerStore struct {
jwtIssuers []*subscription.JWTIssuer
mu sync.RWMutex
mu sync.RWMutex
}

// NewJWTIssuerStore creates a new instance of JWTIssuerStore.
Expand All @@ -49,4 +50,4 @@ func (s *JWTIssuerStore) GetJWTIssuers() []*subscription.JWTIssuer {
s.mu.RLock()
defer s.mu.RUnlock()
return s.jwtIssuers
}
}
Loading

0 comments on commit dba5fa4

Please sign in to comment.