Skip to content

Commit

Permalink
matcher: Implement CEL matcher (envoyproxy#27527)
Browse files Browse the repository at this point in the history
Signed-off-by: tyxia <[email protected]>
  • Loading branch information
tyxia authored May 31, 2023
1 parent 644e991 commit 702edcd
Show file tree
Hide file tree
Showing 21 changed files with 1,401 additions and 29 deletions.
4 changes: 4 additions & 0 deletions CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,14 @@ extensions/filters/http/oauth2 @derekargueta @snowp
/*/extensions/matching/input_matchers/consistent_hashing @snowp @donyu
# runtime fraction input matcher
/*/extensions/matching/input_matchers/runtime_fraction @ravenblackx @UNOWNED
# CEL input matcher
/*/extensions/matching/input_matchers/cel_matcher @tyxia @UNOWNED
# environment generic input
/*/extensions/matching/common_inputs/environment @snowp @donyu
# format string matching
/*/extensions/matching/actions/format_string @kyessenov @UNOWNED
# CEL data input
/*/extensions/matching/http/cel_input @tyxia @UNOWNED
# user space socket pair, event, connection and listener
/*/extensions/io_socket/user_space @kyessenov @UNOWNED
/*/extensions/bootstrap/internal_listener @kyessenov @adisuissa
Expand Down
6 changes: 5 additions & 1 deletion bazel/repository_locations.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -1102,13 +1102,15 @@ REPOSITORY_LOCATIONS_SPEC = dict(
"envoy.stat_sinks.wasm",
"envoy.rbac.matchers.upstream_ip_port",
"envoy.formatter.cel",
"envoy.matching.inputs.cel_data_input",
"envoy.matching.matchers.cel_matcher",
],
release_date = "2023-03-08",
cpe = "N/A",
),
com_github_google_flatbuffers = dict(
project_name = "FlatBuffers",
project_desc = "Cross platform serialization library architected for maximum memory efficiency",
project_desc = "FlatBuffers is a cross platform serialization library architected for maximum memory efficiency",
project_url = "https://github.com/google/flatbuffers",
version = "23.3.3",
sha256 = "8aff985da30aaab37edf8e5b02fda33ed4cbdd962699a8e2af98fdef306f4e4d",
Expand All @@ -1127,6 +1129,8 @@ REPOSITORY_LOCATIONS_SPEC = dict(
"envoy.filters.network.wasm",
"envoy.stat_sinks.wasm",
"envoy.rbac.matchers.upstream_ip_port",
"envoy.matching.inputs.cel_data_input",
"envoy.matching.matchers.cel_matcher",
],
release_date = "2023-03-03",
cpe = "cpe:2.3:a:google:flatbuffers:*",
Expand Down
4 changes: 4 additions & 0 deletions changelogs/current.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,10 @@ new_features:
change: |
added new field ``filter_metadata <envoy_v3_api_field_extensions.filters.http.ext_proc.v3.ExtProc.filter_metadata`` to aid in logging.
Metadata will be stored in StreamInfo filter state under a namespace corresponding to the name of the ext proc filter.
- area: matching
change: |
added CEL(Common Expression Language) matcher support :ref:`CEL data input <extension_envoy.matching.inputs.cel_data_input>`
and :ref:`CEL input matcher <extension_envoy.matching.matchers.cel_matcher>`.
deprecated:
- area: access_log
Expand Down
10 changes: 10 additions & 0 deletions docs/root/intro/arch_overview/advanced/matching/matching_api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,16 @@ are available in some contexts:

* :ref:`Trie-based IP matcher <envoy_v3_api_msg_.xds.type.matcher.v3.IPMatcher>` applies to network inputs.

* `Common Expression Language <https://github.com/google/cel-spec>`_ (CEL) based matching:

.. _extension_envoy.matching.inputs.cel_data_input:

* CEL matching data input: :ref:`CEL data input value <envoy_v3_api_msg_.xds.type.matcher.v3.HttpAttributesCelMatchInput>`.

.. _extension_envoy.matching.matchers.cel_matcher:

* CEL matching input matcher: :ref:`CEL input matcher <envoy_v3_api_msg_.xds.type.matcher.v3.CelMatcher>`.

Matching actions
################

Expand Down
6 changes: 6 additions & 0 deletions source/extensions/extensions_build_config.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ EXTENSIONS = {
"envoy.matching.matchers.consistent_hashing": "//source/extensions/matching/input_matchers/consistent_hashing:config",
"envoy.matching.matchers.ip": "//source/extensions/matching/input_matchers/ip:config",
"envoy.matching.matchers.runtime_fraction": "//source/extensions/matching/input_matchers/runtime_fraction:config",
"envoy.matching.matchers.cel_matcher": "//source/extensions/matching/input_matchers/cel_matcher:config",

#
# Network Matchers
Expand All @@ -102,6 +103,11 @@ EXTENSIONS = {

"envoy.matching.common_inputs.environment_variable": "//source/extensions/matching/common_inputs/environment_variable:config",

#
# CEL Matching Input
#
"envoy.matching.inputs.cel_data_input": "//source/extensions/matching/http/cel_input:cel_input_lib",

#
# Matching actions
#
Expand Down
14 changes: 14 additions & 0 deletions source/extensions/extensions_metadata.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,13 @@ envoy.matching.matchers.runtime_fraction:
status: stable
type_urls:
- envoy.extensions.matching.input_matchers.runtime_fraction.v3.RuntimeFraction
envoy.matching.matchers.cel_matcher:
categories:
- envoy.matching.input_matchers
security_posture: robust_to_untrusted_downstream_and_upstream
status: stable
type_urls:
- xds.type.matcher.v3.CelMatcher
envoy.path.match.uri_template.uri_template_matcher:
categories:
- envoy.path.match
Expand Down Expand Up @@ -1349,6 +1356,13 @@ envoy.matching.inputs.query_params:
status: alpha
type_urls:
- envoy.type.matcher.v3.HttpRequestQueryParamMatchInput
envoy.matching.inputs.cel_data_input:
categories:
- envoy.matching.http.input
security_posture: unknown
status: alpha
type_urls:
- xds.type.matcher.v3.HttpAttributesCelMatchInput
envoy.matching.inputs.destination_ip:
categories:
- envoy.matching.http.input
Expand Down
32 changes: 17 additions & 15 deletions source/extensions/filters/common/expr/context.h
Original file line number Diff line number Diff line change
Expand Up @@ -99,10 +99,10 @@ using WrapperFields = ConstSingleton<WrapperFieldValues>;

class RequestWrapper;

absl::optional<CelValue> convertHeaderEntry(const Http::HeaderEntry* header);
absl::optional<CelValue> convertHeaderEntry(const ::Envoy::Http::HeaderEntry* header);
absl::optional<CelValue>
convertHeaderEntry(Protobuf::Arena& arena,
Http::HeaderUtility::GetAllOfHeaderAsStringResult&& result);
::Envoy::Http::HeaderUtility::GetAllOfHeaderAsStringResult&& result);

template <class T> class HeadersWrapper : public google::api::expr::runtime::CelMap {
public:
Expand All @@ -112,12 +112,12 @@ template <class T> class HeadersWrapper : public google::api::expr::runtime::Cel
return {};
}
auto str = std::string(key.StringOrDie().value());
if (!Http::validHeaderString(str)) {
if (!::Envoy::Http::validHeaderString(str)) {
// Reject key if it is an invalid header string
return {};
}
return convertHeaderEntry(
arena_, Http::HeaderUtility::getAllOfHeaderAsString(*value_, Http::LowerCaseString(str)));
return convertHeaderEntry(arena_, ::Envoy::Http::HeaderUtility::getAllOfHeaderAsString(
*value_, ::Envoy::Http::LowerCaseString(str)));
}
int size() const override { return ListKeys().value()->size(); }
bool empty() const override { return value_ == nullptr ? true : value_->empty(); }
Expand All @@ -126,10 +126,11 @@ template <class T> class HeadersWrapper : public google::api::expr::runtime::Cel
return &WrapperFields::get().Empty;
}
absl::flat_hash_set<absl::string_view> keys;
value_->iterate([&keys](const Http::HeaderEntry& header) -> Http::HeaderMap::Iterate {
keys.insert(header.key().getStringView());
return Http::HeaderMap::Iterate::Continue;
});
value_->iterate(
[&keys](const ::Envoy::Http::HeaderEntry& header) -> ::Envoy::Http::HeaderMap::Iterate {
keys.insert(header.key().getStringView());
return ::Envoy::Http::HeaderMap::Iterate::Continue;
});
std::vector<CelValue> values;
values.reserve(keys.size());
for (const auto& key : keys) {
Expand Down Expand Up @@ -163,26 +164,27 @@ class BaseWrapper : public google::api::expr::runtime::CelMap {

class RequestWrapper : public BaseWrapper {
public:
RequestWrapper(Protobuf::Arena& arena, const Http::RequestHeaderMap* headers,
RequestWrapper(Protobuf::Arena& arena, const ::Envoy::Http::RequestHeaderMap* headers,
const StreamInfo::StreamInfo& info)
: BaseWrapper(arena), headers_(arena, headers), info_(info) {}
absl::optional<CelValue> operator[](CelValue key) const override;

private:
const HeadersWrapper<Http::RequestHeaderMap> headers_;
const HeadersWrapper<::Envoy::Http::RequestHeaderMap> headers_;
const StreamInfo::StreamInfo& info_;
};

class ResponseWrapper : public BaseWrapper {
public:
ResponseWrapper(Protobuf::Arena& arena, const Http::ResponseHeaderMap* headers,
const Http::ResponseTrailerMap* trailers, const StreamInfo::StreamInfo& info)
ResponseWrapper(Protobuf::Arena& arena, const ::Envoy::Http::ResponseHeaderMap* headers,
const ::Envoy::Http::ResponseTrailerMap* trailers,
const StreamInfo::StreamInfo& info)
: BaseWrapper(arena), headers_(arena, headers), trailers_(arena, trailers), info_(info) {}
absl::optional<CelValue> operator[](CelValue key) const override;

private:
const HeadersWrapper<Http::ResponseHeaderMap> headers_;
const HeadersWrapper<Http::ResponseTrailerMap> trailers_;
const HeadersWrapper<::Envoy::Http::ResponseHeaderMap> headers_;
const HeadersWrapper<::Envoy::Http::ResponseTrailerMap> trailers_;
const StreamInfo::StreamInfo& info_;
};

Expand Down
26 changes: 13 additions & 13 deletions source/extensions/filters/common/expr/evaluator.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ using ExpressionPtr = std::unique_ptr<Expression>;
class StreamActivation : public google::api::expr::runtime::BaseActivation {
public:
StreamActivation(const StreamInfo::StreamInfo& info,
const Http::RequestHeaderMap* request_headers,
const Http::ResponseHeaderMap* response_headers,
const Http::ResponseTrailerMap* response_trailers)
const ::Envoy::Http::RequestHeaderMap* request_headers,
const ::Envoy::Http::ResponseHeaderMap* response_headers,
const ::Envoy::Http::ResponseTrailerMap* response_trailers)
: activation_info_(&info), activation_request_headers_(request_headers),
activation_response_headers_(response_headers),
activation_response_trailers_(response_trailers) {}
Expand All @@ -45,17 +45,17 @@ class StreamActivation : public google::api::expr::runtime::BaseActivation {
protected:
void resetActivation() const;
mutable const StreamInfo::StreamInfo* activation_info_{nullptr};
mutable const Http::RequestHeaderMap* activation_request_headers_{nullptr};
mutable const Http::ResponseHeaderMap* activation_response_headers_{nullptr};
mutable const Http::ResponseTrailerMap* activation_response_trailers_{nullptr};
mutable const ::Envoy::Http::RequestHeaderMap* activation_request_headers_{nullptr};
mutable const ::Envoy::Http::ResponseHeaderMap* activation_response_headers_{nullptr};
mutable const ::Envoy::Http::ResponseTrailerMap* activation_response_trailers_{nullptr};
};

// Creates an activation providing the common context attributes.
// The activation lazily creates wrappers during an evaluation using the evaluation arena.
ActivationPtr createActivation(const StreamInfo::StreamInfo& info,
const Http::RequestHeaderMap* request_headers,
const Http::ResponseHeaderMap* response_headers,
const Http::ResponseTrailerMap* response_trailers);
const ::Envoy::Http::RequestHeaderMap* request_headers,
const ::Envoy::Http::ResponseHeaderMap* response_headers,
const ::Envoy::Http::ResponseTrailerMap* response_trailers);

// Creates an expression builder. The optional arena is used to enable constant folding
// for intermediate evaluation results.
Expand All @@ -70,14 +70,14 @@ ExpressionPtr createExpression(Builder& builder, const google::api::expr::v1alph
// results and potentially the final value.
absl::optional<CelValue> evaluate(const Expression& expr, Protobuf::Arena& arena,
const StreamInfo::StreamInfo& info,
const Http::RequestHeaderMap* request_headers,
const Http::ResponseHeaderMap* response_headers,
const Http::ResponseTrailerMap* response_trailers);
const ::Envoy::Http::RequestHeaderMap* request_headers,
const ::Envoy::Http::ResponseHeaderMap* response_headers,
const ::Envoy::Http::ResponseTrailerMap* response_trailers);

// Evaluates an expression and returns true if the expression evaluates to "true".
// Returns false if the expression fails to evaluate.
bool matches(const Expression& expr, const StreamInfo::StreamInfo& info,
const Http::RequestHeaderMap& headers);
const ::Envoy::Http::RequestHeaderMap& headers);

// Returns a string for a CelValue.
std::string print(CelValue value);
Expand Down
25 changes: 25 additions & 0 deletions source/extensions/matching/http/cel_input/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
load(
"//bazel:envoy_build_system.bzl",
"envoy_cc_extension",
"envoy_extension_package",
)

licenses(["notice"]) # Apache 2

envoy_extension_package()

envoy_cc_extension(
name = "cel_input_lib",
srcs = ["cel_input.cc"],
hdrs = ["cel_input.h"],
extra_visibility = [
],
deps = [
"//envoy/http:filter_interface",
"//envoy/http:header_map_interface",
"//source/common/http:header_utility_lib",
"//source/common/http:utility_lib",
"//source/extensions/filters/common/expr:evaluator_lib",
"@envoy_api//envoy/type/matcher/v3:pkg_cc_proto",
],
)
18 changes: 18 additions & 0 deletions source/extensions/matching/http/cel_input/cel_input.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#include "source/extensions/matching/http/cel_input/cel_input.h"

#include "envoy/registry/registry.h"

namespace Envoy {
namespace Extensions {
namespace Matching {
namespace Http {
namespace CelInput {

REGISTER_FACTORY(HttpCelDataInputFactory,
Matcher::DataInputFactory<::Envoy::Http::HttpMatchingData>);

} // namespace CelInput
} // namespace Http
} // namespace Matching
} // namespace Extensions
} // namespace Envoy
82 changes: 82 additions & 0 deletions source/extensions/matching/http/cel_input/cel_input.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
#pragma once

#include "envoy/http/filter.h"
#include "envoy/matcher/matcher.h"
#include "envoy/server/factory_context.h"
#include "envoy/type/matcher/v3/http_inputs.pb.h"
#include "envoy/type/matcher/v3/http_inputs.pb.validate.h"

#include "source/common/http/header_utility.h"
#include "source/common/http/utility.h"
#include "source/extensions/filters/common/expr/evaluator.h"

#include "xds/type/matcher/v3/http_inputs.pb.h"

namespace Envoy {
namespace Extensions {
namespace Matching {
namespace Http {
namespace CelInput {

using ::Envoy::Http::RequestHeaderMapOptConstRef;
using ::Envoy::Http::ResponseHeaderMapOptConstRef;
using ::Envoy::Http::ResponseTrailerMapOptConstRef;

using BaseActivationPtr = std::unique_ptr<google::api::expr::runtime::BaseActivation>;

// CEL matcher specific matching data
class CelMatchData : public ::Envoy::Matcher::CustomMatchData {
public:
explicit CelMatchData(BaseActivationPtr activation) : activation_(std::move(activation)) {}
BaseActivationPtr activation_;
};

class HttpCelDataInput : public Matcher::DataInput<Envoy::Http::HttpMatchingData> {
public:
HttpCelDataInput() = default;
Matcher::DataInputGetResult get(const Envoy::Http::HttpMatchingData& data) const override {
RequestHeaderMapOptConstRef maybe_request_headers = data.requestHeaders();
ResponseHeaderMapOptConstRef maybe_response_headers = data.responseHeaders();
ResponseTrailerMapOptConstRef maybe_response_trailers = data.responseTrailers();

// Returns NotAvailable state when all of three below are empty.
if (!maybe_request_headers && !maybe_response_headers && !maybe_response_trailers) {
return {Matcher::DataInputGetResult::DataAvailability::NotAvailable, absl::monostate()};
}

// CEL library supports mixed matching condition of request headers, response headers and
// response trailers.
std::unique_ptr<google::api::expr::runtime::BaseActivation> activation =
Extensions::Filters::Common::Expr::createActivation(
data.streamInfo(), maybe_request_headers.ptr(), maybe_response_headers.ptr(),
maybe_response_trailers.ptr());

return {Matcher::DataInputGetResult::DataAvailability::AllDataAvailable,
std::make_unique<CelMatchData>(std::move(activation))};
}

absl::string_view dataInputType() const override { return "cel_data_input"; }
};

class HttpCelDataInputFactory : public Matcher::DataInputFactory<Envoy::Http::HttpMatchingData> {
public:
HttpCelDataInputFactory() = default;
std::string name() const override { return "envoy.matching.inputs.cel_data_input"; }

Matcher::DataInputFactoryCb<Envoy::Http::HttpMatchingData>
createDataInputFactoryCb(const Protobuf::Message&, ProtobufMessage::ValidationVisitor&) override {
return [] { return std::make_unique<HttpCelDataInput>(); };
}

ProtobufTypes::MessagePtr createEmptyConfigProto() override {
return std::make_unique<xds::type::matcher::v3::HttpAttributesCelMatchInput>();
}
};

DECLARE_FACTORY(HttpCelDataInputFactory);

} // namespace CelInput
} // namespace Http
} // namespace Matching
} // namespace Extensions
} // namespace Envoy
Loading

0 comments on commit 702edcd

Please sign in to comment.