Skip to content

Commit

Permalink
filter_grep: use common grep APIs
Browse files Browse the repository at this point in the history 
Signed-off-by: Takahiro Yamashita <[email protected]>
  • Loading branch information
@nokute78
nokute78 committed May 7, 2023
1 parent eb30ea9 commit f0deb4a
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 213 deletions.
209 changes: 22 additions & 187 deletions plugins/filter_grep/grep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,177 +29,48 @@
#include <fluent-bit/flb_utils.h>
#include <fluent-bit/flb_pack.h>
#include <fluent-bit/flb_regex.h>
#include <fluent-bit/flb_grep.h>
#include <fluent-bit/flb_record_accessor.h>
#include <fluent-bit/flb_log_event_decoder.h>
#include <fluent-bit/flb_log_event_encoder.h>
#include <msgpack.h>

#include "grep.h"

static void delete_rules(struct grep_ctx *ctx)
{
struct mk_list *tmp;
struct mk_list *head;
struct grep_rule *rule;

mk_list_foreach_safe(head, tmp, &ctx->rules) {
rule = mk_list_entry(head, struct grep_rule, _head);
flb_sds_destroy(rule->field);
flb_free(rule->regex_pattern);
flb_ra_destroy(rule->ra);
flb_regex_destroy(rule->regex);
mk_list_del(&rule->_head);
flb_free(rule);
}
}

static int set_rules(struct grep_ctx *ctx, struct flb_filter_instance *f_ins)
{
int first_rule = GREP_NO_RULE;
flb_sds_t tmp;
int ret;
struct mk_list *head;
struct mk_list *split;
struct flb_split_entry *sentry;
struct flb_kv *kv;
struct grep_rule *rule;

/* Iterate all filter properties */
mk_list_foreach(head, &f_ins->properties) {
kv = mk_list_entry(head, struct flb_kv, _head);

/* Create a new rule */
rule = flb_malloc(sizeof(struct grep_rule));
if (!rule) {
flb_errno();
return -1;
}

/* Get the type */
if (strcasecmp(kv->key, "regex") == 0) {
rule->type = GREP_REGEX;
ret = flb_grep_set_rule_str(ctx->grep, FLB_GREP_REGEX, kv->val);
}
else if (strcasecmp(kv->key, "exclude") == 0) {
rule->type = GREP_EXCLUDE;
}
else {
/* Other property. Skip */
flb_free(rule);
continue;
}

if (ctx->logical_op != GREP_LOGICAL_OP_LEGACY && first_rule != GREP_NO_RULE) {
/* 'AND'/'OR' case */
if (first_rule != rule->type) {
flb_plg_error(ctx->ins, "Both 'regex' and 'exclude' are set.");
delete_rules(ctx);
flb_free(rule);
return -1;
}
}
first_rule = rule->type;

/* As a value we expect a pair of field name and a regular expression */
split = flb_utils_split(kv->val, ' ', 1);
if (mk_list_size(split) != 2) {
flb_plg_error(ctx->ins,
"invalid regex, expected field and regular expression");
delete_rules(ctx);
flb_free(rule);
flb_utils_split_free(split);
return -1;
}

/* Get first value (field) */
sentry = mk_list_entry_first(split, struct flb_split_entry, _head);
if (*sentry->value == '$') {
rule->field = flb_sds_create_len(sentry->value, sentry->len);
ret = flb_grep_set_rule_str(ctx->grep, FLB_GREP_EXCLUDE, kv->val);
}
else {
rule->field = flb_sds_create_size(sentry->len + 2);
tmp = flb_sds_cat(rule->field, "$", 1);
rule->field = tmp;

tmp = flb_sds_cat(rule->field, sentry->value, sentry->len);
rule->field = tmp;
if (ret < 0) {
return ret;
}

/* Get remaining content (regular expression) */
sentry = mk_list_entry_last(split, struct flb_split_entry, _head);
rule->regex_pattern = flb_strndup(sentry->value, sentry->len);
if (rule->regex_pattern == NULL) {
flb_errno();
delete_rules(ctx);
flb_free(rule);
flb_utils_split_free(split);
return -1;
}

/* Release split */
flb_utils_split_free(split);

/* Create a record accessor context for this rule */
rule->ra = flb_ra_create(rule->field, FLB_FALSE);
if (!rule->ra) {
flb_plg_error(ctx->ins, "invalid record accessor? '%s'", rule->field);
delete_rules(ctx);
flb_free(rule);
return -1;
}

/* Convert string to regex pattern */
rule->regex = flb_regex_create(rule->regex_pattern);
if (!rule->regex) {
flb_plg_error(ctx->ins, "could not compile regex pattern '%s'",
rule->regex_pattern);
delete_rules(ctx);
flb_free(rule);
return -1;
}

/* Link to parent list */
mk_list_add(&rule->_head, &ctx->rules);
}

return 0;
}

/* Given a msgpack record, do some filter action based on the defined rules */
static inline int grep_filter_data(msgpack_object map, struct grep_ctx *ctx)
{
ssize_t ret;
struct mk_list *head;
struct grep_rule *rule;

/* For each rule, validate against map fields */
mk_list_foreach(head, &ctx->rules) {
rule = mk_list_entry(head, struct grep_rule, _head);

ret = flb_ra_regex_match(rule->ra, map, rule->regex, NULL);
if (ret <= 0) { /* no match */
if (rule->type == GREP_REGEX) {
return GREP_RET_EXCLUDE;
}
}
else {
if (rule->type == GREP_EXCLUDE) {
return GREP_RET_EXCLUDE;
}
else {
return GREP_RET_KEEP;
}
}
}

return GREP_RET_KEEP;
}

static int cb_grep_init(struct flb_filter_instance *f_ins,
struct flb_config *config,
void *data)
{
int ret;
size_t len;
const char* val;
enum flb_grep_logical_op op = FLB_GREP_LOGICAL_OP_LEGACY;
struct grep_ctx *ctx;

/* Create context */
Expand All @@ -214,30 +85,36 @@ static int cb_grep_init(struct flb_filter_instance *f_ins,
flb_free(ctx);
return -1;
}
mk_list_init(&ctx->rules);
ctx->ins = f_ins;

ctx->logical_op = GREP_LOGICAL_OP_LEGACY;
val = flb_filter_get_property("logical_op", f_ins);
if (val != NULL) {
len = strlen(val);
if (len == 3 && strncasecmp("AND", val, len) == 0) {
flb_plg_info(ctx->ins, "AND mode");
ctx->logical_op = GREP_LOGICAL_OP_AND;
op = FLB_GREP_LOGICAL_OP_AND;
}
else if (len == 2 && strncasecmp("OR", val, len) == 0) {
flb_plg_info(ctx->ins, "OR mode");
ctx->logical_op = GREP_LOGICAL_OP_OR;
op = FLB_GREP_LOGICAL_OP_OR;
}
else if (len == 6 && strncasecmp("legacy", val, len) == 0) {
flb_plg_info(ctx->ins, "legacy mode");
ctx->logical_op = GREP_LOGICAL_OP_LEGACY;
op = FLB_GREP_LOGICAL_OP_LEGACY;
}
}

ctx->grep =flb_grep_create(op);
if (ctx->grep == NULL) {
flb_plg_error(f_ins, "flb_grep_create failed");
flb_free(ctx);
return -1;
}

/* Load rules */
ret = set_rules(ctx, f_ins);
if (ret == -1) {
flb_grep_destroy(ctx->grep);
flb_free(ctx);
return -1;
}
Expand All @@ -247,42 +124,6 @@ static int cb_grep_init(struct flb_filter_instance *f_ins,
return 0;
}

static inline int grep_filter_data_and_or(msgpack_object map, struct grep_ctx *ctx)
{
ssize_t ra_ret;
int found = FLB_FALSE;
struct mk_list *head;
struct grep_rule *rule;

/* For each rule, validate against map fields */
mk_list_foreach(head, &ctx->rules) {
found = FLB_FALSE;
rule = mk_list_entry(head, struct grep_rule, _head);

ra_ret = flb_ra_regex_match(rule->ra, map, rule->regex, NULL);
if (ra_ret > 0) {
found = FLB_TRUE;
}

if (ctx->logical_op == GREP_LOGICAL_OP_OR && found == FLB_TRUE) {
/* OR case: One rule is matched. */
goto grep_filter_data_and_or_end;
}
else if (ctx->logical_op == GREP_LOGICAL_OP_AND && found == FLB_FALSE) {
/* AND case: One rule is not matched */
goto grep_filter_data_and_or_end;
}
}

grep_filter_data_and_or_end:
if (rule->type == GREP_REGEX) {
return found ? GREP_RET_KEEP : GREP_RET_EXCLUDE;
}

/* rule is exclude */
return found ? GREP_RET_EXCLUDE : GREP_RET_KEEP;
}

static int cb_grep_filter(const void *data, size_t bytes,
const char *tag, int tag_len,
void **out_buf, size_t *out_size,
Expand Down Expand Up @@ -335,22 +176,17 @@ static int cb_grep_filter(const void *data, size_t bytes,
/* get time and map */
map = *log_event.body;

if (ctx->logical_op == GREP_LOGICAL_OP_LEGACY) {
ret = grep_filter_data(map, ctx);
}
else {
ret = grep_filter_data_and_or(map, ctx);
}
ret = flb_grep_filter(map, ctx->grep);

if (ret == GREP_RET_KEEP) {
if (ret == FLB_GREP_RET_KEEP) {
ret = flb_log_event_encoder_emit_raw_record(
&log_encoder,
log_decoder.record_base,
log_decoder.record_length);

new_size++;
}
else if (ret == GREP_RET_EXCLUDE) {
else if (ret == FLB_GREP_RET_EXCLUDE) {
/* Do nothing */
}
}
Expand Down Expand Up @@ -398,8 +234,7 @@ static int cb_grep_exit(void *data, struct flb_config *config)
if (!ctx) {
return 0;
}

delete_rules(ctx);
flb_grep_destroy(ctx->grep);
flb_free(ctx);
return 0;
}
Expand Down
28 changes: 2 additions & 26 deletions plugins/filter_grep/grep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,35 +24,11 @@
#include <fluent-bit/flb_filter.h>
#include <fluent-bit/flb_sds.h>
#include <fluent-bit/flb_record_accessor.h>

/* rule types */
#define GREP_NO_RULE 0
#define GREP_REGEX 1
#define GREP_EXCLUDE 2

/* actions */
#define GREP_RET_KEEP 0
#define GREP_RET_EXCLUDE 1

enum _logical_op{
GREP_LOGICAL_OP_LEGACY,
GREP_LOGICAL_OP_OR,
GREP_LOGICAL_OP_AND
} logical_op;
#include <fluent-bit/flb_grep.h>

struct grep_ctx {
struct mk_list rules;
int logical_op;
struct flb_grep *grep;
struct flb_filter_instance *ins;
};

struct grep_rule {
int type;
flb_sds_t field;
char *regex_pattern;
struct flb_regex *regex;
struct flb_record_accessor *ra;
struct mk_list _head;
};

#endif

0 comments on commit f0deb4a

Please sign in to comment.