Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

grep: add common grep APIs #7336

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

grep: add common grep APIs #7336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
30913af
grep: add common grep APIs
nokute78 May 6, 2023
f8e3cde
filter_grep: use common grep APIs
nokute78 May 6, 2023
d8b2619
filter_log_to_metrics: use common grep APIs
nokute78 May 6, 2023
3c81f58
grep: respond to review comment
nokute78 May 11, 2023
011a86e
grep: release rule if flb_sds_create_size failed
nokute78 May 11, 2023
1e303c3
grep: use flb_sds_snprintf to construct rule
nokute78 May 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions include/fluent-bit/flb_grep.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/* Fluent Bit
* ==========
* Copyright (C) 2015-2023 The Fluent Bit Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef FLB_GREP_H
#define FLB_GREP_H

#include <fluent-bit/flb_sds.h>
#include <fluent-bit/flb_regex.h>
#include <fluent-bit/flb_record_accessor.h>

/* rule types */
enum flb_grep_rule_type {
FLB_GREP_NO_RULE,
FLB_GREP_REGEX,
FLB_GREP_EXCLUDE
};

/* actions */
enum flb_grep_action {
FLB_GREP_RET_KEEP,
FLB_GREP_RET_EXCLUDE
};

enum flb_grep_logical_op {
FLB_GREP_LOGICAL_OP_LEGACY,
FLB_GREP_LOGICAL_OP_OR,
FLB_GREP_LOGICAL_OP_AND
};

struct flb_grep_rule {
int type;
flb_sds_t field;
char *regex_pattern;
struct flb_regex *regex;
struct flb_record_accessor *ra;
struct mk_list _head;
};


struct flb_grep {
enum flb_grep_rule_type first_rule;
enum flb_grep_logical_op op;
struct mk_list rules; /* flb_grep_rule list */
};


int flb_grep_filter(msgpack_object map, struct flb_grep *grep_ctx);
int flb_grep_set_rule_str(struct flb_grep *ctx, enum flb_grep_rule_type type, char *rule_str);
struct flb_grep *flb_grep_create(enum flb_grep_logical_op op);
int flb_grep_destroy(struct flb_grep *grep_ctx);

#endif /* FLB_GREP_H */
209 changes: 22 additions & 187 deletions plugins/filter_grep/grep.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,177 +29,48 @@
#include <fluent-bit/flb_utils.h>
#include <fluent-bit/flb_pack.h>
#include <fluent-bit/flb_regex.h>
#include <fluent-bit/flb_grep.h>
#include <fluent-bit/flb_record_accessor.h>
#include <fluent-bit/flb_log_event_decoder.h>
#include <fluent-bit/flb_log_event_encoder.h>
#include <msgpack.h>

#include "grep.h"

static void delete_rules(struct grep_ctx *ctx)
{
struct mk_list *tmp;
struct mk_list *head;
struct grep_rule *rule;

mk_list_foreach_safe(head, tmp, &ctx->rules) {
rule = mk_list_entry(head, struct grep_rule, _head);
flb_sds_destroy(rule->field);
flb_free(rule->regex_pattern);
flb_ra_destroy(rule->ra);
flb_regex_destroy(rule->regex);
mk_list_del(&rule->_head);
flb_free(rule);
}
}

static int set_rules(struct grep_ctx *ctx, struct flb_filter_instance *f_ins)
{
int first_rule = GREP_NO_RULE;
flb_sds_t tmp;
int ret;
struct mk_list *head;
struct mk_list *split;
struct flb_split_entry *sentry;
struct flb_kv *kv;
struct grep_rule *rule;

/* Iterate all filter properties */
mk_list_foreach(head, &f_ins->properties) {
kv = mk_list_entry(head, struct flb_kv, _head);

/* Create a new rule */
rule = flb_malloc(sizeof(struct grep_rule));
if (!rule) {
flb_errno();
return -1;
}

/* Get the type */
if (strcasecmp(kv->key, "regex") == 0) {
rule->type = GREP_REGEX;
ret = flb_grep_set_rule_str(ctx->grep, FLB_GREP_REGEX, kv->val);
}
else if (strcasecmp(kv->key, "exclude") == 0) {
rule->type = GREP_EXCLUDE;
}
else {
/* Other property. Skip */
flb_free(rule);
continue;
}

if (ctx->logical_op != GREP_LOGICAL_OP_LEGACY && first_rule != GREP_NO_RULE) {
/* 'AND'/'OR' case */
if (first_rule != rule->type) {
flb_plg_error(ctx->ins, "Both 'regex' and 'exclude' are set.");
delete_rules(ctx);
flb_free(rule);
return -1;
}
}
first_rule = rule->type;

/* As a value we expect a pair of field name and a regular expression */
split = flb_utils_split(kv->val, ' ', 1);
if (mk_list_size(split) != 2) {
flb_plg_error(ctx->ins,
"invalid regex, expected field and regular expression");
delete_rules(ctx);
flb_free(rule);
flb_utils_split_free(split);
return -1;
}

/* Get first value (field) */
sentry = mk_list_entry_first(split, struct flb_split_entry, _head);
if (*sentry->value == '$') {
rule->field = flb_sds_create_len(sentry->value, sentry->len);
ret = flb_grep_set_rule_str(ctx->grep, FLB_GREP_EXCLUDE, kv->val);
}
else {
rule->field = flb_sds_create_size(sentry->len + 2);
tmp = flb_sds_cat(rule->field, "$", 1);
rule->field = tmp;

tmp = flb_sds_cat(rule->field, sentry->value, sentry->len);
rule->field = tmp;
if (ret < 0) {
return ret;
}

/* Get remaining content (regular expression) */
sentry = mk_list_entry_last(split, struct flb_split_entry, _head);
rule->regex_pattern = flb_strndup(sentry->value, sentry->len);
if (rule->regex_pattern == NULL) {
flb_errno();
delete_rules(ctx);
flb_free(rule);
flb_utils_split_free(split);
return -1;
}

/* Release split */
flb_utils_split_free(split);

/* Create a record accessor context for this rule */
rule->ra = flb_ra_create(rule->field, FLB_FALSE);
if (!rule->ra) {
flb_plg_error(ctx->ins, "invalid record accessor? '%s'", rule->field);
delete_rules(ctx);
flb_free(rule);
return -1;
}

/* Convert string to regex pattern */
rule->regex = flb_regex_create(rule->regex_pattern);
if (!rule->regex) {
flb_plg_error(ctx->ins, "could not compile regex pattern '%s'",
rule->regex_pattern);
delete_rules(ctx);
flb_free(rule);
return -1;
}

/* Link to parent list */
mk_list_add(&rule->_head, &ctx->rules);
}

return 0;
}

/* Given a msgpack record, do some filter action based on the defined rules */
static inline int grep_filter_data(msgpack_object map, struct grep_ctx *ctx)
{
ssize_t ret;
struct mk_list *head;
struct grep_rule *rule;

/* For each rule, validate against map fields */
mk_list_foreach(head, &ctx->rules) {
rule = mk_list_entry(head, struct grep_rule, _head);

ret = flb_ra_regex_match(rule->ra, map, rule->regex, NULL);
if (ret <= 0) { /* no match */
if (rule->type == GREP_REGEX) {
return GREP_RET_EXCLUDE;
}
}
else {
if (rule->type == GREP_EXCLUDE) {
return GREP_RET_EXCLUDE;
}
else {
return GREP_RET_KEEP;
}
}
}

return GREP_RET_KEEP;
}

static int cb_grep_init(struct flb_filter_instance *f_ins,
struct flb_config *config,
void *data)
{
int ret;
size_t len;
const char* val;
enum flb_grep_logical_op op = FLB_GREP_LOGICAL_OP_LEGACY;
struct grep_ctx *ctx;

/* Create context */
Expand All @@ -214,30 +85,36 @@ static int cb_grep_init(struct flb_filter_instance *f_ins,
flb_free(ctx);
return -1;
}
mk_list_init(&ctx->rules);
ctx->ins = f_ins;

ctx->logical_op = GREP_LOGICAL_OP_LEGACY;
val = flb_filter_get_property("logical_op", f_ins);
if (val != NULL) {
len = strlen(val);
if (len == 3 && strncasecmp("AND", val, len) == 0) {
flb_plg_info(ctx->ins, "AND mode");
ctx->logical_op = GREP_LOGICAL_OP_AND;
op = FLB_GREP_LOGICAL_OP_AND;
}
else if (len == 2 && strncasecmp("OR", val, len) == 0) {
flb_plg_info(ctx->ins, "OR mode");
ctx->logical_op = GREP_LOGICAL_OP_OR;
op = FLB_GREP_LOGICAL_OP_OR;
}
else if (len == 6 && strncasecmp("legacy", val, len) == 0) {
flb_plg_info(ctx->ins, "legacy mode");
ctx->logical_op = GREP_LOGICAL_OP_LEGACY;
op = FLB_GREP_LOGICAL_OP_LEGACY;
}
}

ctx->grep =flb_grep_create(op);
if (ctx->grep == NULL) {
flb_plg_error(f_ins, "flb_grep_create failed");
flb_free(ctx);
return -1;
}

/* Load rules */
ret = set_rules(ctx, f_ins);
if (ret == -1) {
flb_grep_destroy(ctx->grep);
flb_free(ctx);
return -1;
}
Expand All @@ -247,42 +124,6 @@ static int cb_grep_init(struct flb_filter_instance *f_ins,
return 0;
}

static inline int grep_filter_data_and_or(msgpack_object map, struct grep_ctx *ctx)
{
ssize_t ra_ret;
int found = FLB_FALSE;
struct mk_list *head;
struct grep_rule *rule;

/* For each rule, validate against map fields */
mk_list_foreach(head, &ctx->rules) {
found = FLB_FALSE;
rule = mk_list_entry(head, struct grep_rule, _head);

ra_ret = flb_ra_regex_match(rule->ra, map, rule->regex, NULL);
if (ra_ret > 0) {
found = FLB_TRUE;
}

if (ctx->logical_op == GREP_LOGICAL_OP_OR && found == FLB_TRUE) {
/* OR case: One rule is matched. */
goto grep_filter_data_and_or_end;
}
else if (ctx->logical_op == GREP_LOGICAL_OP_AND && found == FLB_FALSE) {
/* AND case: One rule is not matched */
goto grep_filter_data_and_or_end;
}
}

grep_filter_data_and_or_end:
if (rule->type == GREP_REGEX) {
return found ? GREP_RET_KEEP : GREP_RET_EXCLUDE;
}

/* rule is exclude */
return found ? GREP_RET_EXCLUDE : GREP_RET_KEEP;
}

static int cb_grep_filter(const void *data, size_t bytes,
const char *tag, int tag_len,
void **out_buf, size_t *out_size,
Expand Down Expand Up @@ -335,22 +176,17 @@ static int cb_grep_filter(const void *data, size_t bytes,
/* get time and map */
map = *log_event.body;

if (ctx->logical_op == GREP_LOGICAL_OP_LEGACY) {
ret = grep_filter_data(map, ctx);
}
else {
ret = grep_filter_data_and_or(map, ctx);
}
ret = flb_grep_filter(map, ctx->grep);

if (ret == GREP_RET_KEEP) {
if (ret == FLB_GREP_RET_KEEP) {
ret = flb_log_event_encoder_emit_raw_record(
&log_encoder,
log_decoder.record_base,
log_decoder.record_length);

new_size++;
}
else if (ret == GREP_RET_EXCLUDE) {
else if (ret == FLB_GREP_RET_EXCLUDE) {
/* Do nothing */
}
}
Expand Down Expand Up @@ -398,8 +234,7 @@ static int cb_grep_exit(void *data, struct flb_config *config)
if (!ctx) {
return 0;
}

delete_rules(ctx);
flb_grep_destroy(ctx->grep);
flb_free(ctx);
return 0;
}
Expand Down
Loading
Loading