diff --git a/include/fluent-bit/flb_lua.h b/include/fluent-bit/flb_lua.h index b2787430284..fb4ad99b49d 100644 --- a/include/fluent-bit/flb_lua.h +++ b/include/fluent-bit/flb_lua.h @@ -33,7 +33,8 @@ enum flb_lua_l2c_type_enum { FLB_LUA_L2C_TYPE_INT, - FLB_LUA_L2C_TYPE_ARRAY + FLB_LUA_L2C_TYPE_ARRAY, + FLB_LUA_L2C_TYPE_MAP }; struct flb_lua_l2c_type { @@ -47,6 +48,27 @@ struct flb_lua_l2c_config { struct mk_list l2c_types; /* data types (lua -> C) */ }; + +/* + * Metatable for Lua table. + * https://www.lua.org/manual/5.1/manual.html#2.8 + */ +struct flb_lua_metadata { + int initialized; + int data_type; /* Map or Array */ +}; + +static inline int flb_lua_metadata_init(struct flb_lua_metadata *meta) +{ + if (meta == NULL) { + return -1; + } + meta->initialized = FLB_TRUE; + meta->data_type = -1; + + return 0; +} + /* convert from negative index to positive index */ static inline int flb_lua_absindex(lua_State *l , int index) { diff --git a/src/flb_lua.c b/src/flb_lua.c index a374752326c..f872b512aa4 100644 --- a/src/flb_lua.c +++ b/src/flb_lua.c @@ -53,12 +53,35 @@ int flb_lua_is_valid_func(lua_State *lua, flb_sds_t func) return ret; } +static int flb_lua_setmetatable(lua_State *l, struct flb_lua_metadata *meta, int index) +{ + int abs_index; + + if (meta->initialized != FLB_TRUE) { + return -1; + } + abs_index = flb_lua_absindex(l, index); + + lua_createtable(l, 0, 1); + + /* data type */ + lua_pushlstring(l, "type", 4); + lua_pushinteger(l, meta->data_type); + lua_settable(l, -3); /* point created table */ + + lua_setmetatable(l, abs_index); + + return 0; +} + int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader) { int ret = 0; mpack_tag_t tag; uint32_t length; uint32_t i; + int index; + struct flb_lua_metadata meta; tag = mpack_read_tag(reader); switch (mpack_tag_type(&tag)) { @@ -88,8 +111,12 @@ int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader) reader->data += length; break; case mpack_type_array: + flb_lua_metadata_init(&meta); + meta.data_type = FLB_LUA_L2C_TYPE_ARRAY; + length = mpack_tag_array_count(&tag); lua_createtable(l, length, 0); + index = lua_gettop(l); /* save index of created table */ for (i = 0; i < length; i++) { ret = flb_lua_pushmpack(l, reader); if (ret) { @@ -97,10 +124,16 @@ int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader) } lua_rawseti(l, -2, i+1); } + flb_lua_setmetatable(l, &meta, index); + break; case mpack_type_map: + flb_lua_metadata_init(&meta); + meta.data_type = FLB_LUA_L2C_TYPE_MAP; + length = mpack_tag_map_count(&tag); lua_createtable(l, length, 0); + index = lua_gettop(l); /* save index of created table */ for (i = 0; i < length; i++) { ret = flb_lua_pushmpack(l, reader); if (ret) { @@ -112,6 +145,8 @@ int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader) } lua_settable(l, -3); } + flb_lua_setmetatable(l, &meta, index); + break; default: return -1; @@ -123,6 +158,8 @@ void flb_lua_pushmsgpack(lua_State *l, msgpack_object *o) { int i; int size; + int index; + struct flb_lua_metadata meta; lua_checkstack(l, 3); @@ -161,28 +198,38 @@ void flb_lua_pushmsgpack(lua_State *l, msgpack_object *o) break; case MSGPACK_OBJECT_ARRAY: + flb_lua_metadata_init(&meta); + meta.data_type = FLB_LUA_L2C_TYPE_ARRAY; + size = o->via.array.size; lua_createtable(l, size, 0); + index = lua_gettop(l); /* save index of created table */ if (size != 0) { msgpack_object *p = o->via.array.ptr; for (i = 0; i < size; i++) { flb_lua_pushmsgpack(l, p+i); - lua_rawseti (l, -2, i+1); + lua_rawseti (l, index, i+1); } } + flb_lua_setmetatable(l, &meta, index); break; case MSGPACK_OBJECT_MAP: + flb_lua_metadata_init(&meta); + meta.data_type = FLB_LUA_L2C_TYPE_MAP; + size = o->via.map.size; lua_createtable(l, 0, size); + index = lua_gettop(l); /* save index of created table */ if (size != 0) { msgpack_object_kv *p = o->via.map.ptr; for (i = 0; i < size; i++) { flb_lua_pushmsgpack(l, &(p+i)->key); flb_lua_pushmsgpack(l, &(p+i)->val); - lua_settable(l, -3); + lua_settable(l, index); } } + flb_lua_setmetatable(l, &meta, index); break; } } @@ -277,10 +324,10 @@ int flb_lua_arraylength(lua_State *l, int index) return max; } -static void lua_toarray(lua_State *l, - msgpack_packer *pck, - int index, - struct flb_lua_l2c_config *l2cc) +static void lua_toarray_msgpack(lua_State *l, + msgpack_packer *pck, + int index, + struct flb_lua_l2c_config *l2cc) { int len; int i; @@ -319,7 +366,6 @@ static void lua_toarray_mpack(lua_State *l, static void try_to_convert_data_type(lua_State *l, msgpack_packer *pck, - int index, struct flb_lua_l2c_config *l2cc) { size_t len; @@ -351,7 +397,7 @@ static void try_to_convert_data_type(lua_State *l, l2c = mk_list_entry(head, struct flb_lua_l2c_type, _head); if (!strncmp(l2c->key, tmp, len) && l2c->type == FLB_LUA_L2C_TYPE_ARRAY) { flb_lua_tomsgpack(l, pck, -1, l2cc); - lua_toarray(l, pck, 0, l2cc); + lua_toarray_msgpack(l, pck, 0, l2cc); return; } } @@ -364,7 +410,6 @@ static void try_to_convert_data_type(lua_State *l, static void try_to_convert_data_type_mpack(lua_State *l, mpack_writer_t *writer, - int index, struct flb_lua_l2c_config *l2cc) { size_t len; @@ -407,6 +452,89 @@ static void try_to_convert_data_type_mpack(lua_State *l, flb_lua_tompack(l, writer, 0, l2cc); } +static int flb_lua_getmetatable(lua_State *l, int index, struct flb_lua_metadata *meta) +{ + int lua_ret; + int abs_index; + const char *str; + size_t len; + + if (meta->initialized != FLB_TRUE) { + return -1; + } + + lua_ret = lua_getmetatable(l, index); + if (lua_ret == 0) { + /* no metadata */ + return -1; + } + else if (lua_type(l, -1) != LUA_TTABLE) { + /* invalid metatable? */ + lua_pop(l, 1); + return -1; + } + + lua_pushnil(l); + abs_index = flb_lua_absindex(l, -2); + while (lua_next(l, abs_index) != 0) { + if (lua_type(l, -2) != LUA_TSTRING) { + /* key is not a string */ + flb_debug("key is not a string"); + lua_pop(l, 1); + continue; + } + + str = lua_tolstring(l, -2, &len); /* key */ + + if (len == 4 && strncmp(str, "type", 4) == 0) { + /* data_type */ + if (lua_type(l, -1) != LUA_TNUMBER) { + /* value is not data type */ + flb_debug("type is not num. type=%s", lua_typename(l, lua_type(l, -1))); + lua_pop(l, 1); + continue; + } + meta->data_type = (int)lua_tointeger(l, -1); + } + lua_pop(l, 1); + } + lua_pop(l, 1); /* pop metatable */ + + return 0; +} + +static void lua_tomap_mpack(lua_State *l, + mpack_writer_t *writer, + int index, + struct flb_lua_l2c_config *l2cc) +{ + int len; + + len = 0; + lua_pushnil(l); + while (lua_next(l, -2) != 0) { + lua_pop(l, 1); + len++; + } + mpack_write_tag(writer, mpack_tag_map(len)); + + lua_pushnil(l); + + if (l2cc->l2c_types_num > 0) { + /* type conversion */ + while (lua_next(l, -2) != 0) { + try_to_convert_data_type_mpack(l, writer, l2cc); + lua_pop(l, 1); + } + } else { + while (lua_next(l, -2) != 0) { + flb_lua_tompack(l, writer, -1, l2cc); + flb_lua_tompack(l, writer, 0, l2cc); + lua_pop(l, 1); + } + } +} + void flb_lua_tompack(lua_State *l, mpack_writer_t *writer, int index, @@ -414,6 +542,8 @@ void flb_lua_tompack(lua_State *l, { int len; int i; + int use_metatable = FLB_FALSE; + struct flb_lua_metadata meta; switch (lua_type(l, -1 + index)) { case LUA_TSTRING: @@ -445,6 +575,23 @@ void flb_lua_tompack(lua_State *l, mpack_write_false(writer); break; case LUA_TTABLE: + flb_lua_metadata_init(&meta); + if (flb_lua_getmetatable(l, -1 + index, &meta) == 0 && + meta.data_type >= 0) { + use_metatable = FLB_TRUE; + } + if (use_metatable) { + if (meta.data_type == FLB_LUA_L2C_TYPE_ARRAY) { + /* array */ + lua_toarray_mpack(l, writer, 0, l2cc); + } + else { + /* map */ + lua_tomap_mpack(l, writer, -1 + index, l2cc); + } + break; + } + len = flb_lua_arraylength(l, -1 + index); if (len > 0) { mpack_write_tag(writer, mpack_tag_array(len)); @@ -453,31 +600,9 @@ void flb_lua_tompack(lua_State *l, flb_lua_tompack(l, writer, 0, l2cc); lua_pop(l, 1); } - } else - { - len = 0; - lua_pushnil(l); - while (lua_next(l, -2) != 0) { - lua_pop(l, 1); - len++; - } - mpack_write_tag(writer, mpack_tag_map(len)); - - lua_pushnil(l); - - if (l2cc->l2c_types_num > 0) { - /* type conversion */ - while (lua_next(l, -2) != 0) { - try_to_convert_data_type_mpack(l, writer, index, l2cc); - lua_pop(l, 1); - } - } else { - while (lua_next(l, -2) != 0) { - flb_lua_tompack(l, writer, -1, l2cc); - flb_lua_tompack(l, writer, 0, l2cc); - lua_pop(l, 1); - } - } + } + else { + lua_tomap_mpack(l, writer, -1 + index, l2cc); } break; case LUA_TNIL: @@ -497,6 +622,41 @@ void flb_lua_tompack(lua_State *l, } } +static inline void lua_tomap_msgpack(lua_State *l, + msgpack_packer *pck, + int index, + struct flb_lua_l2c_config *l2cc) +{ + int len; + int abs_index; + + abs_index = flb_lua_absindex(l, index); + + len = 0; + lua_pushnil(l); + while (lua_next(l, abs_index) != 0) { + lua_pop(l, 1); + len++; + } + msgpack_pack_map(pck, len); + + lua_pushnil(l); + + if (l2cc->l2c_types_num > 0) { + /* type conversion */ + while (lua_next(l, abs_index) != 0) { + try_to_convert_data_type(l, pck, l2cc); + lua_pop(l, 1); + } + } else { + while (lua_next(l, abs_index) != 0) { + flb_lua_tomsgpack(l, pck, -1, l2cc); + flb_lua_tomsgpack(l, pck, 0, l2cc); + lua_pop(l, 1); + } + } +} + void flb_lua_tomsgpack(lua_State *l, msgpack_packer *pck, int index, @@ -504,6 +664,8 @@ void flb_lua_tomsgpack(lua_State *l, { int len; int i; + int use_metatable = FLB_FALSE; + struct flb_lua_metadata meta; switch (lua_type(l, -1 + index)) { case LUA_TSTRING: @@ -536,6 +698,23 @@ void flb_lua_tomsgpack(lua_State *l, msgpack_pack_false(pck); break; case LUA_TTABLE: + flb_lua_metadata_init(&meta); + if (flb_lua_getmetatable(l, -1 + index, &meta) == 0 && + meta.data_type >= 0) { + use_metatable = FLB_TRUE; + } + if (use_metatable) { + if (meta.data_type == FLB_LUA_L2C_TYPE_ARRAY) { + /* array */ + lua_toarray_msgpack(l, pck, 0, l2cc); + } + else { + /* map */ + lua_tomap_msgpack(l, pck, -1 + index, l2cc); + } + break; + } + len = flb_lua_arraylength(l, -1 + index); if (len > 0) { msgpack_pack_array(pck, len); @@ -544,31 +723,9 @@ void flb_lua_tomsgpack(lua_State *l, flb_lua_tomsgpack(l, pck, 0, l2cc); lua_pop(l, 1); } - } else - { - len = 0; - lua_pushnil(l); - while (lua_next(l, -2) != 0) { - lua_pop(l, 1); - len++; - } - msgpack_pack_map(pck, len); - - lua_pushnil(l); - - if (l2cc->l2c_types_num > 0) { - /* type conversion */ - while (lua_next(l, -2) != 0) { - try_to_convert_data_type(l, pck, index, l2cc); - lua_pop(l, 1); - } - } else { - while (lua_next(l, -2) != 0) { - flb_lua_tomsgpack(l, pck, -1, l2cc); - flb_lua_tomsgpack(l, pck, 0, l2cc); - lua_pop(l, 1); - } - } + } + else { + lua_tomap_msgpack(l, pck, -1 + index, l2cc); } break; case LUA_TNIL: diff --git a/tests/runtime/filter_lua.c b/tests/runtime/filter_lua.c index 9f694377eae..afdf8200af1 100644 --- a/tests/runtime/filter_lua.c +++ b/tests/runtime/filter_lua.c @@ -753,6 +753,151 @@ void flb_test_split_record(void) flb_sds_destroy(outbuf); } +void flb_test_empty_array(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + flb_sds_t outbuf = flb_sds_create(""); + char *input = "[0, {\"key\":[]}]"; + struct flb_lib_out_cb cb_data; + + const char *expected = + "[5.000000,{\"key\":[]}]"; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " return 1, 5, record\n" + "end\n"; + + clear_output_num(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_cat; + cb_data.data = &outbuf; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + flb_lib_push(ctx, in_ffd, input, strlen(input)); + wait_with_timeout(2000, &output); + if (!TEST_CHECK(!strcmp(outbuf, expected))) { + TEST_MSG("expected:\n%s\ngot:\n%s\n", expected, outbuf); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); + flb_sds_destroy(outbuf); +} + +void flb_test_invalid_metatable(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int unused = 0; + int filter_ffd; + char *output = NULL; + char *input = "[0, {\"key\":\"val\"}]"; + struct flb_lib_out_cb cb_data; + + char *script_body = "" + "function lua_main(tag, timestamp, record)\n" + " meta = getmetatable(record)\n" + " meta[10] = \"hoge\"\n" + " return 1, timestamp, record\n" + "end\n"; + + clear_output_num(); + + /* Create context, flush every second (some checks omitted here) */ + ctx = flb_create(); + flb_service_set(ctx, "flush", FLUSH_INTERVAL, "grace", "1", NULL); + + /* Prepare output callback context*/ + cb_data.cb = cb_count_msgpack_events; + cb_data.data = &unused; + + ret = create_script(script_body, strlen(script_body)); + TEST_CHECK(ret == 0); + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "lua", NULL); + TEST_CHECK(filter_ffd >= 0); + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "call", "lua_main", + "script", TMP_LUA_PATH, + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Lib output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *)&cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret==0); + + ret = flb_lib_push(ctx, in_ffd, input, strlen(input)); + if (!TEST_CHECK(ret != -1)) { + TEST_MSG("flb_lib_push error"); + } + flb_time_msleep(1500); /* waiting flush */ + + ret = get_output_num(); + if (!TEST_CHECK(ret > 0)) { + TEST_MSG("error. no output"); + } + + /* clean up */ + flb_lib_free(output); + delete_script(); + + flb_stop(ctx); + flb_destroy(ctx); +} + TEST_LIST = { {"hello_world", flb_test_helloworld}, {"append_tag", flb_test_append_tag}, @@ -762,5 +907,7 @@ TEST_LIST = { {"array_contains_null", flb_test_array_contains_null}, {"drop_all_records", flb_test_drop_all_records}, {"split_record", flb_test_split_record}, + {"empty_array", flb_test_empty_array}, + {"invalid_metatable", flb_test_invalid_metatable}, {NULL, NULL} };