From 998905f6f86e14f7441a64257541033c12bd5ac8 Mon Sep 17 00:00:00 2001 From: Brian Nichols Date: Fri, 26 Jan 2024 11:06:09 -0500 Subject: [PATCH] CLIENT-2764 Support persistent list indexes. Add as_operations_list_create_all(). --- modules/common | 2 +- src/include/aerospike/as_cdt_order.h | 4 +- src/include/aerospike/as_list_operations.h | 23 +- src/main/aerospike/as_list_operations.c | 32 +- src/main/aerospike/as_map_operations.c | 6 +- src/test/aerospike_list/list_basics.c | 326 +++++++++++++- src/test/aerospike_map/map_basics.c | 491 ++++++++++++++++++++- src/test/lua/list_unordered.lua | 13 + 8 files changed, 873 insertions(+), 24 deletions(-) create mode 100644 src/test/lua/list_unordered.lua diff --git a/modules/common b/modules/common index 83add8e681..00092452bf 160000 --- a/modules/common +++ b/modules/common @@ -1 +1 @@ -Subproject commit 83add8e681b3e25ed467dc67488d68d4e197ee44 +Subproject commit 00092452bfa984d424fe0d579dbe38ce1c526883 diff --git a/src/include/aerospike/as_cdt_order.h b/src/include/aerospike/as_cdt_order.h index c1151fe477..0b09aab1ce 100644 --- a/src/include/aerospike/as_cdt_order.h +++ b/src/include/aerospike/as_cdt_order.h @@ -1,5 +1,5 @@ /* - * Copyright 2008-2020 Aerospike, Inc. + * Copyright 2008-2024 Aerospike, Inc. * * Portions may be licensed to Aerospike, Inc. under one or more contributor * license agreements. @@ -40,7 +40,7 @@ typedef enum as_list_order_e { /** * List is ordered. */ - AS_LIST_ORDERED = 1, + AS_LIST_ORDERED = 1 } as_list_order; /** diff --git a/src/include/aerospike/as_list_operations.h b/src/include/aerospike/as_list_operations.h index c322a0c89f..d9ba05bba5 100644 --- a/src/include/aerospike/as_list_operations.h +++ b/src/include/aerospike/as_list_operations.h @@ -1,5 +1,5 @@ /* - * Copyright 2008-2022 Aerospike, Inc. + * Copyright 2008-2024 Aerospike, Inc. * * Portions may be licensed to Aerospike, Inc. under one or more contributor * license agreements. @@ -302,6 +302,27 @@ as_operations_list_create( as_operations* ops, const char* name, as_cdt_ctx* ctx, as_list_order order, bool pad ); +/** + * Create list create operation. + * Server creates list at given context level. + * + * @param ops Target operations list. + * @param name Bin name. + * @param ctx Optional path to nested list. If not defined, the top-level list is used. + * @param order List order. + * @param pad If true, the context is allowed to be beyond list boundaries. In that case, nil + * list entries will be inserted to satisfy the context position. + * @param persist_index If true, persist list index. A list index improves lookup performance, + * but requires more storage. A list index can be created for a top-level + * ordered list only. Nested and unordered list indexes are not supported. + * + * @ingroup map_operations + */ +AS_EXTERN bool +as_operations_list_create_all( + as_operations* ops, const char* name, as_cdt_ctx* ctx, as_list_order order, bool pad, bool persist_index + ); + /** * Create set list order operation. * Server sets list order. Server returns null. diff --git a/src/main/aerospike/as_list_operations.c b/src/main/aerospike/as_list_operations.c index 8eb1265d37..c41f9216da 100644 --- a/src/main/aerospike/as_list_operations.c +++ b/src/main/aerospike/as_list_operations.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2022 Aerospike, Inc. + * Copyright 2008-2024 Aerospike, Inc. * * Portions may be licensed to Aerospike, Inc. under one or more contributor * license agreements. @@ -104,6 +104,36 @@ as_operations_list_create( return as_cdt_add_packed(&pk, ops, name, AS_OPERATOR_CDT_MODIFY); } +bool +as_operations_list_create_all( + as_operations* ops, const char* name, as_cdt_ctx* ctx, as_list_order order, bool pad, bool persist_index + ) +{ + // If context not defined, the set order for top-level bin list. + if (! ctx) { + uint64_t flag = (uint64_t)order; + + if (persist_index) { + flag |= 0x10; + } + + as_packer pk = as_cdt_begin(); + as_cdt_pack_header(&pk, ctx, SET_TYPE, 1); + as_pack_uint64(&pk, flag); + as_cdt_end(&pk); + return as_cdt_add_packed(&pk, ops, name, AS_OPERATOR_CDT_MODIFY); + } + + uint32_t flag = as_list_order_to_flag(order, pad); + + // Create nested list. persist_index does not apply here, so ignore it. + as_packer pk = as_cdt_begin(); + as_cdt_pack_header_flag(&pk, ctx, SET_TYPE, 1, flag); + as_pack_uint64(&pk, (uint64_t)order); + as_cdt_end(&pk); + return as_cdt_add_packed(&pk, ops, name, AS_OPERATOR_CDT_MODIFY); +} + bool as_operations_list_set_order( as_operations* ops, const char* name, as_cdt_ctx* ctx, as_list_order order diff --git a/src/main/aerospike/as_map_operations.c b/src/main/aerospike/as_map_operations.c index 7aa2384193..8908817e56 100644 --- a/src/main/aerospike/as_map_operations.c +++ b/src/main/aerospike/as_map_operations.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2023 Aerospike, Inc. + * Copyright 2008-2024 Aerospike, Inc. * * Portions may be licensed to Aerospike, Inc. under one or more contributor * license agreements. @@ -148,7 +148,7 @@ as_operations_map_create( as_operations* ops, const char* name, as_cdt_ctx* ctx, as_map_order order ) { - // If context not defined, the set order for top-level bin list. + // If context not defined, the set order for top-level bin map. if (! ctx) { as_map_policy policy; as_map_policy_set(&policy, order, AS_MAP_UPDATE); @@ -169,7 +169,7 @@ as_operations_map_create_all( as_operations* ops, const char* name, as_cdt_ctx* ctx, as_map_order order, bool persist_index ) { - // If context not defined, the set order for top-level bin list. + // If context not defined, the set order for top-level bin map. if (! ctx) { as_map_policy policy; as_map_policy_set_all(&policy, order, AS_MAP_UPDATE, persist_index); diff --git a/src/test/aerospike_list/list_basics.c b/src/test/aerospike_list/list_basics.c index e15e8bf56c..620a19c767 100644 --- a/src/test/aerospike_list/list_basics.c +++ b/src/test/aerospike_list/list_basics.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2023 Aerospike, Inc. + * Copyright 2008-2024 Aerospike, Inc. * * Portions may be licensed to Aerospike, Inc. under one or more contributor * license agreements. @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -27,13 +28,16 @@ #include #include #include +#include #include #include #include #include +#include #include #include "../test.h" +#include "../util/udf.h" /****************************************************************************** * GLOBAL VARS @@ -73,6 +77,37 @@ typedef struct list_order_type_s * STATIC FUNCTIONS *****************************************************************************/ +#define LUA_DIR AS_START_DIR "src/test/lua/" + +static bool +load_udf(const char* filename) +{ + as_error err; + as_bytes content; + + char namebuf[512]; + sprintf(namebuf, "%s%s", LUA_DIR, filename); + info("reading: %s", namebuf); + bool b = udf_readfile(namebuf, &content); + + if (! b) { + info("read failed"); + return false; + } + + info("uploading: %s",filename); + aerospike_udf_put(as, &err, NULL, filename, AS_UDF_TYPE_LUA, &content); + + if (err.code != AEROSPIKE_OK) { + info("put failed"); + return false; + } + + aerospike_udf_put_wait(as, &err, NULL, filename, 100); + as_bytes_destroy(&content); + return true; +} + static bool as_testlist_compare(as_testlist *tlist); static bool @@ -2718,7 +2753,6 @@ TEST(list_exp_mod, "List Modify Expressions") as_exp_destroy(filter7); } - TEST(list_exp_read, "List Read Expressions") { as_key rkey; @@ -2904,6 +2938,290 @@ TEST(list_exp_infinity, "test as_exp_inf()") as_exp_destroy(read_exp); } +// Add flag for the purpose of tests that bypass user functions and use low-level wire protocol. +#define AS_LIST_FLAG_PERSIST_INDEX 0x10 + +TEST(list_persist_index, "test persist index") +{ + as_key rkey; + as_key_init_int64(&rkey, NAMESPACE, SET, 212); + + as_error err; + as_status status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK || status == AEROSPIKE_ERR_RECORD_NOT_FOUND); + + uint8_t buf[4096]; + as_bytes b; + + // Test out of order. + as_packer pk = { + .buffer = buf, + .capacity = sizeof(buf) + }; + + as_pack_list_header(&pk, 6); + as_pack_ext_header(&pk, 0, AS_LIST_ORDERED | AS_LIST_FLAG_PERSIST_INDEX); + + for (int i = 0; i < 5; i++) { + as_pack_int64(&pk, 5 - i); + } + + as_bytes_init_wrap(&b, buf, pk.offset, false); + as_bytes_set_type(&b, AS_BYTES_LIST); + + as_record* rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_ne(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + as_operations ops; + as_cdt_ctx ctx; + + // Test ctx create. + as_arraylist list; + as_arraylist_init(&list, 1, 0); + as_arraylist_append_int64(&list, 1); + + as_operations_init(&ops, 2); + as_operations_list_create_all(&ops, BIN_NAME, NULL, AS_LIST_UNORDERED, false, true); + as_operations_list_append(&ops, BIN_NAME, NULL, NULL, (as_val*)&list); + + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_operations_destroy(&ops); + as_record_destroy(rec); + rec = NULL; + + // Get. + status = aerospike_key_get(as, &err, NULL, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + + as_list* check_list = as_record_get_list(rec, BIN_NAME); + assert_not_null(check_list); + as_list* check_list1 = as_list_get_list(check_list, 0); + assert_not_null(check_list1); + assert_int_eq(as_list_size(check_list1), 1); + assert_int_eq(as_list_get_int64(check_list1, 0), 1); + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + // Test ctx create UNBOUNDED. + status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK); + + as_arraylist_init(&list, 1, 0); + as_arraylist_append_int64(&list, 1); + + as_operations_init(&ops, 2); + as_operations_list_create_all(&ops, BIN_NAME, NULL, AS_LIST_UNORDERED, true, true); + as_operations_list_insert(&ops, BIN_NAME, NULL, NULL, 10, (as_val*)&list); + + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_operations_destroy(&ops); + as_record_destroy(rec); + rec = NULL; + + // Get. + status = aerospike_key_get(as, &err, NULL, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + check_list = as_record_get_list(rec, BIN_NAME); + assert_int_eq(as_list_size(check_list), 11); + check_list1 = as_list_get_list(check_list, 10); + assert_int_eq(as_list_get_int64(check_list1, 0), 1); + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + // Test ctx create sub presist rejection. + status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK); + + as_operations_init(&ops, 1); + + as_cdt_ctx_init(&ctx, 2); + as_cdt_ctx_add_list_index_create(&ctx, 0, AS_LIST_UNORDERED, false); + as_cdt_ctx_add_list_index_create(&ctx, 0, AS_LIST_ORDERED, false); + as_cdt_ctx_item* hack_item = as_vector_get(&ctx.list, ctx.list.size - 1); + hack_item->type |= 0x100; // hack in a persist flag, do not do this normally + + as_operations_list_append(&ops, BIN_NAME, &ctx, NULL, (as_val*)as_integer_new(1)); + + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_ne(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + as_operations_destroy(&ops); + as_cdt_ctx_destroy(&ctx); +} + +TEST(list_persist_udf, "test persist udf") +{ + load_udf("client_record_lists.lua"); + + as_key rkey; + as_key_init_int64(&rkey, NAMESPACE, SET, 213); + + as_error err; + as_status status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK || status == AEROSPIKE_ERR_RECORD_NOT_FOUND); + + as_arraylist l; + as_arraylist_init(&l, 4, 4); + + as_arraylist_append_int64(&l, 10); + as_arraylist_append_int64(&l, 20); + as_arraylist_append_int64(&l, 30); + as_arraylist_append_int64(&l, 0); + + as_record* rec = as_record_new(1); + as_record_set_list(rec, BIN_NAME, (as_list*)&l); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + as_operations ops; + as_operations_init(&ops, 1); + + as_operations_list_set_order(&ops, BIN_NAME, NULL, AS_LIST_ORDERED); + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + as_operations_destroy(&ops); + + // Get. + as_list *check_list; + status = aerospike_key_get(as, &err, NULL, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + check_list = as_record_get_list(rec, BIN_NAME); + assert_not_null(check_list); + assert_int_eq(as_list_size(check_list), 4); + int64_t check_int = as_list_get_int64(check_list, 0); + for (uint32_t i = 1; i < as_list_size(check_list); i++) { + int64_t num = as_list_get_int64(check_list, i); + assert_true(check_int <= num); + check_int = num; + } + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + as_arraylist args; + as_arraylist_init(&args,2,2); + as_arraylist_append_str(&args, BIN_NAME); + as_arraylist_append_int64(&args, 5); + as_val* val = NULL; + + aerospike_key_apply(as, &err, NULL, &rkey, "client_record_lists", "append", (as_list*)&args, &val); + assert_int_eq(err.code, AEROSPIKE_OK); + as_val_destroy(val); + as_arraylist_destroy(&args); + + // Get. + status = aerospike_key_get(as, &err, NULL, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + //example_dump_record(rec); + check_list = as_record_get_list(rec, BIN_NAME); + assert_not_null(check_list); + assert_int_eq(as_list_size(check_list), 5); + check_int = as_list_get_int64(check_list, 0); + for (uint32_t i = 1; i < as_list_size(check_list); i++) { + int64_t num = as_list_get_int64(check_list, i); + assert_true(check_int <= num); + check_int = num; + } + as_record_destroy(rec); + rec = NULL; +} + +TEST(list_ordered_udf, "test ordered udf") +{ + bool check = load_udf("list_unordered.lua"); + assert_true(check); + + as_key rkey; + as_key_init_int64(&rkey, NAMESPACE, SET, 214); + + as_error err; + as_status status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK || status == AEROSPIKE_ERR_RECORD_NOT_FOUND); + + const char* test_strings[] = { + "1", "231020204109691000", "231020204109704569" + }; + as_arraylist l; + as_arraylist_init(&l, 4, 4); + as_arraylist_append_str(&l, test_strings[1]); + as_arraylist_append_str(&l, test_strings[2]); + as_arraylist_append_str(&l, test_strings[0]); + + as_record* rec = as_record_new(1); + as_record_set_list(rec, "list1", (as_list*)&l); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + as_operations ops; + as_operations_init(&ops, 1); + as_operations_list_set_order(&ops, "list1", NULL, AS_LIST_ORDERED); + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + as_operations_destroy(&ops); + + // Get. + status = aerospike_key_get(as, &err, NULL, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + //example_dump_record(rec); + as_list* check_list = as_record_get_list(rec, "list1"); + for (int i = 0; i < 3; i++) { + assert_string_eq(test_strings[i], as_list_get_str(check_list, i)); + } + as_record_destroy(rec); + rec = NULL; + + const char* test_strings2[] = { + test_strings[0], test_strings[1], + "231020204109704558", + "231020204109704567", test_strings[2] + }; + as_arraylist args; + as_arraylist_init(&args,4,4); + as_arraylist_append_str(&args, test_strings2[2]); + as_arraylist_append_str(&args, test_strings2[3]); + as_val* val = NULL; + + aerospike_key_apply(as, &err, NULL, &rkey, "list_unordered", "list_unordered", (as_list*)&args, &val); + assert_int_eq(err.code, AEROSPIKE_OK); +// char* s = as_val_tostring(val); +// info("ret %s", s); +// info(s); +// free(s); + as_arraylist_destroy(&args); + as_val_destroy(val); + + // Get. + status = aerospike_key_get(as, &err, NULL, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + //example_dump_record(rec); + check_list = as_record_get_list(rec, "list1"); + for (int i = 0; i < 5; i++) { + assert_string_eq(test_strings2[i], as_list_get_str(check_list, i)); + } + as_record_destroy(rec); + rec = NULL; +} + /****************************************************************************** * TEST SUITE *****************************************************************************/ @@ -2939,4 +3257,8 @@ SUITE(list_basics, "aerospike list basic tests") suite_add(exp_returns_list); suite_add(list_exp_infinity); + + suite_add(list_persist_index); + suite_add(list_persist_udf); + suite_add(list_ordered_udf); } diff --git a/src/test/aerospike_map/map_basics.c b/src/test/aerospike_map/map_basics.c index cc03894308..ec47c5a829 100644 --- a/src/test/aerospike_map/map_basics.c +++ b/src/test/aerospike_map/map_basics.c @@ -1,5 +1,5 @@ /* - * Copyright 2008-2023 Aerospike, Inc. + * Copyright 2008-2024 Aerospike, Inc. * * Portions may be licensed to Aerospike, Inc. under one or more contributor * license agreements. @@ -2952,6 +2952,7 @@ TEST(ordered_map_eq_exp, "Ordered Map Equality Expression") as_exp_destroy(filter); } + TEST(map_inverted_exp, "Map Inverted Expression") { as_key rkey; @@ -2961,16 +2962,16 @@ TEST(map_inverted_exp, "Map Inverted Expression") as_status status = aerospike_key_remove(as, &err, NULL, &rkey); assert_true(status == AEROSPIKE_OK || status == AEROSPIKE_ERR_RECORD_NOT_FOUND); - as_hashmap map; - as_hashmap_init(&map, 4); - as_stringmap_set_int64((as_map*)&map, "a", 1); - as_stringmap_set_int64((as_map*)&map, "b", 2); - as_stringmap_set_int64((as_map*)&map, "c", 2); - as_stringmap_set_int64((as_map*)&map, "d", 3); + as_hashmap map; + as_hashmap_init(&map, 4); + as_stringmap_set_int64((as_map*)&map, "a", 1); + as_stringmap_set_int64((as_map*)&map, "b", 2); + as_stringmap_set_int64((as_map*)&map, "c", 2); + as_stringmap_set_int64((as_map*)&map, "d", 3); - const char* bin_name = "smap"; + const char* bin_name = "smap"; as_record rec; - as_record_init(&rec, 1); + as_record_init(&rec, 1); as_record_set_map(&rec, bin_name, (as_map*)&map); status = aerospike_key_put(as, &err, NULL, &rkey, &rec); @@ -2981,13 +2982,13 @@ TEST(map_inverted_exp, "Map Inverted Expression") as_exp_build(expr, as_exp_map_remove_by_value(NULL, AS_MAP_RETURN_INVERTED, as_exp_int(2), as_exp_bin_map(bin_name))); - as_operations ops; - as_operations_inita(&ops, 1); - as_operations_exp_read(&ops, bin_name, expr, AS_EXP_READ_DEFAULT); + as_operations ops; + as_operations_inita(&ops, 1); + as_operations_exp_read(&ops, bin_name, expr, AS_EXP_READ_DEFAULT); - as_record* results = NULL; + as_record* results = NULL; - status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &results); + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &results); assert_int_eq(status, AEROSPIKE_OK); as_map* map_result = as_record_get_map(results, bin_name); @@ -3010,6 +3011,466 @@ TEST(map_inverted_exp, "Map Inverted Expression") as_record_destroy(results); } +TEST(map_self_correct, "Test Map put with wrong order and compactness") +{ + as_key rkey; + as_key_init_int64(&rkey, NAMESPACE, SET, 30); + + as_error err; + as_status status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK || status == AEROSPIKE_ERR_RECORD_NOT_FOUND); + + uint8_t buf[4096]; + as_bytes b; + + // Test out of order rejection. + as_packer pk = { + .buffer = buf, + .capacity = sizeof(buf) + }; + + as_pack_map_header(&pk, 6); + + as_pack_ext_header(&pk, 0, AS_MAP_KEY_ORDERED); + as_pack_nil(&pk); + + for (int i = 0; i < 5; i++) { + as_pack_int64(&pk, 5 - i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + as_record* rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_ne(status, AEROSPIKE_OK); // rejected + as_record_destroy(rec); + rec = NULL; + + // Test compactify. + pk.offset = 0; + as_pack_map_header(&pk, 6); + + as_pack_ext_header(&pk, 0, AS_MAP_KEY_ORDERED); + as_pack_nil(&pk); + + as_pack_int64(&pk, 0); + as_pack_int64(&pk, 0x100000000); + + uint64_t* p64 = (uint64_t*)(pk.buffer + pk.offset - sizeof(uint64_t)); + *p64 = 0; // set it to 0 overpacked as a uint64_t so it cost 9 bytes instead of 1 + + for (int i = 1; i < 5; i++) { + as_pack_int64(&pk, i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + // Get. + as_policy_read rp; + as_policy_read_init(&rp); + rp.deserialize = false; + status = aerospike_key_get(as, &err, &rp, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_bytes *rb = as_record_get_bytes(rec, BIN_NAME); + assert_int_eq(as_bytes_size(rb), 15); + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + // Test compactify 2. + pk.offset = 0; + as_pack_map_header(&pk, 6); + + as_pack_ext_header(&pk, 0, AS_MAP_KEY_ORDERED); + as_pack_nil(&pk); + + as_pack_int64(&pk, 0); + as_pack_int64(&pk, -0x100000000); + + p64 = (uint64_t*)(pk.buffer + pk.offset - sizeof(uint64_t)); + *p64 = cf_swap_to_be64(4294967295); + + as_pack_int64(&pk, 1); + as_pack_int64(&pk, -0x100000000); + + p64 = (uint64_t*)(pk.buffer + pk.offset - sizeof(uint64_t)); + *p64 = cf_swap_to_be64(4294967296); + + for (int i = 2; i < 5; i++) { + as_pack_int64(&pk, i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + // Get. + status = aerospike_key_get(as, &err, &rp, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + rb = as_record_get_bytes(rec, BIN_NAME); + assert_int_eq(as_bytes_size(rb), 27); + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + // Test incorrect offset index. + pk.offset = 0; + as_pack_map_header(&pk, 6); + + uint8_t idxbuf[10]; + + for (size_t i = 0; i < sizeof(idxbuf); i++) { + idxbuf[i] = (uint8_t)i + 55; + } + + as_pack_ext_header(&pk, sizeof(idxbuf), AS_MAP_KEY_ORDERED); + as_pack_append(&pk, idxbuf, sizeof(idxbuf)); + as_pack_nil(&pk); + + for (int i = 0; i < 5; i++) { + as_pack_int64(&pk, i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + // Get. + status = aerospike_key_get(as, &err, &rp, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + rb = as_record_get_bytes(rec, BIN_NAME); + assert_int_eq(as_bytes_size(rb), 15); + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + // Test padding rejection. + pk.offset = 0; + as_pack_map_header(&pk, 6); + + as_pack_ext_header(&pk, 0, AS_MAP_KEY_ORDERED); + as_pack_nil(&pk); + + for (int i = 0; i < 5; i++) { + as_pack_int64(&pk, i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset + 1, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_ne(status, AEROSPIKE_OK); + as_record_destroy(rec); +} + +// Add flag for the purpose of tests that bypass user functions and use low-level wire protocol. +#define AS_MAP_FLAG_PERSIST_INDEX 0x10 + +TEST(map_persist_index, "Test Map Persist Index") +{ + as_key rkey; + as_key_init_int64(&rkey, NAMESPACE, SET, 31); + + as_error err; + as_status status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK || status == AEROSPIKE_ERR_RECORD_NOT_FOUND); + + uint8_t buf[4096]; + as_bytes b; + + // Test out of order rejection. + as_packer pk = { + .buffer = buf, + .capacity = sizeof(buf) + }; + + as_pack_map_header(&pk, 6); + + as_pack_ext_header(&pk, 0, AS_MAP_KEY_ORDERED | AS_MAP_FLAG_PERSIST_INDEX); + as_pack_nil(&pk); + + for (int i = 0; i < 5; i++) { + as_pack_int64(&pk, 5 - i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + as_record* rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_ne(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + // Test compactify. + pk.offset = 0; + as_pack_map_header(&pk, 6); + + as_pack_ext_header(&pk, 0, AS_MAP_KEY_ORDERED | AS_MAP_FLAG_PERSIST_INDEX); + as_pack_nil(&pk); + + as_pack_int64(&pk, 0); + as_pack_int64(&pk, 0x100000000); + + uint64_t* p64 = (uint64_t*)(pk.buffer + pk.offset - sizeof(uint64_t)); + *p64 = 0; // set it to 0 overpacked as a uint64_t so it cost 9 bytes instead of 1 + + for (int i = 1; i < 5; i++) { + as_pack_int64(&pk, i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + // Get. + as_policy_read rp; + as_policy_read_init(&rp); + rp.deserialize = false; + status = aerospike_key_get(as, &err, &rp, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_bytes *rb = as_record_get_bytes(rec, BIN_NAME); + assert_int_eq(as_bytes_size(rb), 15); + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + // Test incorrect offset index. + pk.offset = 0; + as_pack_map_header(&pk, 6); + + uint8_t idxbuf[10]; + + for (size_t i = 0; i < sizeof(idxbuf); i++) { + idxbuf[i] = (uint8_t)i + 55; + } + + as_pack_ext_header(&pk, sizeof(idxbuf), AS_MAP_KEY_ORDERED | AS_MAP_FLAG_PERSIST_INDEX); + as_pack_append(&pk, idxbuf, sizeof(idxbuf)); + as_pack_nil(&pk); + + for (int i = 0; i < 5; i++) { + as_pack_int64(&pk, i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_eq(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + // Get. + status = aerospike_key_get(as, &err, &rp, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + rb = as_record_get_bytes(rec, BIN_NAME); + assert_int_eq(as_bytes_size(rb), 15); + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + // Test padding rejection. + pk.offset = 0; + as_pack_map_header(&pk, 6); + + as_pack_ext_header(&pk, 0, AS_MAP_KEY_ORDERED | AS_MAP_FLAG_PERSIST_INDEX); + as_pack_nil(&pk); + + for (int i = 0; i < 5; i++) { + as_pack_int64(&pk, i); // key + as_pack_int64(&pk, i); // value + } + + as_bytes_init_wrap(&b, buf, pk.offset + 1, false); + as_bytes_set_type(&b, AS_BYTES_MAP); + + rec = as_record_new(1); + as_record_set_bytes(rec, BIN_NAME, (as_bytes*)&b); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_int_ne(status, AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + // Test ctx create. + status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK); + + as_arraylist list; + as_arraylist_init(&list, 1, 0); + as_arraylist_append_int64(&list, 1); + + as_operations ops; + as_operations_init(&ops, 2); + as_operations_map_create_all(&ops, BIN_NAME, NULL, AS_MAP_KEY_ORDERED, true); + as_operations_map_put(&ops, BIN_NAME, NULL, NULL, (as_val*)as_integer_new(0), (as_val*)&list); + + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_operations_destroy(&ops); + as_record_destroy(rec); + rec = NULL; + + // Get. + status = aerospike_key_get(as, &err, NULL, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_map* m = as_record_get_map(rec, BIN_NAME); + example_dump_record(rec); + assert_int_eq(m->flags, AS_MAP_KEY_ORDERED); + as_record_destroy(rec); + rec = NULL; + + // Test map create sub presist rejection. + status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK); + + as_cdt_ctx ctx; + as_cdt_ctx_init(&ctx, 2); + as_cdt_ctx_add_list_index_create(&ctx, 0, AS_LIST_UNORDERED, false); + as_cdt_ctx_add_map_key_create(&ctx, (as_val*)as_integer_new(0), AS_MAP_KEY_ORDERED); + as_cdt_ctx_item* hack_item = as_vector_get(&ctx.list, ctx.list.size - 1); + hack_item->type |= 0x100; // hack in a persist flag, do not do this normally + + as_operations_init(&ops, 1); + as_operations_list_append(&ops, BIN_NAME, &ctx, NULL, (as_val*)as_integer_new(1)); + + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_ne(status, AEROSPIKE_OK); // rejected + as_record_destroy(rec); + rec = NULL; + as_operations_destroy(&ops); + as_cdt_ctx_destroy(&ctx); + + // Test map create sub presist rejection 2. + as_arraylist_init(&list, 1, 0); + as_arraylist_append_int64(&list, 1); + as_operations_init(&ops, 2); + as_operations_map_create_all(&ops, BIN_NAME, NULL, AS_MAP_UNORDERED, true); + as_operations_map_put(&ops, BIN_NAME, NULL, NULL, (as_val*)as_integer_new(0), (as_val*)&list); + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_eq(status, AEROSPIKE_OK); + as_operations_destroy(&ops); + as_record_destroy(rec); + rec = NULL; + + as_cdt_ctx_init(&ctx, 3); + as_cdt_ctx_add_map_key_create(&ctx, (as_val*)as_integer_new(1), AS_MAP_FLAG_PERSIST_INDEX); + as_cdt_ctx_add_map_key_create(&ctx, (as_val*)as_integer_new(1), AS_MAP_FLAG_PERSIST_INDEX); + hack_item = as_vector_get(&ctx.list, ctx.list.size - 1); + hack_item->type |= 0x100; // hack in a persist flag, do not do this normally + as_cdt_ctx_add_map_key_create(&ctx, (as_val*)as_integer_new(2), AS_MAP_FLAG_PERSIST_INDEX); + hack_item = as_vector_get(&ctx.list, ctx.list.size - 1); + hack_item->type |= 0x100; // hack in a persist flag, do not do this normally + + as_operations_init(&ops, 1); + as_operations_list_append(&ops, BIN_NAME, &ctx, NULL, (as_val*)as_integer_new(1)); + + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_int_ne(status, AEROSPIKE_OK); // should be rejected + as_record_destroy(rec); + rec = NULL; + as_operations_destroy(&ops); + as_cdt_ctx_destroy(&ctx); + + // Test set flags. + status = aerospike_key_remove(as, &err, NULL, &rkey); + assert_true(status == AEROSPIKE_OK); + + as_orderedmap m1; + as_orderedmap_init(&m1, 4); + as_orderedmap_set(&m1, (as_val*)as_string_new("a1", false), (as_val*)as_integer_new(1)); + as_orderedmap_set(&m1, (as_val*)as_string_new("b1", false), (as_val*)as_integer_new(2)); + as_orderedmap_set(&m1, (as_val*)as_string_new("c1", false), (as_val*)as_integer_new(3)); + as_orderedmap_set(&m1, (as_val*)as_string_new("pk1", false), (as_val*)as_string_new("231108133342353844", false)); + + rec = as_record_new(1); + as_record_set_map(rec, BIN_NAME, (as_map*)&m1); + + status = aerospike_key_put(as, &err, NULL, &rkey, rec); + assert_true(status == AEROSPIKE_OK); + as_record_destroy(rec); + rec = NULL; + + // Get. + status = aerospike_key_get(as, &err, &rp, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + rb = as_record_get_bytes(rec, BIN_NAME); + uint32_t check_size = as_bytes_size(rb); + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; + + as_map_policy pol; + as_map_policy_init(&pol); + as_map_policy_set_all(&pol, AS_MAP_UNORDERED, 0, true); + as_operations_init(&ops, 1); + as_operations_add_map_set_policy(&ops, BIN_NAME, &pol); + + status = aerospike_key_operate(as, &err, NULL, &rkey, &ops, &rec); + assert_true(status == AEROSPIKE_OK); + as_record_destroy(rec); + as_operations_destroy(&ops); + rec = NULL; + + // Get. + status = aerospike_key_get(as, &err, &rp, &rkey, &rec); + assert_int_eq(status, AEROSPIKE_OK); + rb = as_record_get_bytes(rec, BIN_NAME); + assert_int_eq(as_bytes_size(rb), check_size - 4); // -4 for meta header + //example_dump_record(rec); + as_record_destroy(rec); + rec = NULL; +} + /****************************************************************************** * TEST SUITE *****************************************************************************/ @@ -3047,4 +3508,6 @@ SUITE(map_basics, "aerospike map basic tests") suite_add(map_ordered_result); suite_add(ordered_map_eq_exp); suite_add(map_inverted_exp); + suite_add(map_self_correct); + suite_add(map_persist_index); } diff --git a/src/test/lua/list_unordered.lua b/src/test/lua/list_unordered.lua new file mode 100644 index 0000000000..d3fa5fe565 --- /dev/null +++ b/src/test/lua/list_unordered.lua @@ -0,0 +1,13 @@ +function list_unordered(rec, arg1, arg2) + local ret1 = map() + ret1['arg1'] = tostring(arg1) + ret1['arg2'] = tostring(arg2) + local l1 = rec['list1'] + list.append(l1, arg1) + list.append(l1, arg2) + ret1['l1'] = tostring(l1) + rec['list1'] = l1 + aerospike:update(rec) + ret1['rec'] = tostring(rec) + return ret1 +end