From ef7e82c8e4914112a9bcec07bd14950c872a3a6e Mon Sep 17 00:00:00 2001 From: wangxp006 Date: Mon, 4 Apr 2022 09:55:33 +0800 Subject: [PATCH] add checker merge feature --- keepalived/check/check_api.c | 76 ++++++++++++++++++++++++++++++++- keepalived/check/check_daemon.c | 5 +++ keepalived/check/check_data.c | 38 +++++++++++++++++ keepalived/check/check_parser.c | 10 +++++ keepalived/check/ipwrapper.c | 19 +++++++++ keepalived/include/check_api.h | 3 ++ keepalived/include/check_data.h | 11 ++++- 7 files changed, 160 insertions(+), 2 deletions(-) diff --git a/keepalived/check/check_api.c b/keepalived/check/check_api.c index 72a79c61b9..316b0dca4b 100644 --- a/keepalived/check/check_api.c +++ b/keepalived/check/check_api.c @@ -59,10 +59,18 @@ list_head_t checkers_queue; bool do_checker_debug; #endif + /* free checker data */ void free_checker(checker_t *checker) { + list_head_t *l_checker, *l_tmp; + checker_t *temp; + list_for_each_safe(l_checker, l_tmp, &checker->i_list) { + temp = (checker_t *)list_entry(l_checker, checker_t, i_list); + (*temp->checker_funcs->free_func) (temp); + } + list_del_init(&checker->h_list); list_del_init(&checker->e_list); (*checker->checker_funcs->free_func) (checker); } @@ -75,6 +83,7 @@ free_checker_list(list_head_t *l) free_checker(checker); } + /* dump checker data */ static void dump_checker(FILE *fp, const checker_t *checker) @@ -152,6 +161,9 @@ queue_checker(const checker_funcs_t *funcs PMALLOC(checker); INIT_LIST_HEAD(&checker->e_list); + INIT_LIST_HEAD(&checker->h_list); + INIT_LIST_HEAD(&checker->i_list); + INIT_LIST_HEAD(&checker->u_list); checker->checker_funcs = funcs; checker->launch = launch; checker->vs = vs; @@ -172,6 +184,8 @@ queue_checker(const checker_funcs_t *funcs /* queue the checker */ list_add_tail(&checker->e_list, &checkers_queue); + rs->checker = checker; + if (fd_required) check_data->num_checker_fd_required++; @@ -507,14 +521,67 @@ free_checkers_queue(void) free_checker_list(&checkers_queue); } + +static inline int checker_merge_match(list_head_t * plist) +{ + void *addr = NULL; + void *address = NULL; + sa_family_t family; + checker_t *checker_s = list_entry(plist, checker_t , e_list); + uint32_t key; + checker_t *temp = NULL; + + family = checker_s->rs->addr.ss_family; + if (family == AF_INET6) { + addr = (void *) &((struct sockaddr_in6 *)&checker_s->rs->addr)->sin6_addr; + } else { + addr = (void *) &((struct sockaddr_in *)&checker_s->rs->addr)->sin_addr; + } + + key = real_server_hash(checker_s->rs); + list_for_each_entry(temp, &rs_check_merge[key], h_list) { + + if (family != temp->rs->addr.ss_family) { + log_message(LOG_ALERT, "Checker ip addr family not match, master:%d, slave%d\n", + temp->rs->addr.ss_family, family); + continue; + } + + if (temp->rs->addr.ss_family == AF_INET6) { + address = (void *) &((struct sockaddr_in6 *)&temp->rs->addr)->sin6_addr; + } else { + address = (void *) &((struct sockaddr_in *)&temp->rs->addr)->sin_addr; + } + + if (inaddr_equal(family, addr, address)) { + + list_del_init(plist); + list_add_tail(&checker_s->i_list, &temp->i_list); + + checker_s->rs->checker = checker_s; + return 1; + } + } + + list_add_tail(&checker_s->h_list, &rs_check_merge[key]); + return 0; +} + + /* register checkers to the global I/O scheduler */ void register_checkers_thread(void) { + list_head_t * l_checker, *l_tmp; checker_t *checker; unsigned long warmup; - list_for_each_entry(checker, &checkers_queue, e_list) { + list_for_each_safe(l_checker, l_tmp, &checkers_queue) { + checker = (checker_t *)list_entry(l_checker, checker_t, e_list); + + checker->rs->checker = checker; + + if (checker->launch) { if (checker->vs->ha_suspend && !checker->vs->ha_suspend_addr_count) checker->enabled = false; @@ -533,6 +600,13 @@ register_checkers_thread(void) /* coverity[dont_call] */ warmup = warmup * (unsigned)random() / RAND_MAX; } + + /* merge the later checkers if can be merged. */ + if (checker->vs->check_merge) { + if (checker_merge_match(l_checker)) + continue; + } + thread_add_timer(master, checker->launch, checker, BOOTSTRAP_DELAY + warmup); } diff --git a/keepalived/check/check_daemon.c b/keepalived/check/check_daemon.c index 4c5dcdf823..c70f975100 100644 --- a/keepalived/check/check_daemon.c +++ b/keepalived/check/check_daemon.c @@ -164,6 +164,7 @@ checker_terminate_phase2(void) thread_destroy_master(master); master = NULL; free_checkers_queue(); + free_rs_check_merge_hash(); free_ssl(); set_ping_group_range(false); @@ -319,6 +320,8 @@ start_check(list_head_t *old_checkers_queue, data_t *prev_global_data) return; } + init_rs_check_merge_hash(); + init_data(conf_file, check_init_keywords, false); if (reload) @@ -505,6 +508,7 @@ reload_check_thread(__attribute__((unused)) thread_ref_t thread) thread_cleanup_master(master, true); thread_add_base_threads(master, with_snmp); + free_rs_check_merge_hash(); /* Save previous checker data */ list_copy(&old_checkers_queue, &checkers_queue); init_checkers_queue(); @@ -761,6 +765,7 @@ start_check_child(void) master = thread_make_master(); #endif + /* If last process died during a reload, we can get there and we * don't want to loop again, because we're not reloading anymore. */ diff --git a/keepalived/check/check_data.c b/keepalived/check/check_data.c index 7895c399ff..618bff929c 100644 --- a/keepalived/check/check_data.c +++ b/keepalived/check/check_data.c @@ -46,7 +46,9 @@ /* global vars */ check_data_t *check_data = NULL; + check_data_t *old_check_data = NULL; +list_head_t rs_check_merge[RS_CHECK_MERGE_ENTRY_NUM]; /* SSL facility functions */ ssl_data_t * @@ -823,6 +825,7 @@ alloc_vs(const char *param1, const char *param2) #endif new->alpha = false; new->omega = false; + new->check_merge = 0; new->notify_quorum_up = NULL; new->notify_quorum_down = NULL; new->quorum = 1; @@ -929,6 +932,24 @@ alloc_check_data(void) return new; } +void +init_rs_check_merge_hash(void) +{ + int i; + for (i = 0; i < RS_CHECK_MERGE_ENTRY_NUM; i++) { + INIT_LIST_HEAD(&rs_check_merge[i]); + } +} + +void +free_rs_check_merge_hash(void) +{ + int i; + for (i = 0; i < RS_CHECK_MERGE_ENTRY_NUM; i++) { + list_del_init(&rs_check_merge[i]); + } +} + void free_check_data(check_data_t *data) @@ -1398,3 +1419,20 @@ validate_check_config(void) return true; } + +uint32_t __attribute__ ((pure)) +real_server_hash(real_server_t *s) +{ + struct sockaddr_in *addr = NULL; + struct sockaddr_in6 *addr6 = NULL; + __be32 addr_fold; + if (s->addr.ss_family == AF_INET) { + addr = (struct sockaddr_in *)&s->addr; + addr_fold = ntohl(addr->sin_addr.s_addr); + } else { + addr6 = (struct sockaddr_in6 *)&s->addr; + addr_fold = ntohl((addr6->sin6_addr.s6_addr32[0]) ^ ntohl(addr6->sin6_addr.s6_addr32[1]) ^ + (addr6->sin6_addr.s6_addr32[2]) ^ (addr6->sin6_addr.s6_addr32[3])); + } + return (addr_fold ^ (addr_fold >> 16)) & RS_CHECK_MERGE_MASK; +} \ No newline at end of file diff --git a/keepalived/check/check_parser.c b/keepalived/check/check_parser.c index a3ac244ae6..a85137a755 100644 --- a/keepalived/check/check_parser.c +++ b/keepalived/check/check_parser.c @@ -922,6 +922,15 @@ omega_handler(__attribute__((unused)) const vector_t *strvec) virtual_server_t *vs = list_last_entry(&check_data->vs, virtual_server_t, e_list); vs->omega = true; } + +static void +checker_merge_handler(__attribute__((unused))const vector_t *strvec) +{ + virtual_server_t *vs = list_last_entry(&check_data->vs, virtual_server_t, e_list); + vs->check_merge = 1; +} + + static void quorum_up_handler(const vector_t *strvec) { @@ -1037,6 +1046,7 @@ init_check_keywords(bool active) install_keyword("quorum", &quorum_handler); install_keyword("hysteresis", &hysteresis_handler); install_keyword("weight", &vs_weight_handler); + install_keyword("checker_merge", &checker_merge_handler); /* Real server mapping */ install_keyword("sorry_server", &ssvr_handler); diff --git a/keepalived/check/ipwrapper.c b/keepalived/check/ipwrapper.c index ec108f360d..08b2adf494 100644 --- a/keepalived/check/ipwrapper.c +++ b/keepalived/check/ipwrapper.c @@ -742,6 +742,23 @@ set_checker_state(checker_t *checker, bool up) checker->rs->num_failed_checkers--; } +static void __update_checkers_queue_group(bool alive, checker_t *checker) +{ + checker_t *checker_tmp = NULL; + + if (list_empty(&checker->i_list)) + return; + + list_head_t * l = &checker->i_list; + list_for_each_entry(checker_tmp, l, i_list) { + if ((alive && ISALIVE(checker_tmp->rs)) || (!alive && !ISALIVE(checker_tmp->rs))) + continue; + + perform_svr_state(alive, checker_tmp); + set_checker_state(checker_tmp, alive); + } +} + /* Update checker's state */ void update_svr_checker_state(bool alive, checker_t *checker) @@ -762,6 +779,7 @@ update_svr_checker_state(bool alive, checker_t *checker) if (checker->rs->num_failed_checkers <= 1) { if (!perform_svr_state(true, checker)) return; + __update_checkers_queue_group(alive, checker); } } else { @@ -769,6 +787,7 @@ update_svr_checker_state(bool alive, checker_t *checker) if (checker->rs->num_failed_checkers == 0) { if (!perform_svr_state(false, checker)) return; + __update_checkers_queue_group(alive, checker); } } diff --git a/keepalived/include/check_api.h b/keepalived/include/check_api.h index b0c1104552..7d16b137bf 100644 --- a/keepalived/include/check_api.h +++ b/keepalived/include/check_api.h @@ -86,6 +86,9 @@ typedef struct _checker { /* Linked list member */ list_head_t e_list; + list_head_t h_list; /* rs_hash_match */ + list_head_t i_list; /* check merge list */ + list_head_t u_list; /* unused checker*/ } checker_t; typedef struct _checker_ref { diff --git a/keepalived/include/check_data.h b/keepalived/include/check_data.h index fdaab0ef28..c32615576f 100644 --- a/keepalived/include/check_data.h +++ b/keepalived/include/check_data.h @@ -51,6 +51,9 @@ /* Daemon dynamic data structure definition */ #define KEEPALIVED_DEFAULT_DELAY (60 * TIMER_HZ) +#define RS_CHECK_MERGE_BITS 16 +#define RS_CHECK_MERGE_ENTRY_NUM (1 << RS_CHECK_MERGE_BITS) +#define RS_CHECK_MERGE_MASK (RS_CHECK_MERGE_ENTRY_NUM - 1) #ifdef _WITH_NFTABLES_ /* Used for arrays of protocol entries */ @@ -127,6 +130,7 @@ typedef struct _real_server { list_head_t tracked_bfds; /* cref_tracked_bfd_t */ #endif + struct _checker *checker; /* Linked list member */ list_head_t e_list; } real_server_t; @@ -229,6 +233,8 @@ typedef struct _virtual_server { struct ip_vs_stats64 stats; #endif #endif + + unsigned check_merge; /* Linked list member */ list_head_t e_list; } virtual_server_t; @@ -291,6 +297,7 @@ protocol_to_index(int proto) /* Global vars exported */ extern check_data_t *check_data; extern check_data_t *old_check_data; +extern list_head_t rs_check_merge[RS_CHECK_MERGE_ENTRY_NUM]; /* prototypes */ extern ssl_data_t *alloc_ssl(void) __attribute((malloc)); @@ -314,5 +321,7 @@ extern const char *format_vs (const virtual_server_t *); extern const char *format_vsge (const virtual_server_group_entry_t *); extern const char *format_rs(const real_server_t *, const virtual_server_t *); extern bool validate_check_config(void); - +extern void init_rs_check_merge_hash(void); +extern void free_rs_check_merge_hash(void); +extern uint32_t real_server_hash(real_server_t *); #endif