Merge pull request #4372 from sysown/v2.x_prep_stmt_mem_usage

Added memory usage monitoring for the prepared statement cache.

include/MySQL_PreparedStatement.h

@@ -60,6 +60,7 @@ class MySQL_STMT_Global_info {
 	uint16_t warning_count;
 	MYSQL_FIELD **fields;
 	char* first_comment;
+	uint64_t total_mem_usage;
 //	struct {
 //		int cache_ttl;
 //		int timeout;
@@ -70,6 +71,7 @@ class MySQL_STMT_Global_info {
 	MySQL_STMT_Global_info(uint64_t id, char *u, char *s, char *q, unsigned int ql, char *fc, MYSQL_STMT *stmt, uint64_t _h);
 	void update_metadata(MYSQL_STMT *stmt);
 	~MySQL_STMT_Global_info();
+	void calculate_mem_usage();
 };
 
 
@@ -264,6 +266,7 @@ class MySQL_STMT_Manager_v14 {
 	MySQL_STMT_Global_info * add_prepared_statement(char *u, char *s, char *q, unsigned int ql, char *fc, MYSQL_STMT *stmt, bool lock=true);
 	void get_metrics(uint64_t *c_unique, uint64_t *c_total, uint64_t *stmt_max_stmt_id, uint64_t *cached, uint64_t *s_unique, uint64_t *s_total);
 	SQLite3_result * get_prepared_statements_global_infos();
+	void get_memory_usage(uint64_t& prep_stmt_metadata_mem_usage, uint64_t& prep_stmt_backend_mem_usage);
 };
 
 #endif /* CLASS_MYSQL_PREPARED_STATEMENT_H */

include/proxysql_admin.h

@@ -79,6 +79,8 @@ struct p_admin_gauge {
 		stack_memory_mysql_threads,
 		stack_memory_admin_threads,
 		stack_memory_cluster_threads,
+		prepare_stmt_metadata_memory_bytes,
+		prepare_stmt_backend_memory_bytes,
 		// stmt metrics
 		stmt_client_active_total,
 		stmt_client_active_unique,

lib/MySQL_PreparedStatement.cpp

@@ -140,6 +140,7 @@ MySQL_STMT_Global_info::MySQL_STMT_Global_info(uint64_t id,
                                                char *fc,
                                                MYSQL_STMT *stmt, uint64_t _h) {
 	pthread_rwlock_init(&rwlock_, NULL);
+	total_mem_usage = 0;
 	statement_id = id;
 	ref_count_client = 0;
 	ref_count_server = 0;
@@ -294,6 +295,33 @@ MySQL_STMT_Global_info::MySQL_STMT_Global_info(uint64_t id,
 			memset(params[i], 0, sizeof(MYSQL_BIND));
 		}
 	}
+
+	calculate_mem_usage();
+}
+
+void MySQL_STMT_Global_info::calculate_mem_usage() {
+	total_mem_usage = sizeof(MySQL_STMT_Global_info) +
+		(num_params * (sizeof(MYSQL_BIND) + sizeof(MYSQL_BIND*))) +
+		(num_columns * (sizeof(MYSQL_FIELD) + sizeof(MYSQL_FIELD*))) +
+		query_length + 1;// +
+		//(ref_count_client * 24) +
+		//(ref_count_server * 24);
+
+	if (username) total_mem_usage += strlen(username) + 1;
+	if (schemaname) total_mem_usage += strlen(schemaname) + 1;
+	if (first_comment) total_mem_usage += strlen(first_comment) + 1;
+	if (digest_text) total_mem_usage += strlen(digest_text) + 1;
+
+	for (uint16_t i = 0; i < num_columns; i++) {
+		const MYSQL_FIELD* fd = fields[i];
+		if (fd->name) total_mem_usage += strlen(fd->name) + 1;
+		if (fd->org_name) total_mem_usage += strlen(fd->org_name) + 1;
+		if (fd->table) total_mem_usage += strlen(fd->table) + 1;
+		if (fd->org_table) total_mem_usage += strlen(fd->org_table) + 1;
+		if (fd->db) total_mem_usage += strlen(fd->db) + 1;
+		if (fd->catalog) total_mem_usage += strlen(fd->catalog) + 1;
+		if (fd->def) total_mem_usage += strlen(fd->def) + 1;
+	}
 }
 
 void MySQL_STMT_Global_info::update_metadata(MYSQL_STMT *stmt) {
@@ -478,6 +506,7 @@ void MySQL_STMT_Global_info::update_metadata(MYSQL_STMT *stmt) {
 			}
 		}
 // till here is copied from constructor
+		calculate_mem_usage();
 	}
 	pthread_rwlock_unlock(&rwlock_);
 }
@@ -892,6 +921,33 @@ MySQL_STMT_Global_info *MySQL_STMT_Manager_v14::add_prepared_statement(
 	return ret;
 }
 
+
+void MySQL_STMT_Manager_v14::get_memory_usage(uint64_t& prep_stmt_metadata_mem_usage, uint64_t& prep_stmt_backend_mem_usage) {
+	prep_stmt_backend_mem_usage = 0;
+	prep_stmt_metadata_mem_usage = sizeof(MySQL_STMT_Manager_v14);
+	rdlock();	
+	prep_stmt_metadata_mem_usage += map_stmt_id_to_info.size() * (sizeof(uint64_t) + sizeof(MySQL_STMT_Global_info*));
+	prep_stmt_metadata_mem_usage += map_stmt_hash_to_info.size() * (sizeof(uint64_t) + sizeof(MySQL_STMT_Global_info*));
+	prep_stmt_metadata_mem_usage += free_stmt_ids.size() * (sizeof(uint64_t));
+	for (const auto& keyval : map_stmt_id_to_info) {
+		const MySQL_STMT_Global_info* stmt_global_info = keyval.second;
+		prep_stmt_metadata_mem_usage += stmt_global_info->total_mem_usage;
+		prep_stmt_metadata_mem_usage += stmt_global_info->ref_count_server *
+			((stmt_global_info->num_params * sizeof(MYSQL_BIND)) +
+			(stmt_global_info->num_columns * sizeof(MYSQL_FIELD))) + 16;
+		prep_stmt_metadata_mem_usage += stmt_global_info->ref_count_client *
+			((stmt_global_info->num_params * sizeof(MYSQL_BIND)) +
+			(stmt_global_info->num_columns * sizeof(MYSQL_FIELD))) + 16;
+
+		// backend
+		prep_stmt_backend_mem_usage += stmt_global_info->ref_count_server * (sizeof(MYSQL_STMT) +
+			56 + //sizeof(MADB_STMT_EXTENSION)
+			(stmt_global_info->num_params * sizeof(MYSQL_BIND)) + 
+			(stmt_global_info->num_columns * sizeof(MYSQL_BIND)));
+	}
+	unlock();
+}
+
 void MySQL_STMT_Manager_v14::get_metrics(uint64_t *c_unique, uint64_t *c_total,
                              uint64_t *stmt_max_stmt_id, uint64_t *cached,
                              uint64_t *s_unique, uint64_t *s_total) {

lib/MySQL_Session.cpp

@@ -4326,6 +4326,7 @@ bool MySQL_Session::handler_rc0_PROCESSING_STMT_PREPARE(enum session_status& st,
 			stmt_info->digest_text=strdup(CurrentQuery.QueryParserArgs.digest_text);
 			stmt_info->digest=CurrentQuery.QueryParserArgs.digest;	// copy digest
 			stmt_info->MyComQueryCmd=CurrentQuery.MyComQueryCmd; // copy MyComQueryCmd
+			stmt_info->calculate_mem_usage();
 		}
 	}
 	global_stmtid=stmt_info->statement_id;

lib/MySQL_Thread.cpp

@@ -2280,7 +2280,7 @@ char ** MySQL_Threads_Handler::get_variables_list() {
 		VariablesPointers_int["max_allowed_packet"]        = make_tuple(&variables.max_allowed_packet,        8192,   1024*1024*1024, false);
 		VariablesPointers_int["max_connections"]           = make_tuple(&variables.max_connections,              1,        1000*1000, false);
 		VariablesPointers_int["max_stmts_per_connection"]  = make_tuple(&variables.max_stmts_per_connection,     1,             1024, false);
-		VariablesPointers_int["max_stmts_cache"]           = make_tuple(&variables.max_stmts_cache,           1024,        1024*1024, false);
+		VariablesPointers_int["max_stmts_cache"]           = make_tuple(&variables.max_stmts_cache,            128,        1024*1024, false);
 		VariablesPointers_int["max_transaction_idle_time"] = make_tuple(&variables.max_transaction_idle_time, 1000,  20*24*3600*1000, false);
 		VariablesPointers_int["max_transaction_time"]      = make_tuple(&variables.max_transaction_time,      1000,  20*24*3600*1000, false);
 		VariablesPointers_int["query_cache_size_mb"]       = make_tuple(&variables.query_cache_size_MB,          0,       1024*10240, false);

lib/ProxySQL_Admin.cpp

@@ -912,6 +912,18 @@ admin_metrics_map = std::make_tuple(
 			"This is the number of global prepared statements for which proxysql has metadata.",
 			metric_tags {}
 		),
+		std::make_tuple(
+			p_admin_gauge::prepare_stmt_metadata_memory_bytes,
+			"prepare_stmt_metadata_memory_bytes",
+			"Memory used to store meta data related to prepare statements.",
+			metric_tags{}
+		),
+		std::make_tuple(
+			p_admin_gauge::prepare_stmt_backend_memory_bytes,
+			"prepare_stmt_backend_memory_bytes",
+			"Memory used by backend server related to prepare statements.",
+			metric_tags{}
+		),
 		std::make_tuple (
 			p_admin_gauge::fds_in_use,
 			"proxysql_fds_in_use",
@@ -8981,6 +8993,13 @@ void ProxySQL_Admin::p_stats___memory_metrics() {
 		__sync_fetch_and_add(&GloVars.statuses.stack_memory_cluster_threads, 0);
 	this->metrics.p_gauge_array[p_admin_gauge::stack_memory_cluster_threads]->Set(stack_memory_cluster_threads);
 
+	// proxysql_prepare_statement_memory metric
+	uint64_t prepare_stmt_metadata_mem_used;
+	uint64_t prepare_stmt_backend_mem_used;
+	GloMyStmt->get_memory_usage(prepare_stmt_metadata_mem_used, prepare_stmt_backend_mem_used);
+	this->metrics.p_gauge_array[p_admin_gauge::prepare_stmt_metadata_memory_bytes]->Set(prepare_stmt_metadata_mem_used);
+	this->metrics.p_gauge_array[p_admin_gauge::prepare_stmt_backend_memory_bytes]->Set(prepare_stmt_backend_mem_used);
+
 	// Update opened file descriptors
 	int32_t cur_fds = get_open_fds();
 	if (cur_fds != -1) {
@@ -9095,6 +9114,23 @@ void ProxySQL_Admin::stats___memory_metrics() {
 			statsdb->execute(query);
 			free(query);
 		}
+		if (GloMyStmt) {
+			uint64_t prep_stmt_metadata_mem_usage;
+			uint64_t prep_stmt_backend_mem_usage;
+			GloMyStmt->get_memory_usage(prep_stmt_metadata_mem_usage, prep_stmt_backend_mem_usage);
+			vn = (char*)"prepare_statement_metadata_memory";
+			sprintf(bu, "%llu", prep_stmt_metadata_mem_usage);
+			query=(char*)malloc(strlen(a)+strlen(vn)+strlen(bu)+16);
+			sprintf(query, a, vn, bu);
+			statsdb->execute(query);
+			free(query);
+			vn = (char*)"prepare_statement_backend_memory";
+			sprintf(bu, "%llu", prep_stmt_backend_mem_usage);
+			query=(char*)malloc(strlen(a)+strlen(vn)+strlen(bu)+16);
+			sprintf(query, a, vn, bu);
+			statsdb->execute(query);
+			free(query);
+		}
 		if (GloQPro) {
 			unsigned long long mu = 0;
 			mu = GloQPro->get_mysql_firewall_memory_users_table();

test/tap/tests/test_prepare_statement_memory_usage-t.cpp

@@ -0,0 +1,151 @@
+/**
+ * @file test_prepare_statement_memory_usage-t.cpp
+ * @brief Examines the memory consumption of the prepared statement cache.. 
+ * @details This test assesses the memory utilization of prepared statement metadata/backend cache memory.
+ */
+
+#include <string>
+#include "mysql.h"
+#include "tap.h"
+#include "command_line.h"
+#include "proxysql_utils.h"
+#include "utils.h"
+
+enum ComparisonOperator {
+	kEqual = 0x00000001,
+	kGreaterThan = 0x00000002,
+	kLessThan = 0x00000004
+};
+
+int get_prepare_stmt_mem_usage(MYSQL* admin, uint64_t& prep_stmt_metadata_mem, uint64_t& prep_stmt_backend_mem) {
+	prep_stmt_metadata_mem = prep_stmt_backend_mem = 0;
+	MYSQL_QUERY_T(admin, "SELECT variable_name, variable_value FROM stats_memory_metrics WHERE \
+		variable_name IN ('prepare_statement_metadata_memory', 'prepare_statement_backend_memory')");
+	MYSQL_RES* myres = mysql_store_result(admin);
+	while (MYSQL_ROW myrow = mysql_fetch_row(myres)) {
+		if (strncmp(myrow[0], "prepare_statement_metadata_memory", sizeof("prepare_statement_metadata_memory") - 1) == 0) {
+			prep_stmt_metadata_mem = std::stoull(myrow[1], nullptr, 10);
+		} else if (strncmp(myrow[0], "prepare_statement_backend_memory", sizeof("prepare_statement_backend_memory") - 1) == 0) {
+			prep_stmt_backend_mem = std::stoull(myrow[1], nullptr, 10);
+		} else {
+			fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid resultset");
+			mysql_free_result(myres);
+			return EXIT_FAILURE;
+		}
+	}
+	mysql_free_result(myres);
+	return EXIT_SUCCESS;
+}
+
+int check_prepare_statement_mem_usage(MYSQL* proxysql_admin, MYSQL* proxysql, const char* query, int prep_stmt_metadata_mem_comp,
+	int prep_stmt_backend_mem_comp) {
+	uint64_t old_prep_stmt_metadata_mem, old_prep_stmt_backend_mem;
+	if (get_prepare_stmt_mem_usage(proxysql_admin, old_prep_stmt_metadata_mem, old_prep_stmt_backend_mem) == EXIT_FAILURE) {
+		return EXIT_FAILURE;
+	}
+	MYSQL_STMT* stmt = mysql_stmt_init(proxysql);
+	if (!stmt) {
+		diag("mysql_stmt_init(), out of memory\n");
+		return EXIT_FAILURE;
+	}
+	if (mysql_stmt_prepare(stmt, query, strlen(query))) {
+		diag("query: %s", query);
+		diag("mysql_stmt_prepare at line %d failed: %s", __LINE__, mysql_error(proxysql));
+		mysql_stmt_close(stmt);
+		return EXIT_FAILURE;
+	} else {
+		ok(true, "Prepare succeeded: %s", query);
+	}
+	uint64_t new_prep_stmt_metadata_mem, new_prep_stmt_backend_mem;
+	if (get_prepare_stmt_mem_usage(proxysql_admin, new_prep_stmt_metadata_mem, new_prep_stmt_backend_mem) == EXIT_FAILURE) {
+		mysql_stmt_close(stmt);
+		return EXIT_FAILURE;
+	}
+	auto fnCompare = [](const uint64_t& val1, const uint64_t& val2, int co) -> bool {
+		bool res = false;
+		if ((co & kLessThan) == kLessThan) {
+			if ((co & kEqual) == kEqual) {
+				res = (val1 >= val2);
+			} else {
+				res = (val1 > val2);
+			}
+		} else if ((co & kGreaterThan) == kGreaterThan) {
+			if ((co & kEqual) == kEqual) {
+				res = (val1 <= val2);
+			} else {
+				res = (val1 < val2);
+			}
+		} else {
+			res = (val1 == val2);
+		}
+		return res;
+		};
+
+	ok(fnCompare(old_prep_stmt_metadata_mem, new_prep_stmt_metadata_mem, prep_stmt_metadata_mem_comp),
+		"Memory usage check [%d]. 'prepare_statement_metadata_memory':[%lu] [%lu]", prep_stmt_metadata_mem_comp,
+		old_prep_stmt_metadata_mem, new_prep_stmt_metadata_mem);
+
+	ok(fnCompare(old_prep_stmt_backend_mem, new_prep_stmt_backend_mem, prep_stmt_backend_mem_comp),
+		"Memory usage check [%d]. 'prepare_statement_backend_memory':[%lu] [%lu]", prep_stmt_backend_mem_comp,
+		old_prep_stmt_backend_mem, new_prep_stmt_backend_mem);
+
+	mysql_stmt_close(stmt);
+	return EXIT_SUCCESS;
+}
+
+int main(int argc, char** argv) {
+
+	CommandLine cl;
+
+	if (cl.getEnv()) {
+		diag("Failed to get the required environmental variables.");
+		return -1;
+	}
+
+	plan(4 * // query
+		 3 // checks
+	);
+
+	// Initialize Admin connection
+	MYSQL* proxysql_admin = mysql_init(NULL);
+	if (!proxysql_admin) {
+		fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin));
+		return -1;
+	}
+	// Connnect to ProxySQL Admin
+	if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) {
+		fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin));
+		return exit_status();
+	}
+
+	// Initialize ProxySQL connection
+	MYSQL* proxysql = mysql_init(NULL);
+	if (!proxysql) {
+		fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql));
+		return exit_status();
+	}
+
+	// Connect to ProxySQL
+	if (!mysql_real_connect(proxysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) {
+		fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql));
+		return exit_status();
+	}
+
+	if (check_prepare_statement_mem_usage(proxysql_admin, proxysql, "SELECT 1", kGreaterThan, (kGreaterThan | kEqual)) == EXIT_FAILURE)
+		goto __cleanup;
+
+	if (check_prepare_statement_mem_usage(proxysql_admin, proxysql, "SELECT 2", kGreaterThan, (kGreaterThan | kEqual)) == EXIT_FAILURE)
+		goto __cleanup;
+
+	if (check_prepare_statement_mem_usage(proxysql_admin, proxysql, "SELECT 1", kGreaterThan, kEqual) == EXIT_FAILURE)
+		goto __cleanup;
+
+	if (check_prepare_statement_mem_usage(proxysql_admin, proxysql, "SELECT 2", kGreaterThan, kEqual) == EXIT_FAILURE)
+		goto __cleanup;
+
+__cleanup:
+	mysql_close(proxysql);
+	mysql_close(proxysql_admin);
+
+	return exit_status();
+}