Add MySQL compression level variable and change default to compression level 3

include/MySQL_Thread.h

@@ -470,6 +470,7 @@ class MySQL_Threads_Handler
 		bool parse_failure_logs_digest;
 		bool default_reconnect;
 		bool have_compress;
+		int protocol_compression_level;
 		bool have_ssl;
 		bool multiplexing;
 //		bool stmt_multiplexing;

include/proxysql_structs.h

@@ -1167,6 +1167,7 @@ __thread int mysql_thread___poll_timeout;
 __thread int mysql_thread___poll_timeout_on_failure;
 __thread bool mysql_thread___connection_warming;
 __thread bool mysql_thread___have_compress;
+__thread int mysql_thread___protocol_compression_level;
 __thread bool mysql_thread___have_ssl;
 __thread bool mysql_thread___multiplexing;
 __thread bool mysql_thread___log_unhealthy_connections;
@@ -1463,6 +1464,7 @@ extern __thread int mysql_thread___poll_timeout;
 extern __thread int mysql_thread___poll_timeout_on_failure;
 extern __thread bool mysql_thread___connection_warming;
 extern __thread bool mysql_thread___have_compress;
+extern __thread int mysql_thread___protocol_compression_level;
 extern __thread bool mysql_thread___have_ssl;
 extern __thread bool mysql_thread___multiplexing;
 extern __thread bool mysql_thread___log_unhealthy_connections;

lib/MySQL_Thread.cpp

@@ -504,6 +504,7 @@ static char * mysql_thread_variables_names[]= {
 	(char *)"handle_warnings",
 	(char *)"evaluate_replication_lag_on_servers_load",
 	(char *)"proxy_protocol_networks",
+	(char *)"protocol_compression_level",
 	NULL
 };
 
@@ -1137,6 +1138,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() {
 	variables.enable_load_data_local_infile=false;
 	variables.log_mysql_warnings_enabled=false;
 	variables.data_packets_history_size=0;
+	variables.protocol_compression_level=3;
 	// status variables
 	status_variables.mirror_sessions_current=0;
 	__global_MySQL_Thread_Variables_version=1;
@@ -2264,6 +2266,7 @@ char ** MySQL_Threads_Handler::get_variables_list() {
 		VariablesPointers_int["client_host_error_counts"]      = make_tuple(&variables.client_host_error_counts,      0,      1024*1024, false);
 		VariablesPointers_int["handle_warnings"]			   = make_tuple(&variables.handle_warnings,				  0,			  1, false);
 		VariablesPointers_int["evaluate_replication_lag_on_servers_load"] = make_tuple(&variables.evaluate_replication_lag_on_servers_load, 0, 1, false);
+		VariablesPointers_int["protocol_compression_level"]    = make_tuple(&variables.protocol_compression_level,    0,      9,         false);
 
 		// logs
 		VariablesPointers_int["auditlog_filesize"]     = make_tuple(&variables.auditlog_filesize,    1024*1024, 1*1024*1024*1024, false);
@@ -4180,6 +4183,7 @@ void MySQL_Thread::refresh_variables() {
 	REFRESH_VARIABLE_INT(poll_timeout);
 	REFRESH_VARIABLE_INT(poll_timeout_on_failure);
 	REFRESH_VARIABLE_BOOL(have_compress);
+	REFRESH_VARIABLE_INT(protocol_compression_level);
 	REFRESH_VARIABLE_BOOL(have_ssl);
 	REFRESH_VARIABLE_BOOL(multiplexing);
 	REFRESH_VARIABLE_BOOL(log_unhealthy_connections);
@@ -4264,6 +4268,8 @@ MySQL_Thread::MySQL_Thread() {
 	mysql_thread___ssl_p2s_crl=NULL;
 	mysql_thread___ssl_p2s_crlpath=NULL;
 
+	mysql_thread___protocol_compression_level=3;
+
 	last_maintenance_time=0;
 	last_move_to_idle_thread_time=0;
 	maintenance_loop=true;

lib/mysql_data_stream.cpp

@@ -1321,7 +1321,7 @@ void MySQL_Data_Stream::generate_compressed_packet() {
 			total_size+=p2.size;
 			l_free(p2.size,p2.ptr);
 		}
-		int rc=compress(dest, &destLen, source, sourceLen);
+		int rc=compress2(dest, &destLen, source, sourceLen, GloMTH->variables.protocol_compression_level);
 		assert(rc==Z_OK);
 		l_free(total_size, source);
 		queueOUT.pkt.size=destLen+7;
@@ -1353,9 +1353,9 @@ void MySQL_Data_Stream::generate_compressed_packet() {
 		dest1=(Bytef *)malloc(destLen1+7);
 		destLen2=len2*120/100+12;
 		dest2=(Bytef *)malloc(destLen2+7);
-		rc=compress(dest1+7, &destLen1, (const unsigned char *)p2.ptr, len1);
+		rc=compress2(dest1+7, &destLen1, (const unsigned char *)p2.ptr, len1, GloMTH->variables.protocol_compression_level);
 		assert(rc==Z_OK);
-		rc=compress(dest2+7, &destLen2, (const unsigned char *)p2.ptr+len1, len2);
+		rc=compress2(dest2+7, &destLen2, (const unsigned char *)p2.ptr+len1, len2, GloMTH->variables.protocol_compression_level);
 		assert(rc==Z_OK);
 
 		hdr.pkt_length=destLen1;

test/tap/groups/groups.json

@@ -180,5 +180,6 @@
   "eof_fast_forward-t" : [ "default", "mysql-auto_increment_delay_multiplex=0", "mysql-multiplexing=false", "mysql-query_digests=0", "mysql-query_digests_keep_comment=1" ],
   "eof_mixed_flags_queries-t" : [ "default", "mysql-auto_increment_delay_multiplex=0", "mysql-multiplexing=false", "mysql-query_digests=0", "mysql-query_digests_keep_comment=1" ],
   "eof_packet_mixed_queries-t" : [ "default", "mysql-auto_increment_delay_multiplex=0", "mysql-multiplexing=false", "mysql-query_digests=0", "mysql-query_digests_keep_comment=1" ],
-  "ok_packet_mixed_queries-t" : [ "default", "mysql-auto_increment_delay_multiplex=0", "mysql-multiplexing=false", "mysql-query_digests=0", "mysql-query_digests_keep_comment=1" ]
+  "ok_packet_mixed_queries-t" : [ "default", "mysql-auto_increment_delay_multiplex=0", "mysql-multiplexing=false", "mysql-query_digests=0", "mysql-query_digests_keep_comment=1" ],
+  "mysql-protocol_compression_level-t" : [ "default", "mysql-auto_increment_delay_multiplex=0", "mysql-multiplexing=false", "mysql-query_digests=0", "mysql-query_digests_keep_comment=1" ]
 }

test/tap/tests/mysql-protocol_compression_level-t.cpp

@@ -0,0 +1,184 @@
+#include <cstdlib>
+#include <cstdio>
+#include <cstring>
+#include <unistd.h>
+
+#include <string>
+#include <sstream>
+#include <fstream>
+#include "mysql.h"
+
+#include "tap.h"
+#include "command_line.h"
+#include "utils.h"
+
+inline unsigned long long monotonic_time() {
+  struct timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+  return (((unsigned long long) ts.tv_sec) * 1000000) + (ts.tv_nsec / 1000);
+}
+
+unsigned long calculate_query_execution_time(MYSQL* mysql, const std::string& query) {
+	MYSQL_RES *res;
+	MYSQL_ROW row;
+	unsigned long long begin = monotonic_time();
+	unsigned long long row_count = 0;
+	MYSQL_QUERY(mysql, query.c_str());
+	res = mysql_use_result(mysql);
+	unsigned long long num_rows = mysql_num_rows(res);
+	unsigned int num_fields = mysql_num_fields(res);
+	while ((row = mysql_fetch_row(res))) {
+		for (unsigned int i = 1; i < num_fields; i++) {
+			char *field = row[i];
+		}
+		row_count++;
+	}
+	mysql_free_result(res);	
+	unsigned long long end = monotonic_time();
+	fprintf(stderr, "Row count: %lld\n", row_count);
+	return (end - begin);
+}
+
+void set_compression_level(MYSQL* proxysql_admin, std::string level) {
+	std::string query = "UPDATE global_variables SET variable_value = '" + level + "' WHERE variable_name = 'mysql-protocol_compression_level';";
+	mysql_query(proxysql_admin, query.c_str());
+	query = "LOAD MYSQL VARIABLES TO RUNTIME;";
+	mysql_query(proxysql_admin, query.c_str());
+}
+
+int32_t get_compression_level(MYSQL* proxysql_admin) {
+	int32_t compression_level = -1;
+	const uint32_t SERVERS_COUNT = 10;	
+
+	int err = mysql_query(proxysql_admin, "SELECT * FROM global_variables WHERE variable_name='mysql-protocol_compression_level'");
+	if (err != EXIT_SUCCESS) {
+		diag(
+			"Query for retrieving value of 'mysql-protocol_compression_level' failed with error: (%d, %s)",
+			mysql_errno(proxysql_admin), mysql_error(proxysql_admin)
+		);
+		return compression_level;
+	}
+
+	MYSQL_RES* res =  mysql_store_result(proxysql_admin);
+	if (res != nullptr) {
+		int num_rows = mysql_num_rows(res);
+		MYSQL_ROW row = mysql_fetch_row(res);
+
+		if (num_rows && row != nullptr) {
+			char* endptr = nullptr;
+			compression_level = strtol(row[1], &endptr, SERVERS_COUNT);
+		}
+	}
+	mysql_free_result(res);
+	return compression_level;
+}
+
+MYSQL* initilize_mysql_connection(char* host, char* username, char* password, int port, bool compression) {
+	MYSQL* mysql = mysql_init(NULL);
+	if (!mysql)
+		return nullptr;
+
+	fprintf(stderr, "MySQL connection details: %s %s %d\n", username, password, port);
+	if (!mysql_real_connect(mysql, host, username, password, NULL, port, NULL, 0)) {
+	    fprintf(stderr, "Failed to connect to database: Error: %s\n",
+	              mysql_error(mysql));
+		mysql_close(mysql);
+		return nullptr;
+	}
+	if (compression) {
+		if (mysql_options(mysql, MYSQL_OPT_COMPRESS, nullptr) != 0) {
+			fprintf(stderr, "Failed to set mysql compression option: Error: %s\n",
+					mysql_error(mysql));
+			mysql_close(mysql);
+			return nullptr;
+		}
+	}
+	return mysql;
+}
+
+int main(int argc, char** argv) {
+
+	CommandLine cl;
+
+	if(cl.getEnv())
+		return exit_status();
+
+	plan(5);
+
+	// ProxySQL connection without compression
+	MYSQL* proxysql = initilize_mysql_connection(cl.host, cl.username, cl.password, cl.port, false);
+	if (!proxysql) {
+		return exit_status();
+	}
+
+	// ProxySQL connection with compression
+	MYSQL* proxysql_compression = initilize_mysql_connection(cl.host, cl.username, cl.password, cl.port, true);
+	if (!proxysql_compression) {
+		return exit_status();
+	}
+
+	// MySQL connection with compression
+	MYSQL* mysql_compression = initilize_mysql_connection(cl.host, cl.username, cl.password, cl.mysql_port, true);
+	if (!mysql_compression) {
+		return exit_status();
+	}
+
+	// ProxySQL admin connection
+	MYSQL* proxysql_admin = initilize_mysql_connection(cl.host, cl.admin_username, cl.admin_password, cl.admin_port, false);
+	if (!proxysql_admin) {
+		return exit_status();
+	}
+
+	MYSQL_QUERY(proxysql, "CREATE DATABASE IF NOT EXISTS test");
+	MYSQL_QUERY(proxysql, "DROP TABLE IF EXISTS test.sbtest1");
+
+	mysql_query(proxysql, "CREATE TABLE IF NOT EXISTS test.sbtest1 (id INT UNSIGNED NOT NULL AUTO_INCREMENT, k INT UNSIGNED NOT NULL DEFAULT 0, c CHAR(120) NOT NULL DEFAULT '', pad CHAR(60) NOT NULL DEFAULT '', PRIMARY KEY (id), KEY k_1 (k));");
+
+	MYSQL_QUERY(proxysql, "USE test");
+
+	for (int i = 0; i < 100; i++) {
+		MYSQL_QUERY(proxysql, "INSERT INTO sbtest1 (k, c, pad) SELECT FLOOR(RAND() * 10000), REPEAT('a', 120), REPEAT('b', 60) FROM information_schema.tables LIMIT 1000;");
+	}
+
+	std::string query = "SELECT t1.id id1, t1.k k1, t1.c c1, t1.pad pad1, t2.id id2, t2.k k2, t2.c c2, t2.pad pad2 FROM test.sbtest1 t1 JOIN test.sbtest1 t2 LIMIT 90000000";
+
+	unsigned long time_proxy = calculate_query_execution_time(proxysql, query);
+	diag("Time taken for query with proxysql without compression: %ld", time_proxy);
+
+	unsigned long time_proxy_compressed = calculate_query_execution_time(proxysql_compression, query);
+	diag("Time taken for query with proxysql with compression: %ld", time_proxy_compressed);
+
+	unsigned long diff = abs(time_proxy - time_proxy_compressed);
+	int performance_diff = (diff * 100) / time_proxy;
+
+	ok((performance_diff < 10), "proxysql with compression performed well compared to without compression. Performance difference: %d percentage", performance_diff);
+
+	unsigned long time_mysql_compressed = calculate_query_execution_time(mysql_compression, query);
+	diag("Time taken for query with mysql with compression: %ld", time_mysql_compressed);
+
+	diff = abs(time_mysql_compressed - time_proxy_compressed);
+	performance_diff = (diff * 100) / time_mysql_compressed;
+
+	ok((performance_diff < 20), "proxysql with compression performed well compared to mysql with compression. Performance difference: %d percentage", performance_diff);
+
+	int32_t compression_level = get_compression_level(proxysql_admin);
+	ok(compression_level == 3, "Default compression level is correct: %d", compression_level);
+
+	set_compression_level(proxysql_admin, "8");
+	compression_level = get_compression_level(proxysql_admin);
+	ok(compression_level == 8, "Compression level set correctly: %d", compression_level);
+
+	time_proxy_compressed = calculate_query_execution_time(proxysql_compression, query);
+	diag("Time taken for query with proxysql with compression level 8: %ld", time_proxy_compressed);
+
+	set_compression_level(proxysql_admin, "3");
+	compression_level = get_compression_level(proxysql_admin);
+	ok(compression_level == 3, "Compression level set correctly: %d", compression_level);
+
+	mysql_close(proxysql);
+	mysql_close(proxysql_compression);
+	mysql_close(mysql_compression);
+	mysql_close(proxysql_admin);
+
+	return exit_status();
+}

test/tap/tests/proxysql_reference_select_config_file.cnf

@@ -222,6 +222,7 @@ mysql_variables =
 	use_tcp_keepalive="mysql"
 	verbose_query_error="mysql"
 	wait_timeout="mysql"
+	compression_level="mysql"	
 }
 mysql_users:
 (