From fd8b68be1a18e5ccafc470ed301b4d62aa6b7ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20B=C4=85czkowski?= Date: Mon, 13 Jun 2022 00:58:08 +0200 Subject: [PATCH 1/2] Add re-prepare statement throttle. Limits number of reprepares done within single SpeculativeExecution. In case of going over the limit sets an exception and lets RequestHandler act accordingly to RetryPolicy. Should help break endless loops of repreparing statements. --- .../datastax/driver/core/RequestHandler.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/driver-core/src/main/java/com/datastax/driver/core/RequestHandler.java b/driver-core/src/main/java/com/datastax/driver/core/RequestHandler.java index a8a05f093f5..4dae0722896 100644 --- a/driver-core/src/main/java/com/datastax/driver/core/RequestHandler.java +++ b/driver-core/src/main/java/com/datastax/driver/core/RequestHandler.java @@ -71,6 +71,8 @@ class RequestHandler { private static final boolean HOST_METRICS_ENABLED = Boolean.getBoolean("com.datastax.driver.HOST_METRICS_ENABLED"); private static final QueryLogger QUERY_LOGGER = QueryLogger.builder().build(); + + private static final int SPECULATIVE_EXECUTION_MAX_REPREPARES = 5; static final String DISABLE_QUERY_WARNING_LOGS = "com.datastax.driver.DISABLE_QUERY_WARNING_LOGS"; final String id; @@ -381,6 +383,10 @@ class SpeculativeExecution implements Connection.ResponseCallback { // This is incremented by one writer at a time, so volatile is good enough. private volatile int retriesByPolicy; + // In rare cases we can enter seemingly infinite loop of repreparing a statement. + // This counter is used for breaking such stalemates. + private volatile int retriesByUnprepared; + private volatile Connection.ResponseHandler connectionHandler; SpeculativeExecution(Message.Request request, int position) { @@ -825,6 +831,21 @@ public void onSet( toPrepare.getQueryString())); } + if(retriesByUnprepared > SPECULATIVE_EXECUTION_MAX_REPREPARES) { + connection.release(); + String msg = String.format( + "Statement %s (%s) is not prepared on %s and reprepare threshold (%d) has been reached for this execution. " + + "This might have been caused by driver misuse or the cluster. Check cluster logs for the reason of possible prepared statement cache evictions.", + toPrepare.getQueryString(), + id, + toPrepare.getQueryKeyspace(), + SPECULATIVE_EXECUTION_MAX_REPREPARES); + + logger.error(msg); + setFinalException(connection, new DriverInternalError(msg)); + return; + } + logger.info( "Query {} is not prepared on {}, preparing before retrying executing. " + "Seeing this message a few times is fine, but seeing it a lot may be source of performance problems", @@ -832,6 +853,7 @@ public void onSet( toPrepare.getQueryKeyspace(), connection.endPoint); + retriesByUnprepared++; write(connection, prepareAndRetry(toPrepare.getQueryString())); // we're done for now, the prepareAndRetry callback will handle the rest return; From ae13fd20a4136608b72d2704e1fa0e0a5b3b8007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20B=C4=85czkowski?= Date: Thu, 16 Jun 2022 12:58:40 +0200 Subject: [PATCH 2/2] Formatting. --- .../java/com/datastax/driver/core/RequestHandler.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/driver-core/src/main/java/com/datastax/driver/core/RequestHandler.java b/driver-core/src/main/java/com/datastax/driver/core/RequestHandler.java index 4dae0722896..b8e6ef886c8 100644 --- a/driver-core/src/main/java/com/datastax/driver/core/RequestHandler.java +++ b/driver-core/src/main/java/com/datastax/driver/core/RequestHandler.java @@ -831,11 +831,12 @@ public void onSet( toPrepare.getQueryString())); } - if(retriesByUnprepared > SPECULATIVE_EXECUTION_MAX_REPREPARES) { + if (retriesByUnprepared > SPECULATIVE_EXECUTION_MAX_REPREPARES) { connection.release(); - String msg = String.format( - "Statement %s (%s) is not prepared on %s and reprepare threshold (%d) has been reached for this execution. " + - "This might have been caused by driver misuse or the cluster. Check cluster logs for the reason of possible prepared statement cache evictions.", + String msg = + String.format( + "Statement %s (%s) is not prepared on %s and reprepare threshold (%d) has been reached for this execution. " + + "This might have been caused by driver misuse or the cluster. Check cluster logs for the reason of possible prepared statement cache evictions.", toPrepare.getQueryString(), id, toPrepare.getQueryKeyspace(),