HBASE-26798 improve balancer visibility

hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java

@@ -132,6 +132,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   protected static final String COST_FUNCTIONS_COST_FUNCTIONS_KEY =
           "hbase.master.balancer.stochastic.additionalCostFunctions";
   public static final String OVERALL_COST_FUNCTION_NAME = "Overall";
+  public static final String WIGHTED_IMBALANCE_COST_NAME = "Imbalance";
 
   protected static final Random RANDOM = new Random(System.currentTimeMillis());
   private static final Logger LOG = LoggerFactory.getLogger(StochasticLoadBalancer.class);
@@ -323,7 +324,7 @@ private void updateBalancerTableLoadInfo(TableName tableName,
     initCosts(cluster);
     curOverallCost = computeCost(cluster, Double.MAX_VALUE);
     System.arraycopy(tempFunctionCosts, 0, curFunctionCosts, 0, curFunctionCosts.length);
-    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);
+    updateStochasticCosts(tableName, curOverallCost, curOverallCost/sumMultiplier, curFunctionCosts);
   }
 
   @Override
@@ -424,7 +425,7 @@ protected boolean needsBalance(TableName tableName, Cluster cluster) {
   @InterfaceAudience.Private
   Cluster.Action nextAction(Cluster cluster) {
     return candidateGenerators.get(RANDOM.nextInt(candidateGenerators.size()))
-            .generate(cluster);
+          .generate(cluster);
   }
 
   /**
@@ -480,7 +481,7 @@ public synchronized List<RegionPlan> balanceTable(TableName tableName, Map<Serve
     double currentCost = computeCost(cluster, Double.MAX_VALUE);
     curOverallCost = currentCost;
     System.arraycopy(tempFunctionCosts, 0, curFunctionCosts, 0, curFunctionCosts.length);
-    updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);
+    updateStochasticCosts(tableName, curOverallCost, currentCost / sumMultiplier, curFunctionCosts);
     double initCost = currentCost;
     double newCost;
 
@@ -505,8 +506,8 @@ public synchronized List<RegionPlan> balanceTable(TableName tableName, Map<Serve
       }
     }
     LOG.info("Start StochasticLoadBalancer.balancer, initial weighted average imbalance={}," +
-        " functionCost={} computedMaxSteps={}",
-      currentCost / sumMultiplier, functionCost(), computedMaxSteps);
+        " sumMultiplier={} sumCost={} functionCost={} computedMaxSteps={}",
+      currentCost / sumMultiplier, sumMultiplier, currentCost, functionCost(), computedMaxSteps);
 
     final String initFunctionTotalCosts = totalCostsPerFunc();
     // Perform a stochastic walk to see if we can get a good fit.
@@ -532,6 +533,9 @@ public synchronized List<RegionPlan> balanceTable(TableName tableName, Map<Serve
         curOverallCost = currentCost;
         System.arraycopy(tempFunctionCosts, 0, curFunctionCosts, 0, curFunctionCosts.length);
       } else {
+        //LOG hidden move cost if greatly impacting ability to find better plan
+        logMoveCosts(newCost, tempFunctionCosts);
+
         // Put things back the way they were before.
         // TODO: undo by remembering old values
         Action undoAction = action.undoAction();
@@ -548,7 +552,7 @@ public synchronized List<RegionPlan> balanceTable(TableName tableName, Map<Serve
     metricsBalancer.balanceCluster(endTime - startTime);
 
     if (initCost > currentCost) {
-      updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);
+      updateStochasticCosts(tableName, curOverallCost, currentCost / sumMultiplier, curFunctionCosts);
       plans = createRegionPlans(cluster);
       LOG.info("Finished computing new moving plan. Computation took {} ms" +
           " to try {} different iterations.  Found a solution that moves " +
@@ -559,9 +563,11 @@ public synchronized List<RegionPlan> balanceTable(TableName tableName, Map<Serve
       sendRegionPlansToRingBuffer(plans, currentCost, initCost, initFunctionTotalCosts, step);
       return plans;
     }
+
     LOG.info("Could not find a better moving plan.  Tried {} different configurations in "
-        + "{} ms, and did not find anything with an imbalance score less than {}", step,
+        + "{} ms, and did not find anything with an imbalance score less than {}.", step,
       endTime - startTime, initCost / sumMultiplier);
+
     return null;
   }
 
@@ -603,10 +609,24 @@ private void sendRegionPlansToRingBuffer(List<RegionPlan> plans, double currentC
     }
   }
 
+  private void logMoveCosts(double currentCost, double [] currentFunctionCosts){
+    for (int i = 0; i < costFunctions.size(); i++) {
+      CostFunction costFunction = costFunctions.get(i);
+      if (costFunction instanceof MoveCostFunction){
+        String costFunctionName = costFunction.getClass().getSimpleName();
+        double moveCost = currentFunctionCosts[i];
+        double costPercent = (currentCost == 0) ? 0 : ( moveCost/ currentCost);
+        if (costPercent > .50 ){
+          LOG.info("Move cost greatly impacting overall cost of improvement plan. currentCost={} percentageOfCost={}. Consider lowering moveCost multiplier.",moveCost, costPercent );
+        }
+      }
+
+    }
+  }
   /**
    * update costs to JMX
    */
-  private void updateStochasticCosts(TableName tableName, double overall, double[] subCosts) {
+  private void updateStochasticCosts(TableName tableName, double overall, double weightedImbalance, double[] subCosts) {
     if (tableName == null) {
       return;
     }
@@ -618,13 +638,17 @@ private void updateStochasticCosts(TableName tableName, double overall, double[]
       balancer.updateStochasticCost(tableName.getNameAsString(),
         OVERALL_COST_FUNCTION_NAME, "Overall cost", overall);
 
-      // each cost function
+      // weighted imbalance
+      balancer.updateStochasticCost(tableName.getNameAsString(),
+        WIGHTED_IMBALANCE_COST_NAME, "Weighted imbalance", weightedImbalance);
+
+      // each cost function as a percent of overall cost
       for (int i = 0; i < costFunctions.size(); i++) {
         CostFunction costFunction = costFunctions.get(i);
         String costFunctionName = costFunction.getClass().getSimpleName();
         double costPercent = (overall == 0) ? 0 : (subCosts[i] / overall);
         // TODO: cost function may need a specific description
-        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName,
+        balancer.updateStochasticCost(tableName.getNameAsString(), costFunctionName + "Percentage",
           "The percent of " + costFunctionName, costPercent);
       }
     }
@@ -967,6 +991,7 @@ protected double cost() {
       // Don't let this single balance move more than the max moves.
       // This allows better scaling to accurately represent the actual cost of a move.
       if (moveCost > maxMoves) {
+        LOG.info("Calculated moves exceed maxMoves {}. Greatly increasing move cost. ", maxMoves);
         return 1000000;   // return a number much greater than any of the other cost
       }