From d659a3b86686a8d64643727fa1e6a7fa8cd48b83 Mon Sep 17 00:00:00 2001 From: "juke.mini666" Date: Wed, 31 Oct 2018 08:50:55 +0900 Subject: [PATCH 1/4] Added an empty region merge function for Phoenix's salting table. - To enable function, set the '--phoenix-salting-table' option on the command line. --- .../java/com/kakao/hbase/common/Args.java | 1 + .../java/com/kakao/hbase/ManagerArgs.java | 1 + .../kakao/hbase/manager/command/Merge.java | 31 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/hbase0.98/hbase-common-0.98/src/main/java/com/kakao/hbase/common/Args.java b/hbase0.98/hbase-common-0.98/src/main/java/com/kakao/hbase/common/Args.java index 7af2d4a..329f920 100644 --- a/hbase0.98/hbase-common-0.98/src/main/java/com/kakao/hbase/common/Args.java +++ b/hbase0.98/hbase-common-0.98/src/main/java/com/kakao/hbase/common/Args.java @@ -64,6 +64,7 @@ public abstract class Args { public static final String OPTION_INTERACTIVE = "interactive"; public static final String OPTION_CONF = "conf"; public static final String OPTION_CONF_SHORT = "c"; + public static final String OPTION_PHOENIX = "phoenix-salting-table"; public static final String INVALID_ARGUMENTS = "Invalid arguments"; public static final String ALL_TABLES = ""; diff --git a/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/ManagerArgs.java b/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/ManagerArgs.java index d855713..083e917 100644 --- a/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/ManagerArgs.java +++ b/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/ManagerArgs.java @@ -40,6 +40,7 @@ protected OptionParser createOptionParser() { optionParser.accepts(OPTION_LOCALITY_THRESHOLD).withRequiredArg().ofType(Double.class); optionParser.accepts(OPTION_CF).withRequiredArg().ofType(String.class); optionParser.accepts(OPTION_INTERACTIVE); + optionParser.accepts(OPTION_PHOENIX); return optionParser; } } diff --git a/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java b/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java index cb4c7fc..4874e43 100644 --- a/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java +++ b/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java @@ -28,12 +28,14 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.util.Bytes; import java.io.IOException; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; public class Merge implements Command { private final HBaseAdmin admin; @@ -43,6 +45,7 @@ public class Merge implements Command { private final HConnection connection; private boolean proceed = false; private boolean test = false; + private boolean isPhoenixSaltingTable; public Merge(HBaseAdmin admin, Args args) throws IOException { if (args.getOptionSet().nonOptionArguments().size() < 2 @@ -57,6 +60,8 @@ public Merge(HBaseAdmin admin, Args args) throws IOException { this.connection = HConnectionManager.createConnection(admin.getConfiguration()); tableNameSet = Util.parseTableSet(admin, args); + + if (args.has(Args.OPTION_PHOENIX)) this.isPhoenixSaltingTable = true; } @SuppressWarnings("unused") @@ -213,6 +218,27 @@ private int getSize(TableInfo tableInfo, HRegionInfo region) { return Integer.MAX_VALUE; } + private boolean isRegionBoundaryOfPhoenixSaltingTable(HRegionInfo regionInfo) { + byte[] endKey = regionInfo.getEndKey(); + boolean boundaryRegionForPhoenix = true; + + if (endKey.length > 0) { + for (int i = 1, limit = endKey.length; i < limit; i++) { + if (endKey[i] != 0) { + boundaryRegionForPhoenix = false; + break; + } + } + } + + if (boundaryRegionForPhoenix) { + Util.printVerboseMessage(args, regionInfo.getEncodedName() + " is boundary region of phoenix : " + + Bytes.toStringBinary(regionInfo.getStartKey()) + " ~ " + Bytes.toStringBinary(regionInfo.getEndKey())); + } + + return boundaryRegionForPhoenix; + } + private void emptyFast(TableInfo tableInfo) throws Exception { long timestampPrev; boolean merged; @@ -243,6 +269,11 @@ private void emptyFast(TableInfo tableInfo) throws Exception { for (int i = 0; i < emptyRegions.size(); i++) { HRegionInfo regionA = emptyRegions.get(i); + // 첫번째 리전의 endKey가 피닉스 솔팅 테이블의 리전 바운더리가 아니여야 한다. + if (isPhoenixSaltingTable && isRegionBoundaryOfPhoenixSaltingTable(regionA)) { + continue; + } + if (i != emptyRegions.size() - 1) { HRegionInfo regionB = emptyRegions.get(i + 1); From 566f4df178b313f49cb2a6c97b5cb2dc92e7f613 Mon Sep 17 00:00:00 2001 From: "juke.mini666" Date: Thu, 1 Nov 2018 07:36:54 +0900 Subject: [PATCH 2/4] Fixed compile error - Compare the number type object directly to an integer in the if statement. --- .../main/java/com/kakao/hbase/common/LoadEntry.java | 10 +++++----- .../src/main/java/com/kakao/hbase/stat/load/Load.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hbase0.98/hbase-common-0.98/src/main/java/com/kakao/hbase/common/LoadEntry.java b/hbase0.98/hbase-common-0.98/src/main/java/com/kakao/hbase/common/LoadEntry.java index f5fed24..fe05b1e 100644 --- a/hbase0.98/hbase-common-0.98/src/main/java/com/kakao/hbase/common/LoadEntry.java +++ b/hbase0.98/hbase-common-0.98/src/main/java/com/kakao/hbase/common/LoadEntry.java @@ -400,7 +400,7 @@ private static Number diffRatioNumber(Number one, Number two) { if (one == null) { if (two == null) return null; else { - if (two == 0) { + if (two.intValue() == 0) { return RatioNumber.ZERO.doubleValue(); } else { return -two.doubleValue(); @@ -409,7 +409,7 @@ private static Number diffRatioNumber(Number one, Number two) { } else { if (two == null) return one.doubleValue(); else { - if (two == 0) { + if (two.intValue() == 0) { return one.doubleValue(); } else { return one.doubleValue() - two.doubleValue(); @@ -432,7 +432,7 @@ private static Number addRatioNumber(Number one, Number two) { if (one == null) { if (two == null) return null; else { - if (two == 0) { + if (two.intValue() == 0) { return RatioNumber.ZERO; } else { return two; @@ -441,9 +441,9 @@ private static Number addRatioNumber(Number one, Number two) { } else { if (two == null) return one; else { - if (one == 0) { + if (one.intValue() == 0) { return two; - } else if (two == 0) { + } else if (two.intValue() == 0) { return one; } else { return ((RatioNumber) one).add((RatioNumber) two); diff --git a/hbase0.98/hbase-table-stat-0.98/src/main/java/com/kakao/hbase/stat/load/Load.java b/hbase0.98/hbase-table-stat-0.98/src/main/java/com/kakao/hbase/stat/load/Load.java index 731b9ca..3758589 100644 --- a/hbase0.98/hbase-table-stat-0.98/src/main/java/com/kakao/hbase/stat/load/Load.java +++ b/hbase0.98/hbase-table-stat-0.98/src/main/java/com/kakao/hbase/stat/load/Load.java @@ -145,7 +145,7 @@ public boolean isRecordChanged(Level level) { } public boolean isSummaryChanged(LoadEntry loadEntry) { - return summaryChangeMap.get(loadEntry) == ChangeState.changed.ordinal(); + return summaryChangeMap.get(loadEntry).intValue() == ChangeState.changed.ordinal(); } public boolean isDiffFromStart() { From caf45ccf9d667b9bfcc43907772f71b03a591071 Mon Sep 17 00:00:00 2001 From: "terence.yoo" Date: Thu, 1 Nov 2018 16:02:50 +0900 Subject: [PATCH 3/4] remove unused code --- .../src/main/java/com/kakao/hbase/manager/command/Merge.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java b/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java index 4874e43..fd3440f 100644 --- a/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java +++ b/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java @@ -35,7 +35,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.stream.IntStream; public class Merge implements Command { private final HBaseAdmin admin; From 31a6795363a9d40fd6d5ffaad18781a47d4ed13a Mon Sep 17 00:00:00 2001 From: "terence.yoo" Date: Thu, 1 Nov 2018 16:08:43 +0900 Subject: [PATCH 4/4] add usage --- .../src/main/java/com/kakao/hbase/manager/command/Merge.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java b/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java index fd3440f..382b55b 100644 --- a/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java +++ b/hbase0.98/hbase-manager-0.98/src/main/java/com/kakao/hbase/manager/command/Merge.java @@ -74,6 +74,7 @@ public static String usage() { + " empty - Merge all empty regions.\n" + " options:\n" + " --" + Args.OPTION_MAX_ITERATION + " - Set max iteration.\n" + + " --" + Args.OPTION_PHOENIX + " - Set if the table to be merged is phoenix salted table.\n" + Args.commonUsage(); }