Merge branch 'staging'

atopcalcite/.settings/org.eclipse.core.resources.prefs

@@ -1,6 +1,5 @@
 eclipse.preferences.version=1
 encoding//src/main/java=UTF-8
 encoding//src/main/resources=UTF-8
-encoding//src/test/java=UTF-8
 encoding//src/test/resources=UTF-8
 encoding/<project>=UTF-8

atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java

@@ -1,5 +1,5 @@
 /*
- * OVERRIDE POINTS:
+ * OVERRIDE POINT:
  * - divide(BigDecimal,BigDecimal), was `b0.divide(b1)`, now `b0.divide(b1, MathContext.DECIMAL64);` 
  */
 

atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java

@@ -1,7 +1,8 @@
 /*
- * OVERRIDE POINTS:
+ * OVERRIDE POINT:
  * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
  * - isTrimUnusedFields(), override to false
+ * - AggConverter.visit(SqlCall), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
  */
 
 /*
@@ -4430,7 +4431,7 @@ public Void visit(SqlCall call) {
             // special case for COUNT(*):  delete the *
             if (operand instanceof SqlIdentifier) {
               SqlIdentifier id = (SqlIdentifier) operand;
-              if (id.isStar()) {
+              if (id.isStar() || isSimpleCount(call)) { // OVERRIDE POINT, was just `id.isStar()`
                 assert call.operandCount() == 1;
                 assert args.isEmpty();
                 break;
@@ -4487,6 +4488,18 @@ public Void visit(SqlCall call) {
       return null;
     }
 
+    // OVERRIDE POINT
+    private boolean isSimpleCount(SqlCall call) {
+        if (call.getOperator().isName("COUNT") && call.operandCount() == 1) {
+            final SqlNode parm = call.operand(0);
+            if ((parm instanceof SqlIdentifier || parm instanceof SqlNumericLiteral) //
+                    && call.getFunctionQuantifier() == null) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private int lookupOrCreateGroupExpr(RexNode expr) {
       for (int i = 0; i < convertedInputExprs.size(); i++) {
         RexNode convertedInputExpr = convertedInputExprs.get(i);

common/src/main/java/com/kylinolap/common/KylinConfig.java

@@ -189,7 +189,10 @@ private static UriType decideUriType(String metaUri) {
 
         try {
             File file = new File(metaUri);
-            if (file.exists()) {
+            if (file.exists() || metaUri.contains("/")) {
+                if (file.exists() == false) {
+                    file.mkdirs();
+                }
                 if (file.isDirectory()) {
                     return UriType.LOCAL_FOLDER;
                 } else if (file.isFile()) {
@@ -198,6 +201,8 @@ private static UriType decideUriType(String metaUri) {
                     } else {
                         throw new IllegalStateException("Metadata uri : " + metaUri + " is a local file but not kylin.properties");
                     }
+                } else {
+                    throw new IllegalStateException("Metadata uri : " + metaUri + " looks like a file but it's neither a file nor a directory");
                 }
             } else {
                 if (RestClient.matchFullRestPattern(metaUri))
@@ -209,8 +214,6 @@ private static UriType decideUriType(String metaUri) {
             logger.info(e.getLocalizedMessage());
             throw new IllegalStateException("Metadata uri : " + metaUri + " is not recognized");
         }
-
-        return null;
     }
 
     public static KylinConfig createInstanceFromUri(String uri) {

common/src/main/java/com/kylinolap/common/util/CliCommandExecutor.java

@@ -48,7 +48,7 @@ public void setRunAtLocal() {
         this.remoteUser = null;
         this.remotePwd = null;
     }
-    
+
     public void copyFile(String localFile, String destDir) throws IOException {
         if (remoteHost == null)
             copyNative(localFile, destDir);
@@ -81,7 +81,9 @@ public String execute(String command) throws IOException {
             r = runRemoteCommand(command);
 
         if (r.getFirst() != 0)
-            throw new IOException("OS command error exit with " + r.getFirst() + " -- " + command + "\n" + r.getSecond());
+            throw new IOException("OS command error exit with " + r.getFirst() //
+                    + (remoteHost == null ? "" : " (remoteHost:" + remoteHost + ")") //
+                    + " -- " + command + "\n" + r.getSecond());
 
         return r.getSecond();
     }

common/src/test/java/com/kylinolap/common/util/MailServiceTest.java

@@ -48,7 +48,7 @@ public void testSendEmail() throws IOException {
     private boolean sendTestEmail(MailService mailservice) {
 
         List<String> receivers = new ArrayList<String>(1);
-        receivers.add("shaoshi@ebay.com");
+        receivers.add("foobar@foobar.com");
         try {
             return mailservice.sendMail(receivers, "A test email from Kylin", "Hello!");
         } catch (IOException e) {

cube/src/main/java/com/kylinolap/cube/common/BytesSplitter.java

@@ -77,25 +77,6 @@ public int split(byte[] bytes, int byteLen, byte delimiter) {
         return bufferSize;
     }
 
-    public static List<String> splitToString(byte[] bytes, int offset, byte delimiter) {
-        List<String> splitStrings = new ArrayList<String>();
-        int splitOffset = 0;
-        int splitLength = 0;
-        for (int i = offset; i < bytes.length; i++) {
-            if (bytes[i] == delimiter) {
-                String str = Bytes.toString(bytes, splitOffset, splitLength);
-                splitStrings.add(str);
-                splitOffset = i + 1;
-                splitLength = 0;
-            } else {
-                splitLength++;
-            }
-        }
-        String str = Bytes.toString(bytes, splitOffset, splitLength);
-        splitStrings.add(str);
-        return splitStrings;
-    }
-
     public byte inferByteRowDelimiter(byte[] bytes, int byteLen, int expectedSplits) throws IOException {
 
         if (expectedSplits > this.splitBuffers.length)
@@ -138,4 +119,36 @@ public int detectDelim(Text value, int expectedParts) {
         throw new RuntimeException("Cannot detect delimeter from first line -- " + value.toString() + " -- expect " + expectedParts + " columns");
     }
 
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder();
+        buf.append("[");
+        for (int i = 0; i < bufferSize; i++) {
+            if (i > 0)
+                buf.append(", ");
+
+            buf.append(Bytes.toString(splitBuffers[i].value, 0, splitBuffers[i].length));
+        }
+        return buf.toString();
+    }
+
+    public static List<String> splitToString(byte[] bytes, int offset, byte delimiter) {
+        List<String> splitStrings = new ArrayList<String>();
+        int splitOffset = 0;
+        int splitLength = 0;
+        for (int i = offset; i < bytes.length; i++) {
+            if (bytes[i] == delimiter) {
+                String str = Bytes.toString(bytes, splitOffset, splitLength);
+                splitStrings.add(str);
+                splitOffset = i + 1;
+                splitLength = 0;
+            } else {
+                splitLength++;
+            }
+        }
+        String str = Bytes.toString(bytes, splitOffset, splitLength);
+        splitStrings.add(str);
+        return splitStrings;
+    }
+
 }

deploy/healthmon.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+ALERT="[email protected]"
+
+OUTPUT=$(
+	curl --max-time 20 -# \
+	--data '{"sql":"select count(*) from test_kylin_fact","offset":0,"limit":50000,"acceptPartial":true,"project":"default"}' \
+	-H "Authorization:Basic QURNSU46S1lMSU4=" \
+	-H "Content-Type:application/json;charset=UTF-8" \
+	http://localhost:7070/kylin/api/query \
+)
+
+# ----------------------------------------------------------------------------
+
+date
+
+if [[ $OUTPUT == *"results"* ]]; then
+	echo "Good."
+else
+	echo "Bad."
+	TS_FILE=/tmp/kylin_healthmon_ts
+	LAST_TS=`stat -c%Y $TS_FILE 2>/dev/null`
+	CURR_TS=`date +%s`
+	echo last: $LAST_TS
+	echo curr: $CURR_TS
+	if (( ${LAST_TS:-"0"} < $CURR_TS - 3600 )); then
+		echo "Sending mail..."
+		echo "Kylin Prod health check failed as of $(date)." | mail -s "KYLIN PROD DOWN" $ALERT
+		if [ "$?" == "0" ]; then
+			touch $TS_FILE
+		fi
+	fi
+fi

deploy/install.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 
 cd ~

dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java

@@ -40,6 +40,8 @@
 @SuppressWarnings({ "rawtypes", "unchecked" })
 public class DictionaryGenerator {
 
+    private static final int DICT_MAX_CARDINALITY = 2000000; // 2 million
+
     private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
 
     private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd" };
@@ -71,8 +73,8 @@ else if (dataType.isNumberFamily())
         logger.info("Dictionary value samples: " + buf.toString());
         logger.info("Dictionary cardinality " + info.getCardinality());
 
-        if (values.size() > 1000000)
-            throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary! Are the values stable enough for incremental load??");
+        if (values.size() > DICT_MAX_CARDINALITY)
+            throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary -- " + info.getSourceTable() + "." + info.getSourceColumn() + " cardinality: " + values.size());
 
         return dict;
     }
@@ -113,7 +115,7 @@ public static Dictionary<?> buildDictionary(DictionaryInfo info, ReadableTable i
     private static Dictionary buildDateStrDict(List<byte[]> values, int baseId, int nSamples, ArrayList samples) {
         final int BAD_THRESHOLD = 2;
         String matchPattern = null;
-        
+
         for (String ptn : DATE_PATTERNS) {
             matchPattern = ptn; // be optimistic
             int badCount = 0;

dictionary/src/main/java/com/kylinolap/dict/lookup/SnapshotManager.java

@@ -123,7 +123,7 @@ private String checkDupByInfo(SnapshotTable snapshot) throws IOException {
             // direct
             // load from
             // store
-            if (sig.equals(existingTable.getSignature()))
+            if (existingTable != null && sig.equals(existingTable.getSignature()))
                 return existing;
         }
 
@@ -153,6 +153,7 @@ private void save(SnapshotTable snapshot) throws IOException {
     }
 
     private SnapshotTable load(String resourcePath, boolean loadData) throws IOException {
+        logger.info("Loading snapshotTable from " + resourcePath + ", with loadData: " + loadData);
         ResourceStore store = MetadataManager.getInstance(this.config).getStore();
 
         SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class, loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER);

docs/Installation/Frequently Asked Questions on Installation.md

@@ -0,0 +1,26 @@
+Frequently Asked Questions on Installation
+---
+* Some NPM error causes ERROR exit (中国大陆地区用户请特别注意此问题)?
+> Check out https://github.com/KylinOLAP/Kylin/issues/35
+
+* Can't get master address from ZooKeeper" when installing Kylin on Hortonworks Sandbox
+> Check out https://github.com/KylinOLAP/Kylin/issues/9.
+
+* Install scripted finished in my virtual machine, but cannot visit via http://localhost:9080
+> Check out https://github.com/KylinOLAP/Kylin/issues/12.
+
+* Map Reduce Job information can't display on sandbox deployment
+> Check out https://github.com/KylinOLAP/Kylin/issues/40
+
+* Install Kylin on CDH 5.2 or Hadoop 2.5.x
+> Check out discussion: https://groups.google.com/forum/?utm_medium=email&utm_source=footer#!msg/kylin-olap/X0GZfsX1jLc/nzs6xAhNpLkJ
+> 
+```
+I was able to deploy Kylin with following option in POM.
+<hadoop2.version>2.5.0</hadoop2.version>
+<yarn.version>2.5.0</yarn.version>
+<hbase-hadoop2.version>0.98.6-hadoop2</hbase-hadoop2.version>
+<zookeeper.version>3.4.5</zookeeper.version>
+<hive.version>0.13.1</hive.version>
+My Cluster is running on Cloudera Distribution CDH 5.2.0.
+```

docs/Installation/Off Hadoop CLI Installation (Dev Env Setup).md

@@ -0,0 +1,76 @@
+Off Hadoop CLI Installation (Dev Env Setup)
+===
+Off-Hadoop-CLI installation is usually for **development use**.
+
+Developers want to run kylin test cases or applications at their development machine. The scenario is depicted at https://github.com/KylinOLAP/Kylin#off-hadoop-cli-installation.
+
+
+By following this tutorial, you will be able to build kylin test cubes by running a specific test case, and you can further run other test cases against the cubes having been built.
+
+
+## Environment on the Hadoop CLI
+
+Off-Hadoop-CLI installation requires you having a hadoop CLI machine (or a hadoop sandbox) as well as your local develop machine. To make things easier we strongly recommend you starting with running Kylin on a hadoop sandbox, like <http://hortonworks.com/products/hortonworks-sandbox/>. In the following tutorial we'll go with **Hortonworks Sandbox 2.1**. 
+
+### Start Hadoop
+
+In Hortonworks sandbox, ambari helps to launch hadoop:
+
+	ambari-agent start
+	ambari-server start
+
+With both command successfully run you can go to ambari home page at <http://yoursandboxip:8080> (user:admin,password:admin) to check everything's status. By default ambari disables Hbase, you'll need manually start the `Hbase` service.
+
+For other hadoop distribution, basically start the hadoop cluster, make sure HDFS, YARN, Hive, HBase are running.
+
+
+## Environment on the dev machine
+
+### Install maven
+
+The latest maven can be found at <http://maven.apache.org/download.cgi>, we create a symbolic so that `mvn` can be run anywhere.
+
+	cd ~
+	wget http://apache.proserve.nl/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.tar.gz
+	tar -xzvf apache-maven-3.2.3-bin.tar.gz 
+	ln -s /root/apache-maven-3.2.3/bin/mvn /usr/bin/mvn
+
+### Compile
+
+First clone the Kylin project to your local:
+
+	git clone https://github.com/KylinOLAP/Kylin.git
+
+Install Kylin artifacts to the maven repo
+
+	mvn clean install -DskipTests
+
+### Modify local configuration
+
+Local configuration must be modified to point to your hadoop sandbox (or CLI) machine. If you are using a Hortonworks sandbox, this section may be skipped.
+
+* In **examples/test_case_data/sandbox/kylin.properties**
+   * Find `sandbox` and replace with your hadoop hosts
+   * Find `kylin.job.remote.cli.username` and `kylin.job.remote.cli.password`, fill in the user name and password used to login hadoop cluster for hadoop command execution
+
+* In **examples/test_case_data/sandbox**
+   * For each configuration xml file, find all occurrence of `sandbox` and replace with your hadoop hosts
+
+An alternative to the host replacement is updating your `hosts` file to resolve `sandbox` and `sandbox.hortonworks.com` to the IP of your sandbox machine.
+
+### Run unit tests
+
+Run a end-to-end cube building test
+
+	mvn test -Dtest=com.kylinolap.job.BuildCubeWithEngineTest -DfailIfNoTests=false
+
+Run other tests, the end-to-end cube building test is exclueded
+
+	mvn test
+
+### Launch Kylin Web Server
+
+In your Eclipse IDE, launch `com.kylinolap.rest.DebugTomcat` with specifying VM arguments "-Dspring.profiles.active=sandbox". (By default Kylin server will listen on 7070 port; If you want to use another port, please specify it as a parameter when run `DebugTomcat)
+
+Check Kylin Web available at http://localhost:7070 (user:ADMIN,password:KYLIN)
+