Merge pull request pinterest#76 from mthssdrbrg/travis-integration-tests

(Fully) local integration tests

.travis.yml

@@ -1,6 +1,20 @@
 language: java
+addons:
+  hosts:
+    - test-bucket.localhost
+env:
+  - PATH=$PATH:$HOME/.s3cmd SECOR_LOCAL_S3=true S3CMD=1.0.1
 jdk:
-  - oraclejdk8
+  - openjdk7
   - oraclejdk7
-  - openjdk6
-  - openjdk7
+  - oraclejdk8
+before_install:
+  - wget https://github.com/s3tools/s3cmd/archive/v$S3CMD.tar.gz -O /tmp/s3cmd.tar.gz
+  - tar -xzf /tmp/s3cmd.tar.gz -C $HOME
+  - mv $HOME/s3cmd-$S3CMD $HOME/.s3cmd
+  - cd $HOME/.s3cmd && python setup.py install --user && cd -
+  - gem install fakes3 -v 0.1.7
+script:
+  - make unit
+  - make integration
+

Makefile

@@ -0,0 +1,22 @@
+CONFIG=src/main/config
+TEST_HOME=/tmp/secor_test
+TEST_CONFIG=src/test/config
+JAR_FILE=target/secor-*-SNAPSHOT-bin.tar.gz
+MVN_OPTS=-DskipTests=true -Dmaven.javadoc.skip=true
+
+build:
+	@mvn package $(MVN_OPTS)
+
+unit:
+	@mvn test
+
+integration: build
+	@rm -rf $(TEST_HOME)
+	@mkdir -p $(TEST_HOME)
+	@tar -xzf $(JAR_FILE) -C $(TEST_HOME)
+	@cp $(TEST_CONFIG)/* $(TEST_HOME)
+	@[ ! -e $(CONFIG)/jets3t.properties ] && jar uf $(TEST_HOME)/secor-*.jar -C $(TEST_CONFIG) jets3t.properties
+	cd $(TEST_HOME) && ./scripts/run_tests.sh
+
+test: build unit integration
+

src/main/config/secor.test.backup.properties

@@ -0,0 +1,3 @@
+include=secor.test.properties
+include=secor.dev.backup.properties
+

src/main/config/secor.test.partition.properties

@@ -0,0 +1,3 @@
+include=secor.test.properties
+include=secor.dev.partition.properties
+

src/main/config/secor.test.properties

@@ -0,0 +1,4 @@
+secor.s3.bucket=test-bucket
+aws.access.key=TESTKEY
+aws.secret.key=TESTKEY
+

src/main/scripts/run_common.sh

@@ -12,3 +12,7 @@ if [ -z "${JAVA_HOME}" ]; then
 else
     JAVA="${JAVA_HOME}/bin/java"
 fi
+
+DEFAULT_CLASSPATH="*:lib/*"
+CLASSPATH=${CLASSPATH:-$DEFAULT_CLASSPATH}
+

src/main/scripts/run_kafka_class.sh

@@ -24,22 +24,6 @@ if [ $# -lt 1 ]; then
     exit 1
 fi
 
-base_dir=$(dirname $0)/..
-
-SCALA_VERSION=2.8.0
-
-# assume all dependencies have been packaged into one jar with sbt-assembly's task
-# "assembly-package-dependency"
-# for file in lib/*.jar; do
-#   CLASSPATH=$CLASSPATH:$file
-# done
-
-# for file in $base_dir/kafka*.jar; do
-#   CLASSPATH=$CLASSPATH:$file
-# done
-
-CLASSPATH=${CLASSPATH}:${base_dir}/lib/*
-
 # JMX settings
 KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false "
 

src/main/scripts/run_tests.sh

@@ -36,7 +36,8 @@
 
 PARENT_DIR=/tmp/secor_dev
 LOGS_DIR=${PARENT_DIR}/logs
-S3_LOGS_DIR=s3://sk-cloud-staging/secor_dev
+BUCKET=${SECOR_BUCKET:-test-bucket}
+S3_LOGS_DIR=s3://${BUCKET}/secor_dev
 MESSAGES=1000
 MESSAGE_TYPE=binary
 # For the compression tests to work, set this to the path of the Hadoop native libs.
@@ -51,68 +52,100 @@ READER_WRITERS[binary]=com.pinterest.secor.io.impl.SequenceFileReaderWriterFacto
 
 # The minimum wait time is one minute plus delta.  Secor is configured to upload files older than
 # one minute and we need to make sure that everything ends up on s3 before starting verification.
-WAIT_TIME=120
-base_dir=$(dirname $0)
+WAIT_TIME=${SECOR_WAIT_TIME:-120}
+BASE_DIR=$(dirname $0)
+CONF_DIR=${BASE_DIR}/..
 
-source ${base_dir}/run_common.sh
+source ${BASE_DIR}/run_common.sh
 
 run_command() {
     echo "running $@"
     eval "$@"
 }
 
+check_for_native_libs() {
+    files=($(find "${HADOOP_NATIVE_LIB_PATH}" -maxdepth 1 -name "*.so" 2> /dev/null))
+    if [ ${#files[@]} -eq 0 ]; then
+        echo "Couldn't find Hadoop native libraries, skipping compressed binary tests"
+        SKIP_COMPRESSED_BINARY="true"
+    fi
+}
+
 recreate_dirs() {
     run_command "rm -r -f ${PARENT_DIR}"
-    run_command "s3cmd del --recursive ${S3_LOGS_DIR}"
+    if [ -n ${SECOR_LOCAL_S3} ]; then
+        run_command "s3cmd -c ${CONF_DIR}/test.s3cfg ls ${S3_LOGS_DIR} | awk '{ print \$4 }' | xargs -L 1 s3cmd -c ${CONF_DIR}/test.s3cfg del"
+    else
+        run_command "s3cmd del --recursive ${S3_LOGS_DIR}"
+    fi
     # create logs directory
     if [ ! -d ${LOGS_DIR} ]; then
         run_command "mkdir -p ${LOGS_DIR}"
     fi
 }
 
+start_s3() {
+    if [ -n ${SECOR_LOCAL_S3} ]; then
+        if command -v fakes3 > /dev/null 2>&1; then
+            run_command "fakes3 --root=/tmp/fakes3 --port=5000 --hostname=localhost > /tmp/fakes3.log 2>&1 &"
+            sleep 2
+            run_command "s3cmd -c ${CONF_DIR}/test.s3cfg mb s3://${BUCKET}"
+        else
+            echo "Couldn't find FakeS3 binary, please install it using `gem install fakes3`"
+        fi
+    fi
+}
+
+stop_s3() {
+    if [ -n ${SECOR_LOCAL_S3} ]; then
+        run_command "pkill -9 'fakes3' > /dev/null 2>&1 || true"
+        run_command "rm -r -f /tmp/fakes3"
+    fi
+}
+
 start_zookeeper() {
-    run_command "${base_dir}/run_kafka_class.sh \
-        org.apache.zookeeper.server.quorum.QuorumPeerMain zookeeper.test.properties > \
+    run_command "${BASE_DIR}/run_kafka_class.sh \
+        org.apache.zookeeper.server.quorum.QuorumPeerMain ${CONF_DIR}/zookeeper.test.properties > \
         ${LOGS_DIR}/zookeeper.log 2>&1 &"
 }
 
 stop_zookeeper() {
-    run_command "pkill -f 'org.apache.zookeeper.server.quorum.QuorumPeerMain' | true"
+    run_command "pkill -f 'org.apache.zookeeper.server.quorum.QuorumPeerMain' || true"
 }
 
 start_kafka_server () {
-    run_command "${base_dir}/run_kafka_class.sh kafka.Kafka kafka.test.properties > \
+    run_command "${BASE_DIR}/run_kafka_class.sh kafka.Kafka ${CONF_DIR}/kafka.test.properties > \
         ${LOGS_DIR}/kafka_server.log 2>&1 &"
 }
 
 stop_kafka_server() {
-    run_command "pkill -f 'kafka.Kafka' | true"
+    run_command "pkill -f 'kafka.Kafka' || true"
 }
 
 start_secor() {
     run_command "${JAVA} -server -ea -Dlog4j.configuration=log4j.dev.properties \
-        -Dconfig=secor.dev.backup.properties ${ADDITIONAL_OPTS} -cp secor-0.1-SNAPSHOT.jar:lib/* \
+        -Dconfig=secor.test.backup.properties ${ADDITIONAL_OPTS} -cp $CLASSPATH \
         com.pinterest.secor.main.ConsumerMain > ${LOGS_DIR}/secor_backup.log 2>&1 &"
     if [ "${MESSAGE_TYPE}" = "binary" ]; then
        run_command "${JAVA} -server -ea -Dlog4j.configuration=log4j.dev.properties \
-           -Dconfig=secor.dev.partition.properties ${ADDITIONAL_OPTS} -cp secor-0.1-SNAPSHOT.jar:lib/* \
+           -Dconfig=secor.test.partition.properties ${ADDITIONAL_OPTS} -cp $CLASSPATH \
            com.pinterest.secor.main.ConsumerMain > ${LOGS_DIR}/secor_partition.log 2>&1 &"
     fi
 }
 
 stop_secor() {
-    run_command "pkill -f 'com.pinterest.secor.main.ConsumerMain' | true"
+    run_command "pkill -f 'com.pinterest.secor.main.ConsumerMain' || true"
 }
 
 create_topic() {
-    run_command "${base_dir}/run_kafka_class.sh kafka.admin.TopicCommand --create --zookeeper \
+    run_command "${BASE_DIR}/run_kafka_class.sh kafka.admin.TopicCommand --create --zookeeper \
         localhost:2181 --replication-factor 1 --partitions 2 --topic test > \
         ${LOGS_DIR}/create_topic.log 2>&1"
 }
 
 post_messages() {
     run_command "${JAVA} -server -ea -Dlog4j.configuration=log4j.dev.properties \
-        -Dconfig=secor.dev.backup.properties -cp secor-0.1-SNAPSHOT.jar:lib/* \
+        -Dconfig=secor.test.backup.properties -cp ${CLASSPATH} \
         com.pinterest.secor.main.TestLogMessageProducerMain -t test -m $1 -p 1 -type ${MESSAGE_TYPE} > \
         ${LOGS_DIR}/test_log_message_producer.log 2>&1"
 }
@@ -124,14 +157,16 @@ verify() {
     fi
     for RUNMODE in ${RUNMODE_0} ${RUNMODE_1}; do
       run_command "${JAVA} -server -ea -Dlog4j.configuration=log4j.dev.properties \
-          -Dconfig=secor.dev.${RUNMODE}.properties ${ADDITIONAL_OPTS} -cp secor-0.1-SNAPSHOT.jar:lib/* \
+          -Dconfig=secor.test.${RUNMODE}.properties ${ADDITIONAL_OPTS} -cp ${CLASSPATH} \
           com.pinterest.secor.main.LogFileVerifierMain -t test -m $1 -q > \
           ${LOGS_DIR}/log_verifier_${RUNMODE}.log 2>&1"
       VERIFICATION_EXIT_CODE=$?
       if [ ${VERIFICATION_EXIT_CODE} -ne 0 ]; then
         echo -e "\e[1;41;97mVerification FAILED\e[0m"
         echo "See log ${LOGS_DIR}/log_verifier_${RUNMODE}.log for more details"
         tail -n 50 ${LOGS_DIR}/log_verifier_${RUNMODE}.log
+        stop_all
+        stop_s3
         exit ${VERIFICATION_EXIT_CODE}
       fi
     done
@@ -140,16 +175,15 @@ verify() {
 set_offsets_in_zookeeper() {
     for group in secor_backup secor_partition; do
         for partition in 0 1; do
-            run_command "${base_dir}/run_zookeeper_command.sh localhost:2181 create \
+            run_command "${BASE_DIR}/run_zookeeper_command.sh localhost:2181 create \
                 /consumers \'\' > ${LOGS_DIR}/run_zookeeper_command.log 2>&1"
-            run_command "${base_dir}/run_zookeeper_command.sh localhost:2181 create \
+            run_command "${BASE_DIR}/run_zookeeper_command.sh localhost:2181 create \
                 /consumers/${group} \'\' > ${LOGS_DIR}/run_zookeeper_command.log 2>&1"
-            run_command "${base_dir}/run_zookeeper_command.sh localhost:2181 create \
+            run_command "${BASE_DIR}/run_zookeeper_command.sh localhost:2181 create \
                 /consumers/${group}/offsets \'\' > ${LOGS_DIR}/run_zookeeper_command.log 2>&1"
-            run_command "${base_dir}/run_zookeeper_command.sh localhost:2181 create \
+            run_command "${BASE_DIR}/run_zookeeper_command.sh localhost:2181 create \
                 /consumers/${group}/offsets/test \'\' > ${LOGS_DIR}/run_zookeeper_command.log 2>&1"
-
-            run_command "${base_dir}/run_zookeeper_command.sh localhost:2181 create \
+            run_command "${BASE_DIR}/run_zookeeper_command.sh localhost:2181 create \
                 /consumers/${group}/offsets/test/${partition} $1 > \
                 ${LOGS_DIR}/run_zookeeper_command.log 2>&1"
         done
@@ -250,6 +284,8 @@ post_and_verify_compressed_test() {
     echo -e "\e[1;42;97mpost_and_verify_compressed_test succeeded\e[0m"
 }
 
+check_for_native_libs
+start_s3
 
 for key in ${!READER_WRITERS[@]}; do
    MESSAGE_TYPE=${key}
@@ -258,6 +294,13 @@ for key in ${!READER_WRITERS[@]}; do
    post_and_verify_test
    start_from_non_zero_offset_test
    move_offset_back_test
-   post_and_verify_compressed_test
+   if [ ${key} = "json" ]; then
+       post_and_verify_compressed_test
+   elif [ -z ${SKIP_COMPRESSED_BINARY} ]; then
+       post_and_verify_compressed_test
+   else
+       echo "Skipping compressed tests for ${key}"
+   fi
 done
 
+stop_s3

src/main/scripts/run_zookeeper_command.sh

@@ -25,4 +25,4 @@ fi
 CURR_DIR=`dirname $0`
 source ${CURR_DIR}/run_common.sh
 
-${JAVA} -ea -cp "secor-0.1-SNAPSHOT.jar:lib/*" org.apache.zookeeper.ZooKeeperMain -server $@
+${JAVA} -ea -cp "$CLASSPATH" org.apache.zookeeper.ZooKeeperMain -server $@

src/test/config/jets3t.properties

@@ -0,0 +1,3 @@
+s3service.https-only=false
+s3service.s3-endpoint-http-port=5000
+s3service.s3-endpoint=localhost

src/test/config/test.s3cfg

@@ -0,0 +1,12 @@
+[default]
+access_key = TESTACCESSKEY
+bucket_location = US
+default_mime_type = binary/octet-stream
+encoding = UTF-8
+get_continue = False
+guess_mime_type = True
+host_base = localhost:5000
+host_bucket = %(bucket)s.localhost:5000
+secret_key = TESTSECRETKEY
+use_https = False
+verbosity = WARNING