Skip to content

Commit

Permalink
Enable travis (pingcap#1772)
Browse files Browse the repository at this point in the history
* Try travis

* wip

* Try travis

* try

* try

* try

* try

* Revert "try"

This reverts commit 7ecd73c2eb255c94acec4830d78c09ecb68ccfaa.

* try unbundled

* Try unbundled2

* Try undundled3

* Try unbundled4

* Try5

* try gcc7

* try6

* Try clang

* Try min

* try ccache

* try ccache stats

* no sudo

* try clang

* try ccache

* reset changelog

* Try fix clang

* try limit time

* Faster

* try timeout

* try faster

* Try

* try

* try

* try

* try

* try

* ccache

* Fix

* try test

* try test

* Fix try_listen in some vm's

* fix

* test

* fix

* cache timeout

* packages

* fix

* fix

* fix

* try emb compiler

* Try emb compiler

* Revert "try emb compiler"

This reverts commit 471713cabe5f111debbb4ba8252d5a8336bdecda.

* try

* Revert "Try emb compiler"

This reverts commit 95e632abf676840ded0bb8da945014a13b377978.

* Skip long

* fix

* fix

* ccache

* fix

* debug

* Fxi test

* test fix

* tes fix

* Fix

* Fix link

* Fix odbc link

* Fxi test

* fix  boost unbundled include

* fix test

* fix test

* test fix

* fix tests

* tests

* Test fail
  • Loading branch information
proller authored and alexey-milovidov committed Jan 15, 2018
1 parent 07931ca commit f6d8436
Show file tree
Hide file tree
Showing 46 changed files with 294 additions and 66 deletions.
66 changes: 66 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
language: cpp

matrix:
include:
- os: linux

cache:
ccache: true
timeout: 1000

addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages: [ g++-7, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]
env:
- MATRIX_EVAL="CC=gcc-7 && CXX=g++-7"

script:
- utils/travis/normal.sh


# We need to have gcc7 headers to compile c++17 code on clang
- os: linux

cache:
ccache: true
timeout: 1000

addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-trusty-5.0
packages: [ g++-7, clang-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo ]

env:
- MATRIX_EVAL="CC=clang-5.0 && CXX=clang++-5.0

before_script:
# fix gcc after clang
- export CC=clang-5.0
- export CXX=clang++-5.0

script:
- utils/travis/normal.sh


- os: linux

sudo: required

cache:
timeout: 1000
directories:
- /var/cache/pbuilder/ccache

addons:
apt:
packages: [ pbuilder, fakeroot, debhelper ]

script:
- utils/travis/pbuilder.sh

before_install:
- eval "${MATRIX_EVAL}"
16 changes: 9 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -145,14 +145,14 @@ endif ()

set (CMAKE_BUILD_COLOR_MAKEFILE ON)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
#set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline")
#set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")

set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
#set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CMAKE_C_FLAGS_ADD}")
#set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")

if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_SYSTEM MATCHES "FreeBSD"))
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
Expand Down Expand Up @@ -265,6 +265,8 @@ include (cmake/find_boost.cmake)
# openssl, zlib before poco
include (cmake/find_zlib.cmake)
include (cmake/find_zstd.cmake)
include (cmake/find_ltdl.cmake) # for odbc
include (contrib/poco/cmake/FindODBC.cmake) # for poco
include (cmake/find_poco.cmake)
include (cmake/find_lz4.cmake)
include (cmake/find_sparsehash.cmake)
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.

[Read more...](https://clickhouse.yandex/)

[![Build Status](https://travis-ci.org/yandex/ClickHouse.svg?branch=master)](https://travis-ci.org/yandex/ClickHouse)
4 changes: 4 additions & 0 deletions cmake/Modules/FindPoco.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,10 @@ foreach( component ${components} )
endif()
endforeach()

if(Poco_DataODBC_LIBRARY)
list(APPEND Poco_DataODBC_LIBRARY ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
endif()

if(DEFINED Poco_LIBRARIES)
set(Poco_FOUND true)
endif()
Expand Down
4 changes: 2 additions & 2 deletions cmake/find_ltdl.cmake
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
set (LTDL_PATHS "/usr/local/opt/libtool/lib")
find_library (LTDL_LIB ltdl PATHS ${LTDL_PATHS})
message (STATUS "Using ltdl: ${LTDL_LIB}")
find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS})
message (STATUS "Using ltdl: ${LTDL_LIBRARY}")
5 changes: 1 addition & 4 deletions cmake/find_poco.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ else ()
set (POCO_STATIC ${MAKE_STATIC_LIBRARIES} CACHE BOOL "")
set (POCO_VERBOSE_MESSAGES 1 CACHE BOOL "")

include (${ClickHouse_SOURCE_DIR}/cmake/find_ltdl.cmake)
include (${ClickHouse_SOURCE_DIR}/contrib/poco/cmake/FindODBC.cmake)

# used in internal compiler
list (APPEND Poco_INCLUDE_DIRS
Expand All @@ -44,8 +42,7 @@ else ()

if (ODBC_FOUND)
set (Poco_DataODBC_FOUND 1)
set (Poco_DataODBC_LIBRARY PocoDataODBC)
list (APPEND Poco_DataODBC_LIBRARY ${LTDL_LIB})
set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY})
set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/")
endif ()

Expand Down
5 changes: 3 additions & 2 deletions cmake/find_rdkafka.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,13 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contri
endif ()

if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
find_library (RDKAFKA_LIBRARY rdkafka)
find_library (RDKAFKA_LIB rdkafka)
find_path (RDKAFKA_INCLUDE_DIR NAMES librdkafka/rdkafka.h PATHS ${RDKAFKA_INCLUDE_PATHS})
endif ()

if (RDKAFKA_LIBRARY AND RDKAFKA_INCLUDE_DIR)
if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR)
set (USE_RDKAFKA 1)
set (RDKAFKA_LIBRARY ${RDKAFKA_LIB} ${OPENSSL_LIBRARIES})
elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY)
set (USE_INTERNAL_RDKAFKA_LIBRARY 1)
set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")
Expand Down
8 changes: 6 additions & 2 deletions dbms/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,9 @@ target_link_libraries (clickhouse_common_io
${DOUBLE_CONVERSION_LIBRARIES}
${Poco_Net_LIBRARY}
${Poco_Data_LIBRARY}
${ZLIB_LIBRARIES}
${EXECINFO_LIBRARY}
${Boost_SYSTEM_LIBRARY}
)

target_link_libraries (dbms
Expand All @@ -159,11 +161,9 @@ target_link_libraries (dbms
${MYSQLXX_LIBRARY}
${FARMHASH_LIBRARIES}
${METROHASH_LIBRARIES}
${ZLIB_LIBRARIES}
${RE2_LIBRARY}
${RE2_ST_LIBRARY}
${OPENSSL_CRYPTO_LIBRARY}
${Boost_SYSTEM_LIBRARY}
${BTRIE_LIBRARIES}
)

Expand All @@ -175,6 +175,10 @@ if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
endif ()

if (NOT USE_INTERNAL_BOOST_LIBRARY)
target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
endif ()

if (Poco_DataODBC_FOUND)
target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
endif()
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Analyzers/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ target_link_libraries(analyze_columns dbms clickhouse_storages_system)

add_executable(type_and_constant_inference type_and_constant_inference.cpp)
target_link_libraries(type_and_constant_inference
clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions dbms)
clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)

add_executable(analyze_result_of_query analyze_result_of_query.cpp)
target_link_libraries(analyze_result_of_query dbms clickhouse_storages_system)
Expand Down
1 change: 0 additions & 1 deletion dbms/src/Server/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,6 @@ else ()
clickhouse-extract-from-config-lib
clickhouse-compressor-lib
clickhouse-format-lib
dbms
)

add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse)
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Server/Server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
}
catch (const Poco::Net::NetException & e)
{
if (try_listen && e.code() == POCO_EPROTONOSUPPORT)
if (try_listen && (e.code() == POCO_EPROTONOSUPPORT || e.code() == POCO_EADDRNOTAVAIL))
LOG_ERROR(log, "Listen [" << listen_host << "]: " << e.what() << ": " << e.message()
<< " If it is an IPv6 or IPv4 address and your host has disabled IPv6 or IPv4, then consider to "
"specify not disabled IPv4 or IPv6 address to listen in <listen_host> element of configuration "
Expand Down
14 changes: 12 additions & 2 deletions dbms/tests/clickhouse-test
Original file line number Diff line number Diff line change
Expand Up @@ -156,14 +156,22 @@ def main(args):
print "{0:70}".format(name + ": "),
sys.stdout.flush()

if not args.zookeeper and 'zookeeper' in name:
if args.skip and any(s in name for s in args.skip):
report_testcase.append(et.Element("skipped", attrib = {"message": "skip"}))
print(MSG_SKIPPED + " - skip")
skipped_total += 1
elif not args.zookeeper and 'zookeeper' in name:
report_testcase.append(et.Element("skipped", attrib = {"message": "no zookeeper"}))
print(MSG_SKIPPED + " - no zookeeper")
skipped_total += 1
elif not args.shard and 'shard' in name:
report_testcase.append(et.Element("skipped", attrib = {"message": "no shard"}))
print(MSG_SKIPPED + " - no shard")
skipped_total += 1
elif not args.no_long and 'long' in name:
report_testcase.append(et.Element("skipped", attrib = {"message": "no long"}))
print(MSG_SKIPPED + " - no long")
skipped_total += 1
else:
disabled_file = os.path.join(suite_dir, name) + '.disabled'

Expand Down Expand Up @@ -319,6 +327,8 @@ if __name__ == '__main__':
parser.add_argument('--order', default = 'desc', help = 'Run order (asc, desc, random)')
parser.add_argument('--testname', action = 'store_true', default = None, dest = 'testname', help = 'Make query with test name before test run')

parser.add_argument('--skip', nargs='+', help = "Skip these tests")
parser.add_argument('--no-long', action = 'store_false', default = None, dest = 'no_long', help = 'Do not run long tests')
group = parser.add_mutually_exclusive_group(required = False)
group.add_argument('--zookeeper', action = 'store_true', default = None, dest = 'zookeeper', help = 'Run zookeeper related tests')
group.add_argument('--no-zookeeper', action = 'store_false', default = None, dest = 'zookeeper', help = 'Do not run zookeeper related tests')
Expand All @@ -332,7 +342,7 @@ if __name__ == '__main__':
args.queries = 'queries'
if args.tmp is None:
args.tmp = args.queries
else:
elif args.queries is None:
args.queries = '/usr/share/clickhouse-test/queries'
if args.tmp is None:
args.tmp = '/tmp/clickhouse-test'
Expand Down
3 changes: 3 additions & 0 deletions dbms/tests/client-test.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
<yandex>
<tcp_port>59000</tcp_port>
</yandex>
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bash

CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh

# We should have correct env vars from shell_config.sh to run this test

python $CURDIR/00411_accurate_number_comparison.python
2 changes: 1 addition & 1 deletion dbms/tests/queries/0_stateless/00532_pager_sigpipe.tcl
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ set timeout -1

log_user 0

spawn clickhouse-client --pager=head
spawn sh -c $::env(CLICKHOUSE_CLIENT) --pager=head

expect ":) "

Expand Down
49 changes: 49 additions & 0 deletions dbms/tests/server-test.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<?xml version="1.0"?>
<yandex>
<logger>
<level>trace</level>
<log>/tmp/clickhouse/log/clickhouse-server.log</log>
<errorlog>/tmp/clickhouse/log/clickhouse-server.err.log</errorlog>
<size>10M</size>
<count>1</count>
</logger>
<http_port>58123</http_port>
<tcp_port>59000</tcp_port>
<interserver_http_port>59009</interserver_http_port>
<keep_alive_timeout>3</keep_alive_timeout>
<path>/tmp/clickhouse/data/</path>
<tmp_path>/tmp/clickhouse/tmp/</tmp_path>
<users_config>users.xml</users_config>
<mark_cache_size>5368709120</mark_cache_size>
<default_profile>default</default_profile>
<default_database>default</default_database>
<timezone>Europe/Moscow</timezone>
<remote_servers incl="clickhouse_remote_servers" >
<!-- Test only shard config for testing distributed storage -->
<test_shard_localhost>
<shard>
<replica>
<host>localhost</host>
<port>59000</port>
</replica>
</shard>
</test_shard_localhost>
</remote_servers>
<zookeeper incl="zookeeper-servers" optional="true" />
<macros incl="macros" optional="true" />
<builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
<max_session_timeout>3600</max_session_timeout>
<default_session_timeout>60</default_session_timeout>
<query_log>
<database>system</database>
<table>query_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</query_log>
<dictionaries_config>*_dictionary.xml</dictionaries_config>
<compression incl="clickhouse_compression">
</compression>
<distributed_ddl>
<path>/clickhouse/task_queue/ddl</path>
</distributed_ddl>
<format_schema_path>/tmp/clickhouse/data/format_schemas/</format_schema_path>
</yandex>
39 changes: 39 additions & 0 deletions dbms/tests/server_wrapper.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/bin/bash

set -o errexit
set -o pipefail

CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
ROOTDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)
DATADIR=${DATADIR:=/tmp/clickhouse}
LOGDIR=${LOGDIR:=$DATADIR/log}

rm -rf $DATADIR

mkdir -p $LOGDIR

# Start a local clickhouse server which will be used to run tests
#PATH=$PATH:$ROOTDIR/build${BUILD_TYPE}/dbms/src/Server \
$ROOTDIR/build${BUILD_TYPE}/dbms/src/Server/clickhouse-server --config-file=$CURDIR/server-test.xml > $LOGDIR/stdout 2>&1 &
CH_PID=$!
sleep 3

# Define needed stuff to kill test clickhouse server after tests completion
function finish {
kill $CH_PID || true
wait
tail -n 50 $LOGDIR/stdout
rm -rf $DATADIR
}
trap finish EXIT

# Do tests
export CLICKHOUSE_CONFIG=${CLICKHOUSE_CONFIG:=$CURDIR/server-test.xml}
#cd $CURDIR
if [ -n "$*" ]; then
$*
else
$ROOTDIR/build${BUILD_TYPE}/dbms/src/Server/clickhouse-client --config $CURDIR/client-test.xml -q 'SELECT * from system.build_options;'
PATH=$PATH:$ROOTDIR/build${BUILD_TYPE}/dbms/src/Server \
$CURDIR/clickhouse-test --binary $ROOTDIR/build${BUILD_TYPE}/dbms/src/Server/clickhouse --clientconfig $CURDIR/client-test.xml --tmp $DATADIR/tmp --queries $CURDIR/queries $TEST_OPT
fi
1 change: 1 addition & 0 deletions dbms/tests/users.xml
Loading

0 comments on commit f6d8436

Please sign in to comment.