diff --git a/CMakeLists.txt b/CMakeLists.txt index c78da47..bfae92c 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,6 +66,10 @@ include_directories(${FOLLY_INCLUDE_DIR} ${FBTHRIFT_INCLUDE_DIR} ${PROXYGEN_INCL add_subdirectory(${TP_PROJECTS_DIR}/gtest) include_directories(${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS}) +if( $ENV{BERINGEI_AUTOSETUP} STREQUAL "true") + add_definitions("-DBERINGEI_AUTOSETUP") +endif() + enable_testing() add_subdirectory("beringei") diff --git a/Dockerfile b/Dockerfile index 9647637..97a99da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ -# Pull base image. -FROM ubuntu:16.10 +FROM ubuntu:17.10 ENV WORKDIR /usr/local/beringei ENV RUN_CMD ./beringei/service/beringei_main \ @@ -12,6 +11,8 @@ ENV RUN_CMD ./beringei/service/beringei_main \ -logtostderr \ -v=2 +RUN apt-get update +RUN apt-get -y upgrade # Copy files from CircleCI into docker container. COPY . $WORKDIR @@ -22,14 +23,22 @@ CMD ["bash"] WORKDIR $WORKDIR RUN $WORKDIR/setup_ubuntu.sh +RUN apt-get install -y libmysqlclient20 libmysqlclient-dev libmysql++3v5 libmysqlcppconn7v5 libmysqlcppconn-dev libboost-all-dev libcap-dev libdouble-conversion-dev libevent-dev libgflags2.2 libgoogle-glog-dev libjemalloc-dev libkrb5-dev liblz4-dev liblzma-dev libnuma-dev libsasl2-dev libsnappy-dev libssl-dev zlib1g-dev + +# cmake is restricted to look in /usr/local/mysql, linking here +RUN mkdir -p /usr/local/mysql/lib +RUN ln -s /usr/lib/libmysqlpp.so* /usr/local/mysql/lib +RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.* /usr/local/mysql/lib/ + # Create a build directory. RUN mkdir $WORKDIR/build WORKDIR $WORKDIR/build # Compile and install +ENV BERINGEI_AUTOSETUP true RUN cmake .. -RUN make install +RUN make -j $(nproc --all) install RUN ./beringei/tools/beringei_configuration_generator --host_names localhost --file_path $WORKDIR/beringei.json -ENTRYPOINT $RUN_CMD \ No newline at end of file +ENTRYPOINT $RUN_CMD diff --git a/beringei/client/BeringeiClientImpl.cpp b/beringei/client/BeringeiClientImpl.cpp index f7fdf0d..cab64f8 100644 --- a/beringei/client/BeringeiClientImpl.cpp +++ b/beringei/client/BeringeiClientImpl.cpp @@ -10,9 +10,16 @@ #include "BeringeiClientImpl.h" #include + +#ifndef BERINGEI_AUTOSETUP #include -#include #include +#else +#include +#include +#endif + +#include #include #include "beringei/lib/GorillaStatsManager.h" @@ -634,8 +641,13 @@ BeringeiGetResult BeringeiClientImpl::get( GetDataRequest& request, const std::string& serviceOverride) { auto eb = BeringeiNetworkClient::getEventBase(); +#ifndef BERINGEI_AUTOSETUP return futureGet(request, eb, folly::getCPUExecutor().get(), serviceOverride) .getVia(eb); +#else + return futureGet(request, eb, wangle::getCPUExecutor().get(), serviceOverride) + .getVia(eb); +#endif } void BeringeiClientImpl::writeDataPointsForever(WriteClient* writeClient) { diff --git a/beringei/client/BeringeiClientImpl.h b/beringei/client/BeringeiClientImpl.h index 3b4c578..f864aa7 100644 --- a/beringei/client/BeringeiClientImpl.h +++ b/beringei/client/BeringeiClientImpl.h @@ -13,9 +13,15 @@ #include #include -#include #include + +#ifndef BERINGEI_AUTOSETUP +#include #include +#else +#include +#include +#endif #include "beringei/client/BeringeiConfigurationAdapterIf.h" #include "beringei/client/BeringeiGetResult.h" @@ -90,7 +96,11 @@ class BeringeiClientImpl { folly::Future futureGet( GetDataRequest& request, folly::EventBase* eb, +#ifndef BERINGEI_AUTOSETUP folly::Executor* workExecutor = folly::getCPUExecutor().get(), +#else + folly::Executor* workExecutor = wangle::getCPUExecutor().get(), +#endif const std::string& serviceOverride = ""); // Returns true if reading from gorilla is enabled, false otherwise. diff --git a/beringei/client/BeringeiGetResult.cpp b/beringei/client/BeringeiGetResult.cpp index 0e39c3d..318ff0a 100644 --- a/beringei/client/BeringeiGetResult.cpp +++ b/beringei/client/BeringeiGetResult.cpp @@ -12,7 +12,11 @@ #include #include +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif #include "beringei/lib/GorillaStatsManager.h" #include "beringei/lib/TimeSeries.h" diff --git a/beringei/client/BeringeiNetworkClient.cpp b/beringei/client/BeringeiNetworkClient.cpp index 4575f3f..bb65d10 100644 --- a/beringei/client/BeringeiNetworkClient.cpp +++ b/beringei/client/BeringeiNetworkClient.cpp @@ -126,8 +126,17 @@ vector BeringeiNetworkClient::performPut(PutRequestMap& requests) { } } else { auto exn = state.exception(); + +#ifndef BERINGEI_AUTOSETUP auto error = exn.what().toStdString(); LOG(ERROR) << "putDataPoints Failed. Reason: " << error; +#else + try { + std::rethrow_exception(exn); + } catch (const std::exception& ex) { + LOG(ERROR) << "putDataPoints Failed. Reason: " << ex.what(); + } +#endif std::lock_guard guard(droppedMutex); dropped.insert( @@ -199,8 +208,17 @@ void BeringeiNetworkClient::performGet(GetRequestMap& requests) { } } else { auto exn = state.exception(); + +#ifndef BERINGEI_AUTOSETUP auto error = exn.what().toStdString(); LOG(ERROR) << "getData failed. Reason: " << error; +#else + try { + std::rethrow_exception(exn); + } catch (const std::exception& ex) { + LOG(ERROR) << "getData failed. Reason: " << ex.what(); + } +#endif markRequestResultFailed( request.second.first, request.second.second); diff --git a/beringei/client/BeringeiNetworkClient.h b/beringei/client/BeringeiNetworkClient.h index 427a09e..75bda00 100644 --- a/beringei/client/BeringeiNetworkClient.h +++ b/beringei/client/BeringeiNetworkClient.h @@ -15,7 +15,13 @@ #include #include + +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif + #include "beringei/client/BeringeiConfigurationAdapterIf.h" #include "beringei/if/gen-cpp2/BeringeiService.h" diff --git a/beringei/lib/BucketMap.h b/beringei/lib/BucketMap.h index e4918c6..12806de 100644 --- a/beringei/lib/BucketMap.h +++ b/beringei/lib/BucketMap.h @@ -22,7 +22,11 @@ #include "PersistentKeyList.h" #include "Timer.h" +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif namespace facebook { namespace gorilla { diff --git a/beringei/lib/BucketStorage.cpp b/beringei/lib/BucketStorage.cpp index 836af46..5f03739 100644 --- a/beringei/lib/BucketStorage.cpp +++ b/beringei/lib/BucketStorage.cpp @@ -12,7 +12,12 @@ #include "GorillaStatsManager.h" #include "TimeSeriesStream.h" +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif + #include DEFINE_int32( diff --git a/beringei/lib/BucketStorage.h b/beringei/lib/BucketStorage.h index c4c27c2..7b4825b 100644 --- a/beringei/lib/BucketStorage.h +++ b/beringei/lib/BucketStorage.h @@ -20,7 +20,11 @@ #include "DataBlock.h" #include "DataBlockReader.h" +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif namespace facebook { namespace gorilla { diff --git a/beringei/lib/BucketedTimeSeries.h b/beringei/lib/BucketedTimeSeries.h index a4dbc6e..1dfaeaf 100644 --- a/beringei/lib/BucketedTimeSeries.h +++ b/beringei/lib/BucketedTimeSeries.h @@ -12,7 +12,12 @@ #include #include +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif + #include "BucketStorage.h" #include "TimeSeriesStream.h" #include "beringei/if/gen-cpp2/beringei_data_types.h" diff --git a/beringei/lib/CaseUtils.cpp b/beringei/lib/CaseUtils.cpp index 78dad46..762645e 100644 --- a/beringei/lib/CaseUtils.cpp +++ b/beringei/lib/CaseUtils.cpp @@ -10,7 +10,12 @@ #include "CaseUtils.h" #include + +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif namespace facebook { namespace gorilla { diff --git a/beringei/lib/DataBlockReader.cpp b/beringei/lib/DataBlockReader.cpp index 512e3e7..1098889 100644 --- a/beringei/lib/DataBlockReader.cpp +++ b/beringei/lib/DataBlockReader.cpp @@ -11,7 +11,12 @@ #include "BucketStorage.h" +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif + #include namespace facebook { diff --git a/beringei/lib/PersistentKeyList.cpp b/beringei/lib/PersistentKeyList.cpp index 1207de1..9cf050f 100644 --- a/beringei/lib/PersistentKeyList.cpp +++ b/beringei/lib/PersistentKeyList.cpp @@ -9,7 +9,12 @@ #include "PersistentKeyList.h" +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif + #include #include #include diff --git a/beringei/lib/ShardData.cpp b/beringei/lib/ShardData.cpp index 72cd075..5660eff 100644 --- a/beringei/lib/ShardData.cpp +++ b/beringei/lib/ShardData.cpp @@ -9,7 +9,12 @@ #include "ShardData.h" +#ifndef BERINGEI_AUTOSETUP #include +#else +#include +#endif + #include "beringei/lib/GorillaStatsManager.h" #include "beringei/lib/GorillaTimeConstants.h" diff --git a/setup_ubuntu.sh b/setup_ubuntu.sh index 25e6314..01661c5 100755 --- a/setup_ubuntu.sh +++ b/setup_ubuntu.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -FB_VERSION="2017.05.22.00" +FB_VERSION="2017.07.10.00" ZSTD_VERSION="1.1.1" echo "This script configures ubuntu with everything needed to run beringei." @@ -72,25 +72,28 @@ tar xzvf proxygen-${FB_VERSION}.tar.gz tar xzvf mstch-master.tar.gz tar xzvf zstd-${ZSTD_VERSION}.tar.gz +PROC=`cat /proc/cpuinfo | grep processor | wc -l` +echo ${PROC} + pushd mstch-master cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr/local/facebook-${FB_VERSION} . -make install +make -j ${PROC} install popd pushd zstd-${ZSTD_VERSION} -make install PREFIX=/usr/local/facebook-${FB_VERSION} +make -j ${PROC} install PREFIX=/usr/local/facebook-${FB_VERSION} popd pushd folly-${FB_VERSION}/folly autoreconf -ivf ./configure --prefix=/usr/local/facebook-${FB_VERSION} -make install +make -j ${PROC} install popd pushd wangle-${FB_VERSION}/wangle cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr/local/facebook-${FB_VERSION} -DBUILD_SHARED_LIBS:BOOL=ON . -make +make -j ${PROC} # Wangle tests are broken. Disabling ctest. # ctest make install @@ -99,11 +102,11 @@ popd pushd fbthrift-${FB_VERSION}/thrift autoreconf -ivf ./configure --prefix=/usr/local/facebook-${FB_VERSION} -make install +make -j ${PROC} install popd pushd proxygen-${FB_VERSION}/proxygen autoreconf -ivf ./configure --prefix=/usr/local/facebook-${FB_VERSION} -make install +make -j ${PROC} install popd