set_ubuntu.sh works on 17.10, Dockerfile builds

CMakeLists.txt

@@ -66,6 +66,10 @@ include_directories(${FOLLY_INCLUDE_DIR} ${FBTHRIFT_INCLUDE_DIR} ${PROXYGEN_INCL
 add_subdirectory(${TP_PROJECTS_DIR}/gtest)
 include_directories(${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS})
 
+if( $ENV{BERINGEI_AUTOSETUP} STREQUAL "true")
+    add_definitions("-DBERINGEI_AUTOSETUP")
+endif()
+
 enable_testing()
 add_subdirectory("beringei")
 

Dockerfile

@@ -1,5 +1,4 @@
-# Pull base image.
-FROM ubuntu:16.10
+FROM ubuntu:17.10
 
 ENV WORKDIR /usr/local/beringei
 ENV RUN_CMD ./beringei/service/beringei_main \
@@ -12,6 +11,8 @@ ENV RUN_CMD ./beringei/service/beringei_main \
               -logtostderr \
               -v=2
 
+RUN apt-get update
+RUN apt-get -y upgrade
 # Copy files from CircleCI into docker container.
 COPY . $WORKDIR
 
@@ -22,14 +23,22 @@ CMD ["bash"]
 WORKDIR $WORKDIR
 RUN $WORKDIR/setup_ubuntu.sh
 
+RUN apt-get install -y libmysqlclient20 libmysqlclient-dev libmysql++3v5 libmysqlcppconn7v5 libmysqlcppconn-dev libboost-all-dev libcap-dev libdouble-conversion-dev libevent-dev libgflags2.2 libgoogle-glog-dev libjemalloc-dev libkrb5-dev liblz4-dev liblzma-dev libnuma-dev libsasl2-dev libsnappy-dev libssl-dev zlib1g-dev
+
+# cmake is restricted to look in /usr/local/mysql, linking here
+RUN mkdir -p /usr/local/mysql/lib
+RUN ln -s /usr/lib/libmysqlpp.so* /usr/local/mysql/lib
+RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.* /usr/local/mysql/lib/
+
 # Create a build directory.
 RUN mkdir $WORKDIR/build
 WORKDIR $WORKDIR/build
 
 # Compile and install
+ENV BERINGEI_AUTOSETUP true
 RUN cmake ..
-RUN make install
+RUN make -j $(nproc --all) install
 
 RUN ./beringei/tools/beringei_configuration_generator --host_names localhost --file_path $WORKDIR/beringei.json
 
-ENTRYPOINT $RUN_CMD
+ENTRYPOINT $RUN_CMD

beringei/client/BeringeiClientImpl.cpp

@@ -10,9 +10,16 @@
 #include "BeringeiClientImpl.h"
 
 #include <folly/String.h>
+
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/container/Enumerate.h>
-#include <folly/gen/Base.h>
 #include <folly/synchronization/LifoSem.h>
+#else
+#include <folly/Enumerate.h>
+#include <folly/LifoSem.h>
+#endif
+
+#include <folly/gen/Base.h>
 #include <thrift/lib/cpp2/async/RequestChannel.h>
 
 #include "beringei/lib/GorillaStatsManager.h"
@@ -634,8 +641,13 @@ BeringeiGetResult BeringeiClientImpl::get(
     GetDataRequest& request,
     const std::string& serviceOverride) {
   auto eb = BeringeiNetworkClient::getEventBase();
+#ifndef BERINGEI_AUTOSETUP
   return futureGet(request, eb, folly::getCPUExecutor().get(), serviceOverride)
       .getVia(eb);
+#else
+  return futureGet(request, eb, wangle::getCPUExecutor().get(), serviceOverride)
+      .getVia(eb);
+#endif
 }
 
 void BeringeiClientImpl::writeDataPointsForever(WriteClient* writeClient) {

beringei/client/BeringeiClientImpl.h

@@ -13,9 +13,15 @@
 #include <vector>
 
 #include <folly/Executor.h>
-#include <folly/executors/GlobalExecutor.h>
 #include <folly/experimental/FunctionScheduler.h>
+
+#ifndef BERINGEI_AUTOSETUP
+#include <folly/executors/GlobalExecutor.h>
 #include <folly/synchronization/RWSpinLock.h>
+#else
+#include <wangle/concurrent/GlobalExecutor.h>
+#include <folly/RWSpinLock.h>
+#endif
 
 #include "beringei/client/BeringeiConfigurationAdapterIf.h"
 #include "beringei/client/BeringeiGetResult.h"
@@ -90,7 +96,11 @@ class BeringeiClientImpl {
   folly::Future<BeringeiGetResult> futureGet(
       GetDataRequest& request,
       folly::EventBase* eb,
+#ifndef BERINGEI_AUTOSETUP
       folly::Executor* workExecutor = folly::getCPUExecutor().get(),
+#else
+      folly::Executor* workExecutor = wangle::getCPUExecutor().get(),
+#endif
       const std::string& serviceOverride = "");
 
   // Returns true if reading from gorilla is enabled, false otherwise.

beringei/client/BeringeiGetResult.cpp

@@ -12,7 +12,11 @@
 #include <algorithm>
 #include <vector>
 
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/container/Enumerate.h>
+#else
+#include <folly/Enumerate.h>
+#endif
 
 #include "beringei/lib/GorillaStatsManager.h"
 #include "beringei/lib/TimeSeries.h"

beringei/client/BeringeiNetworkClient.cpp

@@ -126,8 +126,17 @@ vector<DataPoint> BeringeiNetworkClient::performPut(PutRequestMap& requests) {
                   }
                 } else {
                   auto exn = state.exception();
+
+#ifndef BERINGEI_AUTOSETUP
                   auto error = exn.what().toStdString();
                   LOG(ERROR) << "putDataPoints Failed. Reason: " << error;
+#else
+                  try {
+                      std::rethrow_exception(exn);
+                  } catch (const std::exception& ex) {
+                      LOG(ERROR) << "putDataPoints Failed. Reason: " << ex.what();
+                  }
+#endif
 
                   std::lock_guard<std::mutex> guard(droppedMutex);
                   dropped.insert(
@@ -199,8 +208,17 @@ void BeringeiNetworkClient::performGet(GetRequestMap& requests) {
             }
           } else {
             auto exn = state.exception();
+
+#ifndef BERINGEI_AUTOSETUP
             auto error = exn.what().toStdString();
             LOG(ERROR) << "getData failed. Reason: " << error;
+#else
+            try {
+                std::rethrow_exception(exn);
+            } catch (const std::exception& ex) {
+                LOG(ERROR) << "getData failed. Reason: " << ex.what();
+            }
+#endif
 
             markRequestResultFailed(
                 request.second.first, request.second.second);

beringei/client/BeringeiNetworkClient.h

@@ -15,7 +15,13 @@
 #include <vector>
 
 #include <folly/io/async/EventBaseManager.h>
+
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/synchronization/RWSpinLock.h>
+#else
+#include <folly/RWSpinLock.h>
+#endif
+
 #include "beringei/client/BeringeiConfigurationAdapterIf.h"
 #include "beringei/if/gen-cpp2/BeringeiService.h"
 

beringei/lib/BucketMap.h

@@ -22,7 +22,11 @@
 #include "PersistentKeyList.h"
 #include "Timer.h"
 
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/synchronization/RWSpinLock.h>
+#else
+#include <folly/RWSpinLock.h>
+#endif
 
 namespace facebook {
 namespace gorilla {

beringei/lib/BucketStorage.cpp

@@ -12,7 +12,12 @@
 #include "GorillaStatsManager.h"
 #include "TimeSeriesStream.h"
 
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/compression/Compression.h>
+#else
+#include <folly/io/Compression.h>
+#endif
+
 #include <folly/io/IOBuf.h>
 
 DEFINE_int32(

beringei/lib/BucketStorage.h

@@ -20,7 +20,11 @@
 #include "DataBlock.h"
 #include "DataBlockReader.h"
 
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/synchronization/RWSpinLock.h>
+#else
+#include <folly/RWSpinLock.h>
+#endif
 
 namespace facebook {
 namespace gorilla {

beringei/lib/BucketedTimeSeries.h

@@ -12,7 +12,12 @@
 #include <string>
 #include <vector>
 
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/synchronization/SmallLocks.h>
+#else
+#include <folly/SmallLocks.h>
+#endif
+
 #include "BucketStorage.h"
 #include "TimeSeriesStream.h"
 #include "beringei/if/gen-cpp2/beringei_data_types.h"

beringei/lib/CaseUtils.cpp

@@ -10,7 +10,12 @@
 #include "CaseUtils.h"
 
 #include <folly/String.h>
+
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/hash/SpookyHashV2.h>
+#else
+#include <folly/SpookyHashV2.h>
+#endif
 
 namespace facebook {
 namespace gorilla {

beringei/lib/DataBlockReader.cpp

@@ -11,7 +11,12 @@
 
 #include "BucketStorage.h"
 
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/compression/Compression.h>
+#else
+#include <folly/io/Compression.h>
+#endif
+
 #include <folly/io/IOBuf.h>
 
 namespace facebook {

beringei/lib/PersistentKeyList.cpp

@@ -9,7 +9,12 @@
 
 #include "PersistentKeyList.h"
 
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/compression/Compression.h>
+#else
+#include <folly/io/Compression.h>
+#endif
+
 #include <folly/io/IOBuf.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>

beringei/lib/ShardData.cpp

@@ -9,7 +9,12 @@
 
 #include "ShardData.h"
 
+#ifndef BERINGEI_AUTOSETUP
 #include <folly/container/Enumerate.h>
+#else
+#include <folly/Enumerate.h>
+#endif
+
 #include "beringei/lib/GorillaStatsManager.h"
 #include "beringei/lib/GorillaTimeConstants.h"
 

setup_ubuntu.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-FB_VERSION="2017.05.22.00"
+FB_VERSION="2017.07.10.00"
 ZSTD_VERSION="1.1.1"
 
 echo "This script configures ubuntu with everything needed to run beringei."
@@ -72,25 +72,28 @@ tar xzvf proxygen-${FB_VERSION}.tar.gz
 tar xzvf mstch-master.tar.gz
 tar xzvf zstd-${ZSTD_VERSION}.tar.gz
 
+PROC=`cat /proc/cpuinfo | grep processor | wc -l`
+echo ${PROC}
+
 pushd mstch-master
 cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr/local/facebook-${FB_VERSION} .
-make install
+make -j ${PROC} install
 popd
 
 pushd zstd-${ZSTD_VERSION}
-make install PREFIX=/usr/local/facebook-${FB_VERSION}
+make -j ${PROC} install PREFIX=/usr/local/facebook-${FB_VERSION}
 popd
 
 
 pushd folly-${FB_VERSION}/folly
 autoreconf -ivf
 ./configure --prefix=/usr/local/facebook-${FB_VERSION}
-make install
+make -j ${PROC} install
 popd
 
 pushd wangle-${FB_VERSION}/wangle
 cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr/local/facebook-${FB_VERSION} -DBUILD_SHARED_LIBS:BOOL=ON .
-make
+make -j ${PROC}
 # Wangle tests are broken. Disabling ctest.
 # ctest
 make install
@@ -99,11 +102,11 @@ popd
 pushd fbthrift-${FB_VERSION}/thrift
 autoreconf -ivf
 ./configure --prefix=/usr/local/facebook-${FB_VERSION}
-make install
+make -j ${PROC} install
 popd
 
 pushd proxygen-${FB_VERSION}/proxygen
 autoreconf -ivf
 ./configure --prefix=/usr/local/facebook-${FB_VERSION}
-make install
+make -j ${PROC} install
 popd