Merge branch 'main' of github.com:kgpai/velox-1 into agg_fuzzer_query…

…_runner

.circleci/dist_compile.yml

@@ -302,6 +302,7 @@ jobs:
                 --velox_fuzzer_enable_expression_reuse \
                 --max_expression_trees_per_step 2 \
                 --retry_with_try \
+                --enable_dereference \
                 --logtostderr=1 \
                 --minloglevel=0 \
                 --repro_persist_path=/tmp/spark_fuzzer_repro \

.github/workflows/docker.yml

@@ -38,7 +38,33 @@ permissions:
 
 jobs:
   linux:
+    name: "Build and Push ${{ matrix.name }}"
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Check
+            file: "scripts/check-container.dockfile"
+            args: "cpu_target=avx"
+            tags: "ghcr.io/facebookincubator/velox-dev:check-avx"
+          - name: CircleCI
+            file: "scripts/circleci-container.dockfile"
+            args: "cpu_target=avx"
+            tags: "ghcr.io/facebookincubator/velox-dev:circleci-avx"
+          - name: Torcharrow
+            file: "scripts/velox-torcharrow-container.dockfile"
+            args: "cpu_target=avx"
+            tags: "ghcr.io/facebookincubator/velox-dev:torcharrow-avx"
+          - name: Dev
+            file: "scripts/ubuntu-22.04-cpp.dockerfile"
+            args: ""
+            tags: "ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx"
+          - name: Presto Java
+            file: "scripts/prestojava-container.dockerfile"
+            args: "PRESTO_VERSION=0.284"
+            tags: "ghcr.io/facebookincubator/velox-dev:presto-java"
+
     steps:
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -53,38 +79,10 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - uses: actions/checkout@v3
-
-      - name: Build and Push check
-        uses: docker/build-push-action@v3
-        with:
-          context: scripts
-          file: scripts/check-container.dockfile
-          build-args: cpu_target=avx
-          push: ${{ github.repository == 'facebookincubator/velox' && github.event_name != 'pull_request'}}
-          tags: ghcr.io/facebookincubator/velox-dev:check-avx
-
-      - name: Build and Push circle-ci
-        uses: docker/build-push-action@v3
-        with:
-          context: scripts
-          file: scripts/circleci-container.dockfile
-          build-args: cpu_target=avx
-          push: ${{ github.repository == 'facebookincubator/velox' && github.event_name != 'pull_request'}}
-          tags: ghcr.io/facebookincubator/velox-dev:circleci-avx
-
-      - name: Build and Push velox-torcharrow
-        uses: docker/build-push-action@v3
-        with:
-          context: scripts
-          file: scripts/velox-torcharrow-container.dockfile
-          build-args: cpu_target=avx
-          push: ${{ github.repository == 'facebookincubator/velox' && github.event_name != 'pull_request'}}
-          tags: ghcr.io/facebookincubator/velox-dev:torcharrow-avx
-
-      - name: Build and Push dev-image
+      - name: Build and Push
         uses: docker/build-push-action@v3
         with:
-          file: scripts/ubuntu-22.04-cpp.dockerfile
+          file: "${{ matrix.file }}"
+          build-args: "${{ matrix.args }}"
           push: ${{ github.repository == 'facebookincubator/velox' && github.event_name != 'pull_request'}}
-          tags: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx
+          tags: "${{ matrix.tags }}"

.github/workflows/experimental.yml

@@ -109,7 +109,7 @@ jobs:
         with:
           name: aggregation
 
-      - name: "Run Aggregate Fuzzer"
+      - name: "Run Aggregate Fuzzer with Presto as source of truth"
         run: |
           /opt/start-prestojava.sh > /tmp/prestjava.log 2>&1 &  
           mkdir -p /tmp/aggregate_fuzzer_repro/

.github/workflows/scheduled.yml

@@ -188,6 +188,7 @@ jobs:
                 --velox_fuzzer_enable_expression_reuse \
                 --max_expression_trees_per_step 2 \
                 --retry_with_try \
+                --enable_dereference \
                 --duration_sec 1800 \
                 --logtostderr=1 \
                 --minloglevel=0 \
@@ -284,7 +285,7 @@ jobs:
         with:
           name: join
 
-      - name: "Run Aggregate Fuzzer"
+      - name: "Run Join Fuzzer"
         run: |
           mkdir -p /tmp/join_fuzzer_repro/
           rm -rfv /tmp/join_fuzzer_repro/*

CMake/resolve_dependency_modules/re2.cmake

@@ -44,7 +44,6 @@ endif()
 
 set(re2_LIBRARIES ${re2_BINARY_DIR}/libre2.a)
 set(re2_INCLUDE_DIRS ${re2_SOURCE_DIR})
-add_library(re2::re2 ALIAS re2)
 
 set(RE2_ROOT ${re2_BINARY_DIR})
 set(re2_ROOT ${re2_BINARY_DIR})

CMakeLists.txt

@@ -238,7 +238,8 @@ if(VELOX_ENABLE_ABFS)
   if(AZURESDK_ROOT_DIR)
     list(APPEND CMAKE_PREFIX_PATH ${AZURESDK_ROOT_DIR})
   endif()
-  find_package(azure-storage-blobs-cpp CONFIG REQUIRED)
+  # files-datalake is built on blobs
+  find_package(azure-storage-files-datalake-cpp CONFIG REQUIRED)
   add_definitions(-DVELOX_ENABLE_ABFS)
 endif()
 

build/deps/github_hashes/facebook/folly-rev.txt

@@ -1 +1 @@
-Subproject commit 02849dd4bb4553dfd3a8b455dfc52379d6346429
+Subproject commit 72fbaa06b93b6f8aec66ba49d3479acb46ccdf16

build/fbcode_builder/getdeps/builder.py

@@ -113,7 +113,7 @@ def _apply_patchfile(self) -> None:
         patchfile = os.path.join(
             self.build_opts.fbcode_builder_dir, "patches", self.patchfile
         )
-        patchcmd = ["git", "apply"]
+        patchcmd = ["git", "apply", "--ignore-space-change"]
         if self.patchfile_opts:
             patchcmd.append(self.patchfile_opts)
         try:
@@ -364,27 +364,12 @@ def __init__(self, build_opts, ctx, manifest, src_dir, build_dir, inst_dir) -> N
             build_opts, ctx, manifest, src_dir, build_dir, inst_dir
         )
 
-    def _patch(self) -> None:
-        # FBOSS build currently depends on an old version of iproute2 (commit
-        # 7ca63aef7d1b0c808da0040c6b366ef7a61f38c1). This is missing a commit
-        # (ae717baf15fb4d30749ada3948d9445892bac239) needed to build iproute2
-        # successfully. Apply it viz.: include stdint.h
-        # Reference: https://fburl.com/ilx9g5xm
-        with open(self.build_dir + "/tc/tc_core.c", "r") as f:
-            data = f.read()
-
-        with open(self.build_dir + "/tc/tc_core.c", "w") as f:
-            f.write("#include <stdint.h>\n")
-            f.write(data)
-
     def _build(self, install_dirs, reconfigure) -> None:
         configure_path = os.path.join(self.src_dir, "configure")
-
         env = self.env.copy()
         self._run_cmd([configure_path], env=env)
         shutil.rmtree(self.build_dir)
         shutil.copytree(self.src_dir, self.build_dir)
-        self._patch()
         self._run_cmd(["make", "-j%s" % self.num_jobs], env=env)
         install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir]
 

build/fbcode_builder/manifests/iproute2

@@ -8,6 +8,7 @@ sha256 = 46612a1e2d01bb31932557bccdb1b8618cae9a439dfffc08ef35ed8e197f14ce
 [build.os=linux]
 builder = iproute2
 subdir = iproute2-4.12.0
+patchfile = iproute2_oss.patch
 
 [build.not(os=linux)]
 builder = nop

build/fbcode_builder/patches/iproute2_oss.patch

@@ -0,0 +1,36 @@
+diff --git a/bridge/fdb.c b/bridge/fdb.c
+--- a/bridge/fdb.c
++++ b/bridge/fdb.c
+@@ -31,7 +31,7 @@
+
+ static unsigned int filter_index, filter_vlan, filter_state;
+
+-json_writer_t *jw_global;
++static json_writer_t *jw_global;
+
+ static void usage(void)
+ {
+diff --git a/ip/ipmroute.c b/ip/ipmroute.c
+--- a/ip/ipmroute.c
++++ b/ip/ipmroute.c
+@@ -44,7 +44,7 @@
+        exit(-1);
+ }
+
+-struct rtfilter {
++static struct rtfilter {
+        int tb;
+        int af;
+        int iif;
+diff --git a/ip/xfrm_monitor.c b/ip/xfrm_monitor.c
+--- a/ip/xfrm_monitor.c
++++ b/ip/xfrm_monitor.c
+@@ -34,7 +34,7 @@
+ #include "ip_common.h"
+
+ static void usage(void) __attribute__((noreturn));
+-int listen_all_nsid;
++static int listen_all_nsid;
+
+ static void usage(void)
+ {

docker-compose.yml

@@ -69,3 +69,24 @@ services:
     volumes:
       - .:/velox:delegated
     command: cd /velox && make python-test
+
+
+  presto-java:
+  # Usage:
+  #   docker-compose pull presto-java or docker-compose build presto-java
+  #   docker-compose run --rm presto-java
+  #   or
+  #   docker-compose run -e NUM_THREADS=<NUMBER_OF_THREADS_TO_USE> --rm presto-java
+  #   to set the number of threads used during compilation
+    image: ghcr.io/facebookincubator/velox-dev:presto-java
+    build:
+      args:
+        - PRESTO_VERSION=0.284
+      context: .
+      dockerfile: scripts/prestojava-container.dockerfile
+    environment:
+      NUM_THREADS: 8 # default value for NUM_THREADS
+      CCACHE_DIR: "/velox/.ccache"
+    volumes:
+      - .:/velox:delegated
+    command: /bin/bash -c "scl enable gcc-toolset-9 '/velox/scripts/docker-command.sh'"

scripts/check-container.dockfile

@@ -13,6 +13,6 @@
 # limitations under the License.
 FROM amd64/ubuntu:22.04
 ARG cpu_target
-COPY setup-check.sh /root
-COPY setup-helper-functions.sh /
+COPY scripts/setup-check.sh /root
+COPY scripts/setup-helper-functions.sh /
 RUN CPU_TARGET="$cpu_target" bash /root/setup-check.sh

scripts/circleci-container.dockfile

@@ -15,6 +15,6 @@
 #
 FROM quay.io/centos/centos:stream8
 ARG cpu_target
-COPY setup-circleci.sh /
-COPY setup-helper-functions.sh /
+COPY scripts/setup-circleci.sh /
+COPY scripts/setup-helper-functions.sh /
 RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-circleci.sh ) && rm -rf build

scripts/etc/config.properties.example

@@ -0,0 +1,5 @@
+coordinator=true
+node-scheduler.include-coordinator=true
+http-server.http.port=8080
+discovery-server.enabled=true
+discovery.uri=http://localhost:8080

scripts/etc/jvm.config.example

@@ -0,0 +1,9 @@
+-server
+-Xmx1G
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+UseGCOverheadLimit
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
+-Djdk.attach.allowAttachSelf=true

scripts/etc/node.properties

@@ -0,0 +1,2 @@
+node.environment=test
+node.data-dir=/var/lib/presto/data

scripts/prestojava-container.dockerfile

@@ -0,0 +1,47 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Build the test and build container for presto_cpp
+#
+FROM quay.io/centos/centos:stream8
+
+ARG PRESTO_VERSION
+
+ADD scripts /velox/scripts/
+RUN /velox/scripts/setup-centos8.sh
+RUN wget https://repo1.maven.org/maven2/com/facebook/presto/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz
+RUN wget https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar
+
+ARG PRESTO_PKG=presto-server-$PRESTO_VERSION.tar.gz
+ARG PRESTO_CLI_JAR=presto-cli-$PRESTO_VERSION-executable.jar
+
+ENV PRESTO_HOME="/opt/presto-server"
+RUN cp $PRESTO_CLI_JAR /opt/presto-cli
+
+RUN dnf install -y java-11-openjdk less procps python3 \
+    && ln -s $(which python3) /usr/bin/python \
+    && tar -zxf $PRESTO_PKG \
+    && mv ./presto-server-$PRESTO_VERSION $PRESTO_HOME \
+    && chmod +x /opt/presto-cli \
+    && ln -s /opt/presto-cli /usr/local/bin/ \
+    && mkdir -p $PRESTO_HOME/etc \
+    && mkdir -p $PRESTO_HOME/etc/catalog \
+    && mkdir -p /var/lib/presto/data \
+    && mkdir -p /usr/lib/presto/utils
+
+COPY scripts/etc/config.properties.example $PRESTO_HOME/etc/config.properties
+COPY scripts/etc/jvm.config.example $PRESTO_HOME/etc/jvm.config
+COPY scripts/etc/node.properties $PRESTO_HOME/etc/node.properties
+COPY scripts/start-prestojava.sh /opt
+
+WORKDIR /velox

scripts/setup-adapters.sh

@@ -62,7 +62,9 @@ function install_gcs-sdk-cpp {
 }
 
 function install_azure-storage-sdk-cpp {
-  github_checkout azure/azure-sdk-for-cpp azure-storage-blobs_12.8.0
+  vcpkg_commit_id=7a6f366cefd27210f6a8309aed10c31104436509
+  github_checkout azure/azure-sdk-for-cpp azure-storage-files-datalake_12.8.0
+  sed -i "s/set(VCPKG_COMMIT_STRING .*)/set(VCPKG_COMMIT_STRING $vcpkg_commit_id)/" cmake-modules/AzureVcpkg.cmake
 
   cd sdk/core/azure-core
   if ! grep -q "baseline" vcpkg.json; then
@@ -71,7 +73,7 @@ function install_azure-storage-sdk-cpp {
     if [[ "$openssl_version" == 1.1.1* ]]; then
       openssl_version="1.1.1n"
     fi
-    sed -i 's/"version-string"/"builtin-baseline": "dafef74af53669ef1cc9015f55e0ce809ead62aa","version-string"/' vcpkg.json
+    sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" vcpkg.json
     sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" vcpkg.json
   fi
   cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF
@@ -84,9 +86,11 @@ function install_azure-storage-sdk-cpp {
   cd -
   # install azure-storage-blobs
   cd sdk/storage/azure-storage-blobs
-  if ! grep -q "baseline" vcpkg.json; then
-    sed -i 's/"version-semver"/"builtin-baseline": "dafef74af53669ef1cc9015f55e0ce809ead62aa","version-semver"/' vcpkg.json
-  fi
+  cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF
+
+  cd -
+  # install azure-storage-files-datalake
+  cd sdk/storage/azure-storage-files-datalake
   cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF
 }
 

scripts/start-prestojava.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+echo "node.id=$HOSTNAME" >> $PRESTO_HOME/etc/node.properties
+
+$PRESTO_HOME/bin/launcher run

scripts/velox-torcharrow-container.dockfile

@@ -16,6 +16,6 @@
 
 FROM quay.io/pypa/manylinux2014_x86_64
 ARG cpu_target
-COPY setup-velox-torcharrow.sh /
-COPY setup-helper-functions.sh /
+COPY scripts/setup-velox-torcharrow.sh /
+COPY scripts/setup-helper-functions.sh /
 RUN mkdir build && ( cd build && CPU_TARGET="$cpu_target" bash /setup-velox-torcharrow.sh ) && rm -rf build