Dremio Stress JDBC support (#27)

JDBC Support * jdk8 compatible and directions * cleaner error handling * updated Docker image * updated docs
rsvihladremio · Oct 19, 2023 · edc2ec1 · edc2ec1
1 parent 4971c1d
commit edc2ec1
Show file tree

Hide file tree

Showing 55 changed files with 2,806 additions and 2,088 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,34 +1,57 @@
-# If you prefer the allow list template instead of the deny list, see community template:
-# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
-#
-# Binaries for programs and plugins
-*.exe
-*.exe~
-*.dll
-*.so
-*.dylib
-
-# Test binary, built with `go test -c`
-*.test
-
-# Output of the go coverage tool, specifically when used with LiteIDE
-*.out
-
-# Dependency directories (remove the comment below to include it)
-# vendor/
-
-# Go workspace file
-go.work
-
-# ignore the binaries folder
-/bin
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+replay_pid*
+
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+# https://github.com/takari/maven-wrapper#usage-without-binary-jar
+.mvn/wrapper/maven-wrapper.jar
+
+# Eclipse m2e generated files
+# Eclipse Core
+.project
+# JDT-specific (Eclipse Java Development Tools)
+.classpath
+.factorypath
+.settings
 
 # code coverage output file that is often left behind
 coverage.out
 
 # vscode files
 .vscode
 
+# intellij
+
+.idea
+
 # mac directory files
 
 .DS_Store
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.4/apache-maven-3.9.4-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar
diff --git a/Dockerfile b/Dockerfile
@@ -1,32 +1,9 @@
-FROM --platform=linux/amd64 debian:stable-slim
-ARG VERSION=unknown
-ARG GIT_SHA=unknown
-WORKDIR /app
+FROM eclipse-temurin:11
 
-COPY . .
+RUN mkdir -p /app
+WORKDIR /app
+COPY . /app
 
-RUN mkdir build &&  \
-    cd build && \
-    apt update -y && \
-    apt upgrade -y && \ 
-    apt install -y wget odbcinst unixodbc unixodbc-dev alien && \
-    wget https://go.dev/dl/go1.21.3.linux-amd64.tar.gz && \
-    tar -C /usr/local -xzf go1.21.3.linux-amd64.tar.gz && \
-    ln -s /usr/local/go/bin/go /usr/local/bin/go && \
-    wget https://download.dremio.com/arrow-flight-sql-odbc-driver/arrow-flight-sql-odbc-driver-LATEST.x86_64.rpm && \
-    alien --scripts $(ls ./arrow-flight-sql-odbc-driver-*) && \
-    apt install -y $(ls ./arrow-flight-sql-odbc-driver*.deb) && \
-    rm $(ls ./arrow-flight-sql-odbc-driver-*) && \
-    cd /usr/lib/x86_64-linux-gnu/ && \
-    ln -s libodbcinst.so.2.0.0 libodbcinst.so.1 && \
-    ln -s libodbc.so.2.0.0 libodbc.so.1 && \
-    ldconfig && \
-    cd /app && \
-    apt remove -y wget && \
-    rm -fr build && \
-    go build -tags odbc -ldflags "-X github.com/rsvihladremio/dremio-stress/cmd.GitSha=${GIT_SHA} -X github.com/rsvihladremio/dremio-stress/cmd.Version=${VERSION}" -o /usr/bin/dremio-stress . && \
-    apt clean all && \
-    rm -rf /var/cache/yum && \
-    rm -fr /app
+RUN ./mvnw validate && ./mvnw package -Pnative -DskipTests=true && echo "#!/usr/bin/env bash\n_JAVA_OPTIONS='--add-opens=java.base/java.nio=ALL-UNNAMED' java -jar $(ls ./target/dremio-stress-*-jar-with-dependencies.jar) \"\$@\"" >> /usr/bin/dremio-stress && chmod +x /usr/bin/dremio-stress
 
-ENTRYPOINT ["/usr/bin/dremio-stress"]
+CMD ["bash", "/usr/bin/dremio-stress"]
diff --git a/README.md b/README.md
@@ -1,39 +1,26 @@
 # dremio-stress
 
-Simple tool to stress Dremio via ODBC and REST interfaces
+Simple tool to stress Dremio via JDBC and REST interfaces written for Java 8 (but works with 17)
 
 ## Run via the REST interface
 
 ```bash
-dremio-stress -user dremio -password dremio123 -url http://localhost:9047 -conf ./stress.json
+java -jar dremio-stress.jar -u dremio  -p dremio123 -l http://localhost:9047 ./stress.json
 ```
 
-## Run via ODBC 
+## Run via JDBC
 
 
 ### Run via Docker
 
 ```bash
-docker run -it -v $(pwd):/mnt ghcr.io/rsvihladremio/dremio-stress -protocol odbc -user dremio -password dremio123 -url "Driver={Arrow Flight SQL ODBC Driver};ConnectionType=Direct;AuthenticationType=Plain;Host=host.docker.internal;Port=32010;useEncryption=false"  -conf /mnt/stress.json
+docker run -it -v $(pwd):/mnt ghcr.io/rsvihladremio/dremio-stress dremio-stress --protocol JDBC -l "jdbc:arrow-flight-sql://host.docker.internal:32010/?useEncryption=false&user=dremio&password=dremio123"  /mnt/stress.json
 ```
 
-### Run ODBC Directly With a Binary
-
-#### Dependencies on Windows
-
-* Dremio ODBC Driver
-* X86\_64 (or AMD64) processor
-
-#### Dependencies on Mac and Linux
-
-* UnixODBC
-* Dremio ODBC Driver
-* X86\_64 (or AMD64) processor
-* Apple Silicon not supported use the Docker option
-
+### Run JDBC Directly With a Binary
 
 ```bash
-dremio-stress -protocol odbc -user dremio -password dremio123 -url  "Driver={Arrow Flight SQL ODBC Driver};ConnectionType=Direct;AuthenticationType=Plain;Host=localhost;Port=32010;useEncryption=false" -conf ./stress.json
+java -jar dremio-stress.jar --protocol JDBC "jdbc:arrow-flight-sql://localhost:32010/?useEncryption=false&user=dremio&password=dremio" ./stress.json
 ```
 
 ## Example stress.json files
@@ -42,81 +29,83 @@ dremio-stress -protocol odbc -user dremio -password dremio123 -url  "Driver={Arr
 
 ```json
 {
-  "queries": [
-    {
-      "query": "select * FROM Samples.\"samples.dremio.com\".\"SF weather 2018-2019.csv LIMIT 50\"",
-      "frequency": 5
-    },
-    {
-      "query": "select * FROM Samples.\"samples.dremio.com\".\"SF weather 2018-2019.csv\" where \"DATE\" between ':start' and ':end'",
-      "frequency": 5,
-      "parameters": {
-        "start": ["2018-02-04","2018-02-05"],
-        "end": ["2018-02-14","2018-02-15"]
-      }
-    }
-  ]
+"queries": [
+	{
+	"query": "select * FROM Samples.\"samples.dremio.com\".\"SF weather 2018-2019.csv LIMIT 50\"",
+	"frequency": 5
+	},
+	{
+	"query": "select * FROM Samples.\"samples.dremio.com\".\"SF weather 2018-2019.csv\" where \"DATE\" between ':start' and ':end'",
+	"frequency": 5,
+	"parameters": {
+		"start": ["2018-02-04","2018-02-05"],
+		"end": ["2018-02-14","2018-02-15"]
+	}
+	}
+]
 }
 ```
 
 
-### Using queryGroups to preform several ops in order, the "schemops" group  will be called roughly 10% of the time
+### Using queryGroups to preform several ops in order, the "schemaops" group  will be called roughly 10% of the time
 
 ```json
 {
-  "queryGroups": [
-    {
-      "name": "schemaops",
-      "queries": [
-        "drop table if exists samples.\"samples.dremio.com\".\"A\"",
-        "create table samples.\"samples.dremio.com\".\"A\" STORE AS (type => 'iceberg') AS SELECT \"a\",\"b\" FROM (values('a', 'b')) as t(\"a\",\"b\")",
-        "select * from  samples.\"samples.dremio.com\".\"A\""
-      ]
-    }
-  ],
-  "queries": [
-    {
-      "queryGroup": "schemaops",
-      "frequency": 1
-    },
-    {
-      "query": "select * FROM Samples.\"samples.dremio.com\".\"SF weather 2018-2019.csv\" where \"DATE\" between ':start' and ':end'",
-      "frequency": 9,
-      "parameters": {
-          "start": ["2018-02-04", "2018-02-05"],
-          "end": ["2018-02-14","2018-02-15"]
-      }
-    }
-  ]
+"queryGroups": [
+	{
+	"name": "schemaops",
+	"queries": [
+		"drop table if exists samples.\"samples.dremio.com\".\"A\"",
+		"create table samples.\"samples.dremio.com\".\"A\" STORE AS (type => 'iceberg') AS SELECT \"a\",\"b\" FROM (values('a', 'b')) as t(\"a\",\"b\")",
+		"select * from  samples.\"samples.dremio.com\".\"A\""
+	]
+	}
+],
+"queries": [
+	{
+	"queryGroup": "schemaops",
+	"frequency": 1
+	},
+	{
+	"query": "select * FROM Samples.\"samples.dremio.com\".\"SF weather 2018-2019.csv\" where \"DATE\" between ':start' and ':end'",
+	"frequency": 9,
+	"parameters": {
+		"start": ["2018-02-04", "2018-02-05"],
+		"end": ["2018-02-14","2018-02-15"]
+	}
+	}
+]
 }
 ```
 
 
 ## Flags
 
 ```bash
-  -conf string
-    	location of the stress.json to define the stress job. If one is not provided a default stress job is used (default "stress.json")
-  -duration duration
-    	duration of dremio-stress run (default 10m0s)
-  -max-concurrency int
-    	max number of concurrent queries (default 4)
-  -password string
-    	password to use at login (default "dremio123")
-  -protocol string
-    	communication protocol to use for stress http or odbc are available (default "http")
-  -skip-ssl
-    	works with '-protocol http' when using https
-  -timeout duration
-    	default timeout for queries (default 1m0s)
-  -url string
-    	http(s) URL used for '-protocol http' or a odbc connection string for '-protocol odbc' (default "http://localhost:9047")
-  -user string
-    	user to use at login (default "dremio")
-  -v	add more verbose output
+Usage: java -jar dremio-stress.jar [-sv] [-d=<durationSeconds>] [-l=<dremioUrl>] [-p=<dremioHttpPassword>] [--protocol=<protocol>] [-q=<maxQueriesInFlight>] [-t=<httpTimeoutSeconds>] [-u=<dremioHttpUser>]
+ <jsonConfig> [COMMAND]
+using a defined JSON run a series of queries against dremio using various approaches
+      <jsonConfig>        The file to use for stress definitions
+  -d, --duration-seconds=<durationSeconds>
+                          duration in seconds to run stress
+  -l, --url=<dremioUrl>   JDBC connection string or HTTP url to connect
+  -p, --http-password=<dremioHttpPassword>
+                          the password of the user used to submit HTTP queries
+      --protocol=<protocol>
+                          protocol to use HTTP or JDBC
+  -q, --max-queries-in-flight=<maxQueriesInFlight>
+                          max number of queries in flight (if possible)
+  -s, --http-skip-ssl-verification
+                          whether to skip ssl verification for HTTP queries or not
+  -t, --http-timeout-seconds=<httpTimeoutSeconds>
+                          HTTP timeout for queries
+  -u, --http-user=<dremioHttpUser>
+                          the user used to submit HTTP queries
+  -v, --verbose           -v for info, -vv for debug, -vvv for trace
 ```
 
 ## Running Tests
 
 * Install docker or have it running
+* JDK 8+
 * ./script/test