scylladb · CodeLieutenant · Oct 23, 2024 · Oct 25, 2024 · Oct 25, 2024 · Nov 14, 2024
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+version: 2
+
+updates:
+  - package-ecosystem: "github-actions"
+    directory: /
+    schedule:
+      interval: "monthly"
+
+  - package-ecosystem: "gomod"
+    directory: /
+    schedule:
+      interval: "weekly"
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -29,9 +29,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        gemini-features: ["basic", "normal", "all"]
+        gemini-features: ["basic", "normal"]
         gemini-concurrency: [4]
-        duration: ["5m"]
+        duration: ["1m"]
         dataset-size: [large, small]
         oracle-scylla-version: ["6.1"]
         test-scylla-version: ["6.2"]
@@ -54,6 +54,7 @@ jobs:
             CONCURRENCY=${{ matrix.gemini-concurrency }} \
             CQL_FEATURES=${{ matrix.gemini-features }} \
             DURATION=${{ matrix.duration }} \
+            WARMUP=30s \
             DATASET_SIZE=${{ matrix.dataset-size }} \
       - name: Shutdown ScyllaDB
         shell: bash

diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,4 @@ cmd/gemini/dist/
 bin/
 coverage.txt
 dist/
+results/*.log
diff --git a/.run/Run Gemini Mixed.run.xml b/.run/Run Gemini Mixed.run.xml
@@ -0,0 +1,13 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Run Gemini Mixed" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="gemini" />
+    <working_directory value="$PROJECT_DIR$" />
+    <go_parameters value="-gcflags &quot;all=-N -l&quot;" />
+    <parameters value="--mode mixed --seed=150 --schema-seed=150 --dataset-size=large --cql-features normal --warmup 0 --duration 2m --drop-schema true --fail-fast --level info --test-host-selection-policy token-aware --oracle-host-selection-policy token-aware --test-cluster=192.168.100.2 --oracle-cluster=192.168.100.3 --non-interactive --request-timeout 180s --connect-timeout 120s --use-server-timestamps false --async-objects-stabilization-attempts 10 --async-objects-stabilization-backoff 100ms --replication-strategy &quot;{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}&quot; --oracle-replication-strategy &quot;{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}&quot; --max-mutation-retries 5 --max-mutation-retries-backoff 1000ms --concurrency 16 --profiling-port 6060" />
+    <kind value="PACKAGE" />
+    <package value="github.com/scylladb/gemini/cmd/gemini" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$/cmd/gemini/main.go" />
+    <method v="2" />
+  </configuration>
+</component>
diff --git a/.run/Run Gemini Read.run.xml b/.run/Run Gemini Read.run.xml
@@ -0,0 +1,13 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Run Gemini Read" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="gemini" />
+    <working_directory value="$PROJECT_DIR$" />
+    <go_parameters value="-gcflags &quot;all=-N -l&quot;" />
+    <parameters value="--mode read --seed=100 --schema-seed=100 --dataset-size=small --cql-features normal --warmup 0 --duration 2m --drop-schema true --fail-fast --level info --test-host-selection-policy token-aware --oracle-host-selection-policy token-aware --test-cluster=192.168.100.2 --oracle-cluster=192.168.100.3 --non-interactive --request-timeout 180s --connect-timeout 120s --use-server-timestamps false --async-objects-stabilization-attempts 10 --async-objects-stabilization-backoff 100ms --replication-strategy &quot;{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}&quot; --oracle-replication-strategy &quot;{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}&quot; --max-mutation-retries 5 --max-mutation-retries-backoff 1000ms --concurrency 1 --profiling-port 6060" />
+    <kind value="PACKAGE" />
+    <package value="github.com/scylladb/gemini/cmd/gemini" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$/cmd/gemini/main.go" />
+    <method v="2" />
+  </configuration>
+</component>
diff --git a/.run/Run Gemini Write.run.xml b/.run/Run Gemini Write.run.xml
@@ -0,0 +1,13 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Run Gemini Write" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="gemini" />
+    <working_directory value="$PROJECT_DIR$" />
+    <go_parameters value="-gcflags &quot;all=-N -l&quot;" />
+    <parameters value="--mode write --seed=100 --schema-seed=100 --dataset-size=small --cql-features normal --warmup 0 --duration 2m --drop-schema true --fail-fast --level info --test-host-selection-policy token-aware --oracle-host-selection-policy token-aware --test-cluster=192.168.100.2 --oracle-cluster=192.168.100.3 --non-interactive --request-timeout 180s --connect-timeout 120s --use-server-timestamps false --async-objects-stabilization-attempts 10 --async-objects-stabilization-backoff 100ms --replication-strategy &quot;{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}&quot; --oracle-replication-strategy &quot;{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}&quot; --max-mutation-retries 5 --max-mutation-retries-backoff 1000ms --concurrency 1 --profiling-port 6060" />
+    <kind value="PACKAGE" />
+    <package value="github.com/scylladb/gemini/cmd/gemini" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$/cmd/gemini/main.go" />
+    <method v="2" />
+  </configuration>
+</component>
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,4 @@
 FROM golang:1.23-bookworm AS build
-
 ENV GO111MODULE=on
 ENV GOAMD64=v3
 ENV GOARM64=v8.3,crypto

diff --git a/Makefile b/Makefile
@@ -20,7 +20,7 @@ define dl_tgz
 		chmod +x "$(GOBIN)/$(1)"; \
 	fi
 endef
-
+	
 $(GOBIN)/golangci-lint: GOLANGCI_VERSION = 1.62.0
 $(GOBIN)/golangci-lint: Makefile
 	$(call dl_tgz,golangci-lint,https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_VERSION)/golangci-lint-$(GOLANGCI_VERSION)-$(GOOS)-amd64.tar.gz)
@@ -67,36 +67,33 @@ scylla-shutdown:
 test:
 	@go test -covermode=atomic -race -coverprofile=coverage.txt -timeout 5m -json -v ./... 2>&1 | gotestfmt -showteststatus
 
-CQL_FEATURES ?= all
-CONCURRENCY ?= 50
+CQL_FEATURES ?= normal
+CONCURRENCY ?= 1
 DURATION ?= 10m
-WARMUP ?= 1m
+WARMUP ?= 0
+MODE ?= mixed
 DATASET_SIZE ?= large
 SEED ?= $(shell date +%s)
 GEMINI_BINARY ?= $(PWD)/bin/gemini
 GEMINI_TEST_CLUSTER ?= $(shell docker inspect --format='{{ .NetworkSettings.Networks.gemini.IPAddress }}' gemini-test)
 GEMINI_ORACLE_CLUSTER ?= $(shell docker inspect --format='{{ .NetworkSettings.Networks.gemini.IPAddress }}' gemini-oracle)
 GEMINI_DOCKER_NETWORK ?= gemini
-GEMINI_FLAGS =	--fail-fast \
+GEMINI_FLAGS =--fail-fast \
 	--level=info \
 	--non-interactive \
-	--materialized-views=false \
 	--consistency=LOCAL_QUORUM \
 	--test-host-selection-policy=token-aware \
 	--oracle-host-selection-policy=token-aware \
-	--mode=mixed \
+	--mode=$(MODE) \
 	--non-interactive \
 	--request-timeout=5s \
 	--connect-timeout=15s \
 	--use-server-timestamps=false \
 	--async-objects-stabilization-attempts=10 \
 	--max-mutation-retries=10 \
-	--async-objects-stabilization-backoff=1000ms \
-	--max-mutation-retries-backoff=1000ms \
 	--replication-strategy="{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}" \
 	--oracle-replication-strategy="{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}" \
 	--concurrency=$(CONCURRENCY) \
-	--use-lwt=true \
 	--dataset-size=$(DATASET_SIZE) \
 	--seed=$(SEED) \
 	--schema-seed=$(SEED) \
@@ -108,19 +105,23 @@ GEMINI_FLAGS =	--fail-fast \
 
 .PHONY: pprof-profile
 pprof-profile:
-	go tool pprof -http=:8080 -intel_syntax -call_tree -seconds 60 http://localhost:6060/debug/pprof/profile
+	go tool pprof -http=:8080 http://localhost:6060/debug/pprof/profile
 
 .PHONY: pprof-heap
 pprof-heap:
-	go tool pprof -http=:8080 -intel_syntax -call_tree -seconds 60 http://localhost:6060/debug/pprof/heap
+	go tool pprof -http=:8081 http://localhost:6060/debug/pprof/heap
 
 .PHONY: pprof-goroutine
 pprof-goroutine:
-	go tool pprof -http=:8080 -intel_syntax -call_tree -seconds 60 http://localhost:6060/debug/pprof/goroutine
+	go tool pprof -http=:8082 http://localhost:6060/debug/pprof/goroutine
 
 .PHONY: pprof-block
 pprof-block:
-	go tool pprof -http=:8080 -intel_syntax -call_tree -seconds 60 http://localhost:6060/debug/pprof/block
+	go tool pprof -http=:8083 http://localhost:6060/debug/pprof/block
+
+.PHONY: pprof-mutex
+pprof-mutex:
+	go tool pprof -http=:8084 http://localhost:6060/debug/pprof/mutex
 
 .PHONY: docker-integration-test
 docker-integration-test:
@@ -130,6 +131,7 @@ docker-integration-test:
 	docker run \
 		-it \
 		--rm \
+		--memory=4G \
 		-p 6060:6060 \
 		--name gemini \
 		--network $(GEMINI_DOCKER_NETWORK) \
@@ -138,24 +140,16 @@ docker-integration-test:
 		scylladb/gemini:$(DOCKER_VERSION) \
 			--test-cluster=gemini-test \
 			--oracle-cluster=gemini-oracle \
-			--outfile=/results/gemini_result.log \
-			--tracing-outfile=/results/gemini_tracing.log \
-			--test-statement-log-file=/results/gemini_test_statement.log \
-			--oracle-statement-log-file=/results/gemini_oracle_statement.log \
 			$(GEMINI_FLAGS)
 
 .PHONY: integration-test
 integration-test:
-	@mkdir -p $(PWD)/results
-	@touch $(PWD)/results/gemini_seed
-	@echo $(GEMINI_SEED) > $(PWD)/results/gemini_seed
-	@$(GEMINI_BINARY) \
+	mkdir -p $(PWD)/results
+	touch $(PWD)/results/gemini_seed
+	echo $(GEMINI_SEED) > $(PWD)/results/gemini_seed
+	$(GEMINI_BINARY) \
 		--test-cluster=$(GEMINI_TEST_CLUSTER) \
 		--oracle-cluster=$(GEMINI_ORACLE_CLUSTER) \
-		--outfile=$(PWD)/results/gemini_result.log \
-		--tracing-outfile=$(PWD)/results/gemini_tracing.log \
-		--test-statement-log-file=$(PWD)/results/gemini_test_statement.log \
-		--oracle-statement-log-file=$(PWD)/results/gemini_oracle_statement.log \
 		$(GEMINI_FLAGS)
 
 .PHONY: clean
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,3 +4,4 @@ cmd/gemini/dist/ @@
     bin/
     coverage.txt
     dist/
+    results/*.log