VEC-317 0.9.1 support

.github/workflows/create-prerelease.yml

@@ -34,11 +34,11 @@ jobs:
               run: |
                 /usr/local/bin/brew install --overwrite [email protected] || echo "I1.1"
                 /usr/local/bin/brew link --overwrite [email protected] || echo "I1.2"
-                /usr/local/bin/brew install --overwrite dpkg upx zip make wget jq rpm || echo "I2"
+                /usr/local/bin/brew install --overwrite dpkg zip make wget jq rpm || echo "I2"
                 /usr/local/bin/brew link --overwrite [email protected] || echo "I1.3"
                 /usr/local/bin/brew install [email protected] || echo "I1.4"
                 /usr/local/bin/brew install [email protected] || echo "I1.5"
-                for i in dpkg upx zip make wget jq rpm python3.11; do command -v $i || exit 1; done
+                for i in dpkg zip make wget jq rpm python3.11; do command -v $i || exit 1; done
                 echo "Dependencies checked"
             - name: Get go version from go.mod
               run: |

Makefile

@@ -395,6 +395,8 @@ pkg-zip: pkg-zip-amd64 pkg-zip-arm64
 pkg-rpm-amd64:
 	rm -rf $(BIN_DIR)/asvec-rpm-centos
 	cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos
+	mkdir -p $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin
+	mkdir -p $(BIN_DIR)/asvec-rpm-centos/usr/bin
 	sed -i.bak "s/VERSIONHERE/${rpm_ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec
 	cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin/asvec
 	rm -f $(BIN_DIR)/asvec-linux-x86_64.rpm
@@ -406,6 +408,8 @@ pkg-rpm-amd64:
 pkg-rpm-arm64:
 	rm -rf $(BIN_DIR)/asvec-rpm-centos
 	cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos
+	mkdir -p $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin
+	mkdir -p $(BIN_DIR)/asvec-rpm-centos/usr/bin
 	sed -i.bak "s/VERSIONHERE/${rpm_ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec
 	cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec-rpm-centos/opt/aerospike/bin/asvec
 	rm -f $(BIN_DIR)/asvec-linux-arm64.rpm

bin/asvecrpm/asvec.spec

@@ -12,11 +12,16 @@ Group: aerospike
 %define _binaries_in_noarch_packages_terminate_build 0
 
 %description
-
 Tool for deploying non-prod Aerospike server clusters on docker or in AWS
 
 %files
 /opt/aerospike/bin/asvec
+/usr/bin/asvec
+
+%install
+# Ensure the buildroot directories exist
+mkdir -p %{buildroot}/opt/aerospike/bin
+mkdir -p %{buildroot}/usr/bin
 
 %prep
 ln -sf /opt/aerospike/bin/asvec %{buildroot}/usr/bin/asvec

cmd/flags/hnsw.go

@@ -80,11 +80,11 @@ func NewHnswHealerFlags() *HealerFlags {
 
 func (cf *HealerFlags) NewFlagSet() *pflag.FlagSet {
 	flagSet := &pflag.FlagSet{}
-	flagSet.Var(&cf.MaxScanRatePerNode, HnswHealerMaxScanRatePerNode, "Maximum allowed record scan rate per AVS node.")                         //nolint:lll // For readability
-	flagSet.Var(&cf.MaxScanPageSize, HnswHealerMaxScanPageSize, "Maximum number of records in a single scanned page.")                          //nolint:lll // For readability
-	flagSet.Var(&cf.ReindexPercent, HnswHealerReindexPercent, "Percentage of good records randomly selected for reindexing in a healer cycle.") //nolint:lll // For readability
-	flagSet.Var(&cf.Schedule, HnswHealerSchedule, "The quartz cron expression defining schedule at which the healer cycle is invoked.")         //nolint:lll // For readability
-	flagSet.Var(&cf.Parallelism, HnswHealerParallelism, "Maximum number of records to heal in parallel.")                                       //nolint:lll // For readability
+	flagSet.Var(&cf.MaxScanRatePerNode, HnswHealerMaxScanRatePerNode, "Maximum allowed record scan rate per AVS node.")                           //nolint:lll // For readability
+	flagSet.Var(&cf.MaxScanPageSize, HnswHealerMaxScanPageSize, "Maximum number of records in a single scanned page.")                            //nolint:lll // For readability
+	flagSet.Var(&cf.ReindexPercent, HnswHealerReindexPercent, "Percentage of good records randomly selected for reindexing in a healer cycle.")   //nolint:lll // For readability
+	flagSet.Var(&cf.Schedule, HnswHealerSchedule, "The quartz cron expression defining the schedule at which the index healer cycle is invoked.") //nolint:lll // For readability
+	flagSet.Var(&cf.Parallelism, HnswHealerParallelism, "Maximum number of records to heal in parallel.")                                         //nolint:lll // For readability
 
 	return flagSet
 }

cmd/query.go

@@ -146,7 +146,7 @@ asvec query -i my-index -n my-namespace -v "[1,0,1,0,0,0,1,0,1,1]" --max-keys 10
 					return
 				}
 			} else {
-				indexDef, err := client.IndexGet(ctx, queryFlags.namespace, queryFlags.indexName)
+				indexDef, err := client.IndexGet(ctx, queryFlags.namespace, queryFlags.indexName, false)
 				if err != nil {
 					logger.ErrorContext(ctx, "unable to get index definition", slog.Any("error", err))
 					view.Errorf("Failed to get index definition: %s", err)

e2e_test.go

@@ -172,7 +172,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 			"index0",
 			"test",
 			"index create -y -n test -i index0 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector0 --index-labels model=all-MiniLM-L6-v2,foo=bar",
-			tests.NewIndexDefinitionBuilder("index0", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0").
+			tests.NewIndexDefinitionBuilder(false, "index0", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector0").
 				WithLabels(map[string]string{"model": "all-MiniLM-L6-v2", "foo": "bar"}).
 				Build(),
 		},
@@ -181,7 +181,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 			"index1",
 			"test",
 			"index create -y -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar",
-			tests.NewIndexDefinitionBuilder("index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1").
+			tests.NewIndexDefinitionBuilder(false, "index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1").
 				WithStorageNamespace("bar").
 				WithStorageSet("testbar").
 				Build(),
@@ -191,7 +191,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 			"index2",
 			"test",
 			"index create -y -n test -i index2 -d 256 -m HAMMING --vector-field vector2 --hnsw-m 10 --hnsw-ef 11 --hnsw-ef-construction 12 --hnsw-max-mem-queue-size 10",
-			tests.NewIndexDefinitionBuilder("index2", "test", 256, protos.VectorDistanceMetric_HAMMING, "vector2").
+			tests.NewIndexDefinitionBuilder(false, "index2", "test", 256, protos.VectorDistanceMetric_HAMMING, "vector2").
 				WithHnswM(10).
 				WithHnswEf(11).
 				WithHnswEfConstruction(12).
@@ -203,7 +203,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 			"index3",
 			"test",
 			"index create -y -n test -i index3 -d 256 -m COSINE --vector-field vector3 --hnsw-batch-interval 50s --hnsw-batch-max-records 10001",
-			tests.NewIndexDefinitionBuilder("index3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector3").
+			tests.NewIndexDefinitionBuilder(false, "index3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector3").
 				WithHnswBatchingMaxRecord(10001).
 				WithHnswBatchingInterval(50000).
 				Build(),
@@ -213,7 +213,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 			"index4",
 			"test",
 			"index create -y -n test -i index4 -d 256 -m COSINE --vector-field vector4 --hnsw-cache-max-entries 1000 --hnsw-cache-expiry 10s",
-			tests.NewIndexDefinitionBuilder("index4", "test", 256, protos.VectorDistanceMetric_COSINE, "vector4").
+			tests.NewIndexDefinitionBuilder(false, "index4", "test", 256, protos.VectorDistanceMetric_COSINE, "vector4").
 				WithHnswCacheExpiry(10000).
 				WithHnswCacheMaxEntries(1000).
 				Build(),
@@ -223,7 +223,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 			"index5",
 			"test",
 			"index create -y -n test -i index5 -d 256 -m COSINE --vector-field vector5 --hnsw-healer-max-scan-rate-per-node 1000 --hnsw-healer-max-scan-page-size 1000 --hnsw-healer-reindex-percent 10.10 --hnsw-healer-schedule \"0 0 0 ? * *\" --hnsw-healer-parallelism 10",
-			tests.NewIndexDefinitionBuilder("index5", "test", 256, protos.VectorDistanceMetric_COSINE, "vector5").
+			tests.NewIndexDefinitionBuilder(false, "index5", "test", 256, protos.VectorDistanceMetric_COSINE, "vector5").
 				WithHnswHealerMaxScanRatePerNode(1000).
 				WithHnswHealerMaxScanPageSize(1000).
 				WithHnswHealerReindexPercent(10.10).
@@ -235,17 +235,18 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 			"test with hnsw merge params",
 			"index6",
 			"test",
-			"index create -y -n test -i index6 -d 256 -m COSINE --vector-field vector6 --hnsw-merge-index-parallelism 10",
-			tests.NewIndexDefinitionBuilder("index6", "test", 256, protos.VectorDistanceMetric_COSINE, "vector6").
+			"index create -y -n test -i index6 -d 256 -m COSINE --vector-field vector6 --hnsw-merge-index-parallelism 10 --hnsw-merge-reindex-parallelism 11",
+			tests.NewIndexDefinitionBuilder(false, "index6", "test", 256, protos.VectorDistanceMetric_COSINE, "vector6").
 				WithHnswMergeIndexParallelism(10).
+				WithHnswMergeReIndexParallelism(11).
 				Build(),
 		},
 		{
 			"test with yaml file",
 			"yaml-file-index",
 			"test",
 			fmt.Sprintf("index create -y --file tests/indexDef.yaml"),
-			tests.NewIndexDefinitionBuilder("yaml-file-index", "test", 10, protos.VectorDistanceMetric_COSINE, "vector").
+			tests.NewIndexDefinitionBuilder(false, "yaml-file-index", "test", 10, protos.VectorDistanceMetric_COSINE, "vector").
 				WithSet("testset").
 				WithHnswEf(101).
 				WithHnswEfConstruction(102).
@@ -262,6 +263,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 				WithHnswHealerSchedule("0 15 10 ? * 6L 2022-2025").
 				WithHnswMergeIndexParallelism(7).
 				WithHnswMergeReIndexParallelism(5).
+				WithStorageNamespace("test").
 				WithStorageSet("name").
 				Build(),
 		},
@@ -276,7 +278,7 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() {
 				suite.FailNow("unable to index create")
 			}
 
-			actual, err := suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName)
+			actual, err := suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName, false)
 
 			if err != nil {
 				suite.FailNowf("unable to get index", "%v", err)
@@ -300,10 +302,10 @@ func (suite *CmdTestSuite) TestPipeFromListIndexToCreateIndex() {
 		{
 			"test with all indexes succeed",
 			[]*protos.IndexDefinition{
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"exists1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector",
 				).Build(),
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"exists2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector",
 				).WithSet("barset").Build(),
 			},
@@ -318,10 +320,10 @@ func (suite *CmdTestSuite) TestPipeFromListIndexToCreateIndex() {
 		{
 			"test with one index that fails",
 			[]*protos.IndexDefinition{
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"exists3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector",
 				).Build(),
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"exists4", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector",
 				).WithSet("barset").Build(),
 			},
@@ -336,10 +338,10 @@ func (suite *CmdTestSuite) TestPipeFromListIndexToCreateIndex() {
 		{
 			"test with no index successfully created",
 			[]*protos.IndexDefinition{
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"exists1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector",
 				).Build(),
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"exists2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector",
 				).WithSet("barset").Build(),
 			},
@@ -454,7 +456,7 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() {
 	index := "successful-update"
 
 	newBuilder := func() *tests.IndexDefinitionBuilder {
-		return tests.NewIndexDefinitionBuilder(index, ns, 256, protos.VectorDistanceMetric_COSINE, "field")
+		return tests.NewIndexDefinitionBuilder(true, index, ns, 256, protos.VectorDistanceMetric_COSINE, "field")
 	}
 
 	testCases := []struct {
@@ -511,9 +513,10 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() {
 			"test with hnsw merge params",
 			"successful-update",
 			"test",
-			"index update -y -n test -i successful-update --hnsw-merge-index-parallelism 10",
+			"index update -y -n test -i successful-update --hnsw-merge-index-parallelism 10 --hnsw-merge-reindex-parallelism 11",
 			newBuilder().
 				WithHnswMergeIndexParallelism(10).
+				WithHnswMergeReIndexParallelism(11).
 				Build(),
 		},
 	}
@@ -536,7 +539,7 @@ func (suite *CmdTestSuite) TestSuccessfulUpdateIndexCmd() {
 
 			time.Sleep(5 * time.Second)
 
-			actual, err := suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName)
+			actual, err := suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName, false)
 
 			if err != nil {
 				suite.FailNowf("unable to get index", "%v", err)
@@ -642,7 +645,7 @@ func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() {
 				suite.FailNow("unable to index drop")
 			}
 
-			_, err = suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName)
+			_, err = suite.AvsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName, false)
 
 			time.Sleep(time.Second * 3)
 
@@ -677,7 +680,7 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() {
 		{
 			"single index",
 			[]*protos.IndexDefinition{
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"list", "test", 256, protos.VectorDistanceMetric_COSINE, "vector",
 				).Build(),
 			},
@@ -690,10 +693,10 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() {
 		{
 			"double index with set",
 			[]*protos.IndexDefinition{
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"list1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector",
 				).Build(),
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"list2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector",
 				).WithSet("barset").Build(),
 			},
@@ -707,13 +710,13 @@ func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() {
 		{
 			"double index with set and verbose",
 			[]*protos.IndexDefinition{
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"list1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector",
 				).WithLabels(map[string]string{"foo": "bar"}).
 					WithHnswMergeIndexParallelism(80).
 					WithHnswMergeReIndexParallelism(26).
 					Build(),
-				tests.NewIndexDefinitionBuilder(
+				tests.NewIndexDefinitionBuilder(false,
 					"list2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector",
 				).WithSet("barset").
 					WithHnswMergeIndexParallelism(80).
@@ -1108,10 +1111,10 @@ func (suite *CmdTestSuite) TestSuccessfulQueryCmd() {
 	strIndexName := "query-str-index"
 	intIndexName := "query-int-index"
 	indexes := []*protos.IndexDefinition{
-		tests.NewIndexDefinitionBuilder(
+		tests.NewIndexDefinitionBuilder(false,
 			strIndexName, "test", 10, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "float32-str",
 		).Build(),
-		tests.NewIndexDefinitionBuilder(
+		tests.NewIndexDefinitionBuilder(false,
 			intIndexName, "test", 10, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "float32-int",
 		).Build(),
 	}

go.mod

@@ -2,10 +2,8 @@ module asvec
 
 go 1.22.5
 
-// replace github.com/aerospike/avs-client-go => ../avs-client-go
-
 require (
-	github.com/aerospike/avs-client-go v0.0.0-20240829195926-fe3d122cb3df
+	github.com/aerospike/avs-client-go v0.0.0-20240906211641-97c1df4005ae
 	github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6
 	github.com/jedib0t/go-pretty/v6 v6.5.9
 	github.com/spf13/cobra v1.8.1

go.sum

@@ -2,8 +2,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/aerospike/aerospike-client-go/v7 v7.6.0 h1:jAOlsxOaWbmtGzB1yP9x1komh4x14BvCb5HRu5AzVIo=
 github.com/aerospike/aerospike-client-go/v7 v7.6.0/go.mod h1:uCbSYMpjlRcH/9f26VSF/luzDDXrcDaV8c6/WIcKtT4=
-github.com/aerospike/avs-client-go v0.0.0-20240829195926-fe3d122cb3df h1:tcnQhx/wQWam1Qu/3vG49ywf7/B6SrodjiyizuyrOnE=
-github.com/aerospike/avs-client-go v0.0.0-20240829195926-fe3d122cb3df/go.mod h1:FsWI2J5UqmMpiLTfXiqSXf+tBBGzlOtW8HVmhiac0sY=
+github.com/aerospike/avs-client-go v0.0.0-20240906211641-97c1df4005ae h1:SF5A5zEeuBFTCV19ocWsdnK2bB7P3MnTnvyJANae5qM=
+github.com/aerospike/avs-client-go v0.0.0-20240906211641-97c1df4005ae/go.mod h1:FsWI2J5UqmMpiLTfXiqSXf+tBBGzlOtW8HVmhiac0sY=
 github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6 h1:tOLqcGsc6A656WNEZhYZYxDiX7d6wCkEN6+jLDSDeUU=
 github.com/aerospike/tools-common-go v0.0.0-20240701164814-36eec593d9c6/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=