From 4ef27d960b88a7663889b4e10fafdb3348f11c2f Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 21 May 2024 15:52:49 -0700 Subject: [PATCH 01/35] WIP can create an index --- .github/workflows/golangci-lint.yml | 27 ++++++ .gitignore | 1 + .golangci.yml | 88 ++++++++++++++++++ LICENSE | 0 cmd/create.go | 40 +++++++++ cmd/createIndex.go | 93 +++++++++++++++++++ cmd/list.go | 40 +++++++++ cmd/listIndex.go | 40 +++++++++ cmd/root.go | 56 ++++++++++++ cmd/view.go | 34 +++++++ go.mod | 41 +++++++++ go.sum | 134 ++++++++++++++++++++++++++++ main.go | 11 +++ 13 files changed, 605 insertions(+) create mode 100644 .github/workflows/golangci-lint.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 LICENSE create mode 100644 cmd/create.go create mode 100644 cmd/createIndex.go create mode 100644 cmd/list.go create mode 100644 cmd/listIndex.go create mode 100644 cmd/root.go create mode 100644 cmd/view.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..486c06d --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,27 @@ +name: golangci-lint +on: + push: + tags: + - v* + branches: + - main + pull_request: + branches: + - main + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v3 + - name: Setup-go + uses: actions/setup-go@v3 + with: + go-version: 1.21 + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + version: v1.58 + args: --timeout=5m --out-format=colored-line-number diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..92c8db2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +./docker/* \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..f41cb1c --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,88 @@ + +linters-settings: + goconst: + min-len: 2 + min-occurrences: 3 + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + govet: + check-shadowing: true + enable: + - fieldalignment + nolintlint: + require-explanation: true + require-specific: true + depguard: + rules: + Main: + allow: + - $gostd + - github.com/aerospike/aerospike-proximus-client-go/protos + +linters: + disable-all: true + enable: + - bodyclose +# - unused # intentionally commented to avoid unused func warning as this repo is library + - depguard + - dogsled + - dupl + - errcheck + - exportloopref + - exhaustive + - goconst + - gocritic + - gofmt + - goimports + - gocyclo + - gosec + - gosimple + - govet + - ineffassign + - misspell + - nolintlint + - nakedret + - prealloc # pre-allocate slices with define size if the slice size is known in advance + - predeclared + - revive + - staticcheck + - stylecheck + - thelper + - tparallel + - typecheck + - unconvert + - unparam + - whitespace + - lll + - wsl # While space linter + +run: + issues-exit-code: 1 + go: '1.21' +# skip-dirs: +# - sample +# skip-files: +# - sample + +# issues: +# exclude-rules: +# - path: info/as_parser_test\.go +# linters: +# - lll # Test code is allowed to have long lines +# - path: asconfig/generate_test\.go +# linters: +# - dupl # Test code is allowed to have duplicate code +# - path: asconfig/asconfig_test\.go +# linters: +# - dupl # Test code is allowed to have duplicate code +# - path: '(.+)test\.go' +# linters: +# - govet # Test code field alignment for sake of space is not a concern +# - linters: +# - lll +# source: "// " diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e69de29 diff --git a/cmd/create.go b/cmd/create.go new file mode 100644 index 0000000..a5e613c --- /dev/null +++ b/cmd/create.go @@ -0,0 +1,40 @@ +/* +Copyright © 2024 NAME HERE + +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// createCmd represents the create command +var createCmd = &cobra.Command{ + Use: "create", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("create called") + }, +} + +func init() { + rootCmd.AddCommand(createCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // createCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // createCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/createIndex.go b/cmd/createIndex.go new file mode 100644 index 0000000..ecf218d --- /dev/null +++ b/cmd/createIndex.go @@ -0,0 +1,93 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "context" + "log/slog" + "net" + + avs "github.com/aerospike/aerospike-proximus-client-go" + "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// createIndexCmd represents the createIndex command +var createIndexCmd = &cobra.Command{ + Use: "index", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + host := viper.GetString("host") + port := viper.GetInt("port") + hostPort := avs.NewHostPort(host, port, false) + namespace := viper.GetString("namespace") + sets := viper.GetStringSlice("sets") + indexName := viper.GetString("index-name") + vectorField := viper.GetString("vector-field") + dimension := viper.GetUint32("dimension") + // distanceMetric := viper.GetInt("distance-metric") + indexMeta := viper.GetStringMapString("index-meta") + + logger.Debug("Parsed flags", slog.String("host", host), slog.Int("port", port), slog.String("namespace", namespace), slog.Any("sets", sets), slog.String("index-name", indexName), slog.String("vector-field", vectorField), slog.Uint64("dimension", uint64(dimension)), slog.Any("index-meta", indexMeta)) + + ctx := context.TODO() + + adminClient, err := avs.NewAdminClient(ctx, []*avs.HostPort{hostPort}, nil, false, logger) + if err != nil { + logger.Error("failed to create AVS client", slog.Any("error", err)) + view.Printf("Failed to connect to AVS: %v", err) + return + } + + // TODO: parse cosine + err = adminClient.IndexCreate(ctx, namespace, sets, indexName, vectorField, dimension, protos.VectorDistanceMetric_COSINE, nil, indexMeta) + if err != nil { + logger.Error("unable to create index", slog.Any("error", err)) + view.Printf("Unable to create index: %v", err) + return + } + + view.Printf("Successfully created index %s.%s", namespace, indexName) + }, +} + +func init() { + createCmd.AddCommand(createIndexCmd) + createIndexCmd.PersistentFlags().IPP("host", "h", net.ParseIP("127.0.0.1"), "TODO") + createIndexCmd.PersistentFlags().IntP("port", "p", 5000, "TODO") + createIndexCmd.Flags().StringP("namespace", "n", "", "TODO") + createIndexCmd.Flags().StringArrayP("sets", "s", nil, "TODO") + createIndexCmd.Flags().StringP("index-name", "i", "", "TODO") + createIndexCmd.Flags().StringP("vector-field", "v", "vector", "TODO") + createIndexCmd.Flags().IntP("dimension", "d", 0, "TODO") + createIndexCmd.Flags().Uint32P("distance-metric", "m", 0, "TODO") + createIndexCmd.Flags().StringToStringP("index-meta", "e", nil, "TODO") + // TODO hnsw metadata + + createIndexCmd.MarkFlagRequired("namespace") + createIndexCmd.MarkFlagRequired("set") + createIndexCmd.MarkFlagRequired("index-name") + // createIndexCmd.MarkFlagRequired("vector-field") + createIndexCmd.MarkFlagRequired("dimension") + // createIndexCmd.MarkFlagRequired("distance-metric") + viper.BindPFlags(createIndexCmd.PersistentFlags()) + viper.BindPFlags(createIndexCmd.Flags()) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // createIndexCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // createIndexCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/list.go b/cmd/list.go new file mode 100644 index 0000000..b8e09c2 --- /dev/null +++ b/cmd/list.go @@ -0,0 +1,40 @@ +/* +Copyright © 2024 NAME HERE + +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// listCmd represents the list command +var listCmd = &cobra.Command{ + Use: "list", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("list called") + }, +} + +func init() { + rootCmd.AddCommand(listCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // listCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // listCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/listIndex.go b/cmd/listIndex.go new file mode 100644 index 0000000..250e02b --- /dev/null +++ b/cmd/listIndex.go @@ -0,0 +1,40 @@ +/* +Copyright © 2024 NAME HERE + +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// listIndexCmd represents the listIndex command +var listIndexCmd = &cobra.Command{ + Use: "listIndex", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("listIndex called") + }, +} + +func init() { + listCmd.AddCommand(listIndexCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // listIndexCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // listIndexCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..c9dcd00 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,56 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "io" + "log/slog" + "os" + + common "github.com/aerospike/tools-common-go/flags" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var logger = slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError})) +var view = NewView(os.Stdout) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "asvec", + Short: "A brief description of your application", + Long: `A longer description that spans multiple lines and likely contains +examples and usage of using your application. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + // Uncomment the following line if your bare application + // has an action associated with it: + // Run: func(cmd *cobra.Command, args []string) { }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.asvec.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + // rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") + viper.SetEnvPrefix("AVS") + viper.AutomaticEnv() +} diff --git a/cmd/view.go b/cmd/view.go new file mode 100644 index 0000000..8ba83c2 --- /dev/null +++ b/cmd/view.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "fmt" + "io" +) + +type View struct { + writer io.Writer +} + +func NewView(writer io.Writer) *View { + return &View{writer: writer} +} + +func (v *View) Print(a ...any) { + s := fmt.Sprint(a...) + fmt.Print(s) + v.writer.Write([]byte(fmt.Sprint(a...))) + v.Newline() +} + +func (v *View) Printf(f string, a ...any) { + s := fmt.Sprintf(f, a...) + v.writer.Write([]byte(s)) + v.Newline() +} + +func (v *View) Newline() { + _, err := v.writer.Write([]byte("\n")) + if err != nil { + panic(err) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7ff50a1 --- /dev/null +++ b/go.mod @@ -0,0 +1,41 @@ +module asvec + +go 1.21.7 + +replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go + +require ( + github.com/aerospike/aerospike-proximus-client-go v0.0.0 + github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 + github.com/spf13/cobra v1.8.0 + github.com/spf13/viper v1.18.2 +) + +require ( + github.com/aerospike/aerospike-client-go/v7 v7.2.1 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..04dc9dd --- /dev/null +++ b/go.sum @@ -0,0 +1,134 @@ +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/aerospike/aerospike-client-go/v7 v7.2.1 h1:4A6CxgJMRlDnHx4ycyJ1a5lUzxMS7u4byG1XlhCrvSg= +github.com/aerospike/aerospike-client-go/v7 v7.2.1/go.mod h1:sKfNsnAKgkGtAlYxdgWNOJm3ykm49s/p6xjEB/cX8/k= +github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= +github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v26.1.0+incompatible h1:W1G9MPNbskA6VZWL7b3ZljTh0pXI68FpINx0GKaOdaM= +github.com/docker/docker v26.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= +github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..bb0c559 --- /dev/null +++ b/main.go @@ -0,0 +1,11 @@ +/* +Copyright © 2024 NAME HERE + +*/ +package main + +import "asvec/cmd" + +func main() { + cmd.Execute() +} From 5a9143817ad4ea89ecc046aadc7c3360f2927f75 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 22 May 2024 15:55:33 -0700 Subject: [PATCH 02/35] add hnsw params --- cmd/createIndex.go | 101 +++++++++++++++++++++++++++----------------- cmd/delete.go | 40 ++++++++++++++++++ cmd/deleteIndex.go | 40 ++++++++++++++++++ cmd/flags.go | 102 +++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 3 +- 5 files changed, 245 insertions(+), 41 deletions(-) create mode 100644 cmd/delete.go create mode 100644 cmd/deleteIndex.go create mode 100644 cmd/flags.go diff --git a/cmd/createIndex.go b/cmd/createIndex.go index ecf218d..a6297c6 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -6,14 +6,32 @@ package cmd import ( "context" "log/slog" - "net" avs "github.com/aerospike/aerospike-proximus-client-go" "github.com/aerospike/aerospike-proximus-client-go/protos" + commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" "github.com/spf13/viper" ) +var requiredFlags = []string{ + flagNameNamespace, + flagNameIndexName, + flagNameDimension, + flagNameDistance, +} + +var persistentRequiredFlags = []string{} + +const ( + flagNameMaxEdges = "hnsw-max-edges" + flagNameConstructionEf = "hnsw-ef-construction" + flagNameEf = "hnsw-ef" + flagNameBatchMaxRecords = "hnsw-batch-max-records" + flagNameBatchInterval = "hnsw-batch-interval" + flagNameBatchDisabled = "hnsw-batch-disabled" +) + // createIndexCmd represents the createIndex command var createIndexCmd = &cobra.Command{ Use: "index", @@ -25,18 +43,18 @@ Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, Run: func(cmd *cobra.Command, args []string) { - host := viper.GetString("host") - port := viper.GetInt("port") - hostPort := avs.NewHostPort(host, port, false) - namespace := viper.GetString("namespace") - sets := viper.GetStringSlice("sets") - indexName := viper.GetString("index-name") - vectorField := viper.GetString("vector-field") - dimension := viper.GetUint32("dimension") - // distanceMetric := viper.GetInt("distance-metric") - indexMeta := viper.GetStringMapString("index-meta") - - logger.Debug("Parsed flags", slog.String("host", host), slog.Int("port", port), slog.String("namespace", namespace), slog.Any("sets", sets), slog.String("index-name", indexName), slog.String("vector-field", vectorField), slog.Uint64("dimension", uint64(dimension)), slog.Any("index-meta", indexMeta)) + seed := viper.GetString(flagNameSeeds) + port := viper.GetInt(flagNamePort) + hostPort := avs.NewHostPort(seed, port, false) + namespace := viper.GetString(flagNameNamespace) + sets := viper.GetStringSlice(flagNameSets) + indexName := viper.GetString(flagNameIndexName) + vectorField := viper.GetString(flagNameVector) + dimension := viper.GetUint32(flagNameDimension) + indexMeta := viper.GetStringMapString(flagNameIndexMeta) + distanceMetric := viper.GetString(flagNameDistance) + + logger.Debug("parsed flags", slog.String("seeds", seed), slog.Int("port", port), slog.String("namespace", namespace), slog.Any("sets", sets), slog.String("index-name", indexName), slog.String("vector-field", vectorField), slog.Uint64("dimension", uint64(dimension)), slog.Any("index-meta", indexMeta)) ctx := context.TODO() @@ -48,7 +66,7 @@ to quickly create a Cobra application.`, } // TODO: parse cosine - err = adminClient.IndexCreate(ctx, namespace, sets, indexName, vectorField, dimension, protos.VectorDistanceMetric_COSINE, nil, indexMeta) + err = adminClient.IndexCreate(ctx, namespace, sets, indexName, vectorField, dimension, protos.VectorDistanceMetric(protos.VectorDistanceMetric_value[distanceMetric]), nil, indexMeta) if err != nil { logger.Error("unable to create index", slog.Any("error", err)) view.Printf("Unable to create index: %v", err) @@ -61,33 +79,38 @@ to quickly create a Cobra application.`, func init() { createCmd.AddCommand(createIndexCmd) - createIndexCmd.PersistentFlags().IPP("host", "h", net.ParseIP("127.0.0.1"), "TODO") - createIndexCmd.PersistentFlags().IntP("port", "p", 5000, "TODO") - createIndexCmd.Flags().StringP("namespace", "n", "", "TODO") - createIndexCmd.Flags().StringArrayP("sets", "s", nil, "TODO") - createIndexCmd.Flags().StringP("index-name", "i", "", "TODO") - createIndexCmd.Flags().StringP("vector-field", "v", "vector", "TODO") - createIndexCmd.Flags().IntP("dimension", "d", 0, "TODO") - createIndexCmd.Flags().Uint32P("distance-metric", "m", 0, "TODO") - createIndexCmd.Flags().StringToStringP("index-meta", "e", nil, "TODO") - // TODO hnsw metadata - createIndexCmd.MarkFlagRequired("namespace") - createIndexCmd.MarkFlagRequired("set") - createIndexCmd.MarkFlagRequired("index-name") - // createIndexCmd.MarkFlagRequired("vector-field") - createIndexCmd.MarkFlagRequired("dimension") - // createIndexCmd.MarkFlagRequired("distance-metric") - viper.BindPFlags(createIndexCmd.PersistentFlags()) - viper.BindPFlags(createIndexCmd.Flags()) + persistentFlags := NewFlagSetBuilder(createIndexCmd.PersistentFlags()) + flags := NewFlagSetBuilder(createIndexCmd.Flags()) + + persistentFlags.AddSeedFlag() + persistentFlags.AddPortFlag() + + flags.AddNamespaceFlag() + flags.AddSetsFlag() + flags.AddIndexNameFlag() + flags.AddVectorFieldFlag() + flags.AddDimensionFlag() + flags.AddDistanceMetricFlag() + flags.AddIndexMetaFlag() + + flags.Uint32(flagNameMaxEdges, 0, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) + flags.Uint32(flagNameConstructionEf, 0, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) + flags.Uint32(flagNameEf, 0, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) + flags.Uint32(flagNameBatchMaxRecords, 0, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) + flags.Uint32(flagNameBatchInterval, 0, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) + flags.Bool(flagNameBatchDisabled, false, commonFlags.DefaultWrapHelpString("Disables batching for index updates. Default is false meaning batching is enabled.")) - // Here you will define your flags and configuration settings. + for _, flag := range requiredFlags { + createIndexCmd.MarkFlagRequired(flag) + } - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // createIndexCmd.PersistentFlags().String("foo", "", "A help for foo") + for _, flag := range persistentRequiredFlags { + createIndexCmd.MarkPersistentFlagRequired(flag) + } + + // TODO hnsw metadata + viper.BindPFlags(createIndexCmd.PersistentFlags()) + viper.BindPFlags(createIndexCmd.Flags()) - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // createIndexCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } diff --git a/cmd/delete.go b/cmd/delete.go new file mode 100644 index 0000000..679dbb1 --- /dev/null +++ b/cmd/delete.go @@ -0,0 +1,40 @@ +/* +Copyright © 2024 NAME HERE + +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// deleteCmd represents the delete command +var deleteCmd = &cobra.Command{ + Use: "delete", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("delete called") + }, +} + +func init() { + rootCmd.AddCommand(deleteCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // deleteCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // deleteCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/deleteIndex.go b/cmd/deleteIndex.go new file mode 100644 index 0000000..39a6052 --- /dev/null +++ b/cmd/deleteIndex.go @@ -0,0 +1,40 @@ +/* +Copyright © 2024 NAME HERE + +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// deleteIndexCmd represents the deleteIndex command +var deleteIndexCmd = &cobra.Command{ + Use: "deleteIndex", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("deleteIndex called") + }, +} + +func init() { + deleteCmd.AddCommand(deleteIndexCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // deleteIndexCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // deleteIndexCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/flags.go b/cmd/flags.go new file mode 100644 index 0000000..a4f2887 --- /dev/null +++ b/cmd/flags.go @@ -0,0 +1,102 @@ +package cmd + +import ( + "fmt" + "net" + "strings" + + "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/spf13/pflag" +) + +const ( + flagNameSeeds = "seeds" + flagNamePort = "port" + flagNameNamespace = "namespace" + flagNameSets = "sets" + flagNameIndexName = "index-name" + flagNameVector = "vector-field" + flagNameDimension = "dimension" + flagNameDistance = "distance-metric" + flagNameIndexMeta = "index-meta" +) + +type FlagSetBuilder struct { + *pflag.FlagSet +} + +func NewFlagSetBuilder(flagSet *pflag.FlagSet) *FlagSetBuilder { + return &FlagSetBuilder{ + flagSet, + } +} + +// TODO: Should this be a list of IPs? Should we support IP:PORT? +func (fsb *FlagSetBuilder) AddSeedFlag() { + fsb.IPP(flagNameSeeds, "h", net.ParseIP("127.0.0.1"), "The AVS seed host for cluster discovery.") +} + +func (fsb *FlagSetBuilder) AddPortFlag() { + fsb.IntP(flagNamePort, "p", 5000, "The AVS seed port for cluster discovery.") +} + +func (fsb *FlagSetBuilder) AddNamespaceFlag() { + fsb.StringP(flagNameNamespace, "n", "", "The namespace for the index.") +} + +func (fsb *FlagSetBuilder) AddSetsFlag() { + fsb.StringArrayP(flagNameSets, "s", nil, "The sets for the index.") +} + +func (fsb *FlagSetBuilder) AddIndexNameFlag() { + fsb.StringP(flagNameIndexName, "i", "", "The name of the index.") + +} + +func (fsb *FlagSetBuilder) AddVectorFieldFlag() { + fsb.StringP(flagNameVector, "v", "vector-field", "The name of the vector field.") + +} + +func (fsb *FlagSetBuilder) AddDimensionFlag() { + fsb.IntP(flagNameDimension, "d", 0, "The dimension of the vector field.") + +} + +func (fsb *FlagSetBuilder) AddDistanceMetricFlag() { + distMetric := DistanceMetricFlag("") + fsb.VarP(&distMetric, "distance-metric", "m", "The distance metric for the index.") +} + +func (fsb *FlagSetBuilder) AddIndexMetaFlag() { + fsb.StringToStringP(flagNameIndexMeta, "e", nil, "The metadata for the index.") +} + +type DistanceMetricFlag string + +// This is just a set of valid VectorDistanceMetrics. The value does not have meaning +var distanceMetricSet = protos.VectorDistanceMetric_value + +func (mode *DistanceMetricFlag) Set(val string) error { + val = strings.ToUpper(val) + if val, ok := distanceMetricSet[val]; ok { + *mode = DistanceMetricFlag(val) + return nil + } + + return fmt.Errorf("unrecognized distance metric") +} + +func (mode *DistanceMetricFlag) Type() string { + names := []string{} + + for key := range distanceMetricSet { + names = append(names, key) + } + + return strings.Join(names, ",") +} + +func (mode *DistanceMetricFlag) String() string { + return string(*mode) +} diff --git a/cmd/root.go b/cmd/root.go index c9dcd00..9cec31f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,7 +4,6 @@ Copyright © 2024 NAME HERE package cmd import ( - "io" "log/slog" "os" @@ -13,7 +12,7 @@ import ( "github.com/spf13/viper" ) -var logger = slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError})) +var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})) var view = NewView(os.Stdout) // rootCmd represents the base command when called without any subcommands From 75460ef6379cb204ed30650f0edb4aa4973dcb5b Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 16:27:33 -0700 Subject: [PATCH 03/35] add indexlist, add workflow to build packages --- .github/workflows/create-prerelease.yml | 134 +++++ Makefile | 491 +++++++++++++++++ VERSION.md | 1 + bin/asvecrpm/asvec.spec | 20 + bin/asvecrpm/usr/bin/.gitignore | 1 + bin/macos-pkg/.gitignore | 3 + bin/macos-pkg/AsVec-template.pkgproj | 698 ++++++++++++++++++++++++ bin/macos-pkg/AsVec.pkgproj | 698 ++++++++++++++++++++++++ bin/macos-pkg/asvec/.gitignore | 1 + bin/macos-pkg/install.sh | 11 + bin/packages/.gitignore | 2 + cmd/create.go | 6 - cmd/createIndex.go | 62 ++- cmd/deleteIndex.go | 40 -- cmd/{delete.go => drop.go} | 18 +- cmd/dropIndex.go | 90 +++ cmd/flags.go | 76 ++- cmd/listIndex.go | 85 ++- cmd/root.go | 34 +- cmd/view.go | 37 +- cmd/writers/default.go | 20 + cmd/writers/indexList.go | 65 +++ cmd/writers/transformers.go | 20 + go.mod | 10 +- go.sum | 10 + 25 files changed, 2498 insertions(+), 135 deletions(-) create mode 100644 .github/workflows/create-prerelease.yml create mode 100644 Makefile create mode 100644 VERSION.md create mode 100644 bin/asvecrpm/asvec.spec create mode 100644 bin/asvecrpm/usr/bin/.gitignore create mode 100644 bin/macos-pkg/.gitignore create mode 100755 bin/macos-pkg/AsVec-template.pkgproj create mode 100644 bin/macos-pkg/AsVec.pkgproj create mode 100644 bin/macos-pkg/asvec/.gitignore create mode 100755 bin/macos-pkg/install.sh create mode 100644 bin/packages/.gitignore delete mode 100644 cmd/deleteIndex.go rename cmd/{delete.go => drop.go} (66%) create mode 100644 cmd/dropIndex.go create mode 100644 cmd/writers/default.go create mode 100644 cmd/writers/indexList.go create mode 100644 cmd/writers/transformers.go diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml new file mode 100644 index 0000000..ff8a995 --- /dev/null +++ b/.github/workflows/create-prerelease.yml @@ -0,0 +1,134 @@ +name: Build and Create Pre-Release + +on: + workflow_dispatch: + inputs: + addCommit: + description: 'Dev build?' + required: false + type: boolean + deletePrevBuild: + description: 'Delete existing pre-releases?' + required: false + type: boolean +jobs: + build-and-release: + runs-on: macos-13 + steps: + - name: "Git checkout" + uses: actions/checkout@v3 + - name: "Install Homebrew" + run: /bin/bash -c "NONINTERACTIVE=1 $(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + - name: "Install Dependencies" + run: | + /usr/local/bin/brew install --overwrite python@3.11 || echo "I1.1" + /usr/local/bin/brew link --overwrite python@3.11 || echo "I1.2" + /usr/local/bin/brew install --overwrite dpkg upx zip make wget jq rpm || echo "I2" + /usr/local/bin/brew link --overwrite python@3.11 || echo "I1.3" + /usr/local/bin/brew install python-gdbm@3.11 || echo "I1.4" + /usr/local/bin/brew install python-tk@3.11 || echo "I1.5" + for i in dpkg upx zip make wget jq rpm python3.11; do command -v $i || exit 1; done + echo "Dependencies checked" + - name: "Install golang" + run: | + wget https://go.dev/dl/go1.22.2.darwin-amd64.pkg + sudo installer -store -pkg go1.22.2.darwin-amd64.pkg -target / + - name: "Install Packages.pkg for making macos PKG files" + run: | + wget http://s.sudre.free.fr/Software/files/Packages.dmg + hdiutil attach -mountpoint /Volumes/Packages Packages.dmg + cd /Volumes/Packages + sudo installer -pkg Install\ Packages.pkg -target / + - name: "Compile" + env: + ADDCOMMIT: ${{ inputs.addCommit }} + run: | + buildcmd="build-prerelease" + [ "${ADDCOMMIT}" = "false" ] && buildcmd="build-official" + export PATH=$PATH:/usr/local/bin:/usr/local/go/bin + cd ~/work/asvec/asvec/src && make cleanall && make ${buildcmd} + - name: "Create linux packages" + env: + ADDCOMMIT: ${{ inputs.addCommit }} + run: | + buildcmd="build-prerelease" + [ "${ADDCOMMIT}" = "false" ] && buildcmd="build-official" + export PATH=$PATH:/usr/local/bin:/usr/local/go/bin + cd ~/work/asvec/asvec/src && make pkg-linux + - name: "Create windows zips" + env: + ADDCOMMIT: ${{ inputs.addCommit }} + run: | + buildcmd="build-prerelease" + [ "${ADDCOMMIT}" = "false" ] && buildcmd="build-official" + export PATH=$PATH:/usr/local/bin:/usr/local/go/bin + cd ~/work/asvec/asvec/src && make pkg-windows-zip + - name: "Print asvec version" + run: cd ~/work/asvec/asvec/src && ../bin/asvec-macos-amd64 version + - name: "Prepare keychain for signing MacOS" + env: + keypw: ${{ secrets.APPLEPW }} + INSTALLERP12: ${{ secrets.INSTALLERP12 }} + APPLICATIONP12: ${{ secrets.APPLICATIONP12 }} + run: | + set -e + security create-keychain -p mysecretpassword build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p mysecretpassword build.keychain + security set-keychain-settings build.keychain + security unlock-keychain -p mysecretpassword build.keychain + echo "$APPLICATIONP12" | base64 -d > app.p12 + echo "$INSTALLERP12" | base64 -d > install.p12 + security import app.p12 -k build.keychain -P $keypw -A + security import install.p12 -k build.keychain -P $keypw -A + security set-key-partition-list -S apple-tool:,apple: -s -k mysecretpassword build.keychain + - name: "Sign and build MacOS" + env: + xasvec_appleid: ${{ secrets.APPLEUSER }} + xasvec_applepw: ${{ secrets.APPLEPW }} + xasvec_signer: ${{ secrets.APPLESIGNER }} + xasvec_installsigner: ${{ secrets.APPLEINSTALLSIGNER }} + xasvec_teamid: ${{ secrets.APPLETEAMID }} + run: | + set -e + export asvec_appleid="${xasvec_appleid}" + export asvec_applepw="${xasvec_applepw}" + export asvec_signer="${xasvec_signer}" + export asvec_installsigner="${xasvec_installsigner}" + export asvec_teamid="${xasvec_teamid}" + export PATH=$PATH:/usr/local/bin:/usr/local/go/bin && cd ~/work/asvec/asvec/src && make macos-build-all && make macos-notarize-all + - name: "Create a new pre-release" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ADDCOMMIT: ${{ inputs.addCommit }} + run: | + set -e + echo "${ADDCOMMIT}" + cd ~/work/asvec/asvec/bin/packages + COMMIT=$(git rev-parse --short HEAD) + VER=$(cat ../../VERSION.md) + BRANCH=$(git rev-parse --abbrev-ref HEAD) + TAG=${VER}-${COMMIT} + [ "${ADDCOMMIT}" = "false" ] && TAG=${VER} + FULLCOMMIT=$(git rev-parse HEAD) + gh release create -R github.com/aerospike/asvec --notes-file ../../RELEASE.md --prerelease --target ${FULLCOMMIT} --title "Asvec - v${TAG}" ${TAG} asvec-linux-amd64-${VER}.deb asvec-linux-amd64-${VER}.rpm asvec-linux-amd64-${VER}.zip asvec-linux-arm64-${VER}.deb asvec-linux-arm64-${VER}.rpm asvec-linux-arm64-${VER}.zip asvec-macos-${VER}.pkg asvec-macos-amd64-${VER}.zip asvec-macos-arm64-${VER}.zip asvec-windows-amd64-${VER}.zip asvec-windows-arm64-${VER}.zip + - name: "Delete previous pre-release" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DELPREV: ${{ inputs.deletePrevBuild }} + run: | + if [ "${DELPREV}" = "true" ] + then + set -e + gh release list -R github.com/aerospike/asvec -L 100 |grep Pre-release |awk -F'\t' '{print $3}' |while read line + do + if [ "$line" != "${TAG}" ] + then + if [[ $line =~ ^${VER}- ]] + then + echo "Removing $line" + gh release delete $line -R github.com/aerospike/asvec --yes --cleanup-tag + fi + fi + done + fi \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..62b54a7 --- /dev/null +++ b/Makefile @@ -0,0 +1,491 @@ +.NOTPARALLEL: + +## requirements: make dpkg rpmbuild upx golang zip wget jq +## macos pkg requirement: https://docker-laptop.s3.eu-west-1.amazonaws.com/Packages.pkg + +## environment variable import +SIGNER := "$(asvec_signer)" +INSTALLSIGNER := "$(asvec_installsigner)" +APPLEID := "$(asvec_appleid)" +APPLEPW := "$(asvec_applepw)" +TEAMID := "$(asvec_teamid)" +BIN_DIR = ./bin + +## available make commands +.PHONY: help +help: + @printf "\nSHORTHANDS:\n\ + \tmake build && make install - build and install on current system\n\ + \tmake build-linux && make pkg-linux - build and package all linux releases\n\ + \n\ + UPDATE COMMANDS:\n\ + \tdeps - Update all dependencies in the project\n\ + \n\ + BUILD COMMANDS:\n\ + \tbuild - A version for the current system (linux and mac only)\n\ + \tbuildall - All versions for all supported systems\n\ + \tbuild-linux-amd64 - Linux on x86_64\n\ + \tbuild-linux-arm64 - Linux on aarch64\n\ + \tbuild-linux - Linux on x86_64 and aarach64\n\ + \tbuild-darwin-amd64 - MacOS on x86_64\n\ + \tbuild-darwin-arm64 - MacOS on M1/M2 style aarach64\n\ + \tbuild-darwin - MacOS on x86_64 and aarch64\n\ + \tbuild-windows-amd64- Windows on x86_64 and aarch64\n\ + \tbuild-windows-arm64- Windows on ARM\n\ + \tbuild-windows - Windows on x86_64 and arm64\n\ + \n\ + INSTALL COMMANDS:\n\ + \tinstall - Install a previously built asvec on the current system (linux and mac only)\n\ + \n\ + CLEAN COMMANDS:\n\ + \tclean - Remove remainders of a build and reset source modified during build\n\ + \tcleanall - Clean and remove all created packages\n\ + \n\ + LINUX PACKAGING COMMANDS:\n\ + \tpkg-linux - Package all linux packages - zip, rpm and deb\n\ + \tpkg-zip - Package linux zip\n\ + \tpkg-rpm - Package linux rpm\n\ + \tpkg-deb - Package linux deb\n\ + \tpkg-zip-amd64 - Package linux zip for amd64 only\n\ + \tpkg-rpm-amd64 - Package linux rpm for amd64 only\n\ + \tpkg-deb-amd64 - Package linux deb for amd64 only\n\ + \tpkg-zip-arm64 - Package linux zip for arm64 only\n\ + \tpkg-rpm-arm64 - Package linux rpm for arm64 only\n\ + \tpkg-deb-arm64 - Package linux deb for arm64 only\n\ + \n\ + WINDOWS PACKAGING COMMANDS:\n\ + \tpkg-windows-zip - Package windows zip\n\ + \tpkg-windows-zip-amd64 - Package windows zip for amd64 only\n\ + \tpkg-windows-zip-arm64 - Package windows zip for arm64 only\n\ + \n\ + MACOS PACKAGING COMMANDS:\n\ + \tmacos-codesign - Codesign MacOS binaries\n\ + \tmacos-pkg-build - Create MacOS pkg installer\n\ + \tmacos-pkg-sign - Productsign MacOS pkg installer\n\ + \tmacos-pkg-notarize - Codesign MacOS binaries\n\ + \tmacos-zip-build - Create MacOS zip packages\n\ + \tmacos-zip-notarize - Notarize MacOS ZIP packages\n\ + \tmacos-build-all - Build and sign pkg and zip\n\ + \tmacos-notarize-all - Notarize pkg and zip\n\ + \n\ + OUTPUTS: $(BIN_DIR)/ and $(BIN_DIR)/packages/\n\ + " + +.PHONY: deps +deps: + go get -u && go mod tidy && GOWORK=off go mod vendor + + +.PHONY: macos-build-all +macos-build-all: macos-codesign macos-zip-build macos-pkg-build macos-pkg-sign + +.PHONY: macos-notarize-all +macos-notarize-all: macos-pkg-notarize macos-zip-notarize + +.PHONY: build +build: run_build + +.PHONY: buildall +buildall: prep compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_linux_amd64 compile_linux_arm64 compile_darwin compile_windows reset2 + +.PHONY: build-linux-amd64 +build-linux-amd64: prep compile_linux_wip_arm64 reset1 compile_linux_amd64 reset2 + +.PHONY: build-linux-arm64 +build-linux-arm64: prep compile_linux_wip_amd64 reset1 compile_linux_arm64 reset2 + +.PHONY: build-linux +build-linux: prep compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_linux_amd64 compile_linux_arm64 reset2 + +.PHONY: build-darwin-amd64 +build-darwin-amd64: prep compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_darwin_amd64 reset2 + +.PHONY: build-darwin-arm64 +build-darwin-arm64: prep compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_darwin_arm64 reset2 + +.PHONY: build-darwin +build-darwin: prep compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_darwin reset2 + +.PHONY: build-windows-amd64 +build-windows-amd64: prep compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_windows_amd64 reset2 + +.PHONY: build-windows-arm64 +build-windows-arm64: prep compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_windows_arm64 reset2 + +.PHONY: build-windows +build-windows: prep compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_windows reset2 + +.PHONY: install +install: run_install + +.PHONY: cleanall +cleanall: clean + rm -f $(BIN_DIR)/packages/* + rm -f notarize_result_pkg notarize_result_amd64 notarize_result_arm64 + rm -f $(BIN_DIR)/AsVec.pkg + +.PHONY: clean +clean: + rm -f asvec-linux-amd64-wip + rm -f asvec-linux-arm64-wip + rm -f *.upx + rm -f asvec-linux-amd64 + rm -f asvec-linux-arm64 + rm -f asvec-macos-amd64 + rm -f asvec-macos-arm64 + rm -f asvec-windows-amd64.exe + rm -f asvec-windows-arm64.exe + rm -f embed_*.txt + rm -f myFunction.zip + rm -f gcpMod.txt + rm -f gcpFunction.txt + rm -f $(BIN_DIR)/asvec-* + rm -f $(BIN_DIR)/deb + rm -f $(BIN_DIR)/deb.deb + rm -f $(BIN_DIR)/asvec + printf "package main\n\nvar nLinuxBinaryX64 []byte\n\nvar nLinuxBinaryArm64 []byte\n" > embed_linux.go + cp embed_linux.go embed_darwin.go + cp embed_linux.go embed_windows.go + +## actual code + +OS := $(shell uname -o) +CPU := $(shell uname -m) +ver:=$(shell V=$$(git branch --show-current); if [[ $$V == v* ]]; then printf $${V:1} > ./VERSION.md; fi; cat ./VERSION.md) +define _amddebscript +ver=$(cat ./VERSION.md) +cat < $(BIN_DIR)/deb/DEBIAN/control +Website: www.aerospike.com +Maintainer: Aerospike +Name: Aerospike Vector CLI +Package: asvec +Section: aerospike +Version: ${ver} +Architecture: amd64 +Description: Tool for deploying non-prod Aerospike server clusters on docker, GCP or in AWS +EOF +endef +export amddebscript = $(value _amddebscript) +define _armdebscript +ver=$(cat ./VERSION.md) +cat < $(BIN_DIR)/deb/DEBIAN/control +Website: www.aerospike.com +Maintainer: Aerospike +Name: Aerospike Vector CLI +Package: asvec +Section: aerospike +Version: ${ver} +Architecture: arm64 +Description: Tool for deploying non-prod Aerospike server clusters on docker, GCP or in AWS +EOF +endef +export armdebscript = $(value _armdebscript) + +.PHONY: run_build +run_build: +ifeq ($(OS), Darwin) +ifeq ($(CPU), x86_64) + $(MAKE) build-darwin-amd64 +else + $(MAKE) build-darwin-arm64 +endif +else +ifeq ($(CPU), x86_64) + $(MAKE) build-linux-amd64 +else + $(MAKE) build-linux-arm64 +endif +endif + +.PHONY: run_install +run_install: +ifeq ($(OS), Darwin) +ifeq ($(CPU), x86_64) + sudo cp $(BIN_DIR)/asvec-macos-amd64 /usr/local/bin/asvec +else + sudo cp $(BIN_DIR)/asvec-macos-arm64 /usr/local/bin/asvec +endif +else +ifeq ($(CPU), x86_64) + sudo cp $(BIN_DIR)/asvec-linux-amd64 /usr/local/bin/asvec +else + sudo cp $(BIN_DIR)/asvec-linux-arm64 /usr/local/bin/asvec +endif +endif + +.PHONY: reset1 +reset1: + printf "package main\n\nvar nLinuxBinaryX64 []byte\n\nvar nLinuxBinaryArm64 []byte\n" > embed_linux.go + cp embed_linux.go embed_darwin.go + cp embed_linux.go embed_windows.go + +.PHONY: reset2 +reset2: + rm -f asvec-linux-amd64-wip + rm -f asvec-linux-arm64-wip + rm -f *.upx + rm -f embed_*.txt + printf "package main\n\nvar nLinuxBinaryX64 []byte\n\nvar nLinuxBinaryArm64 []byte\n" > embed_linux.go + cp embed_linux.go embed_darwin.go + cp embed_linux.go embed_windows.go + +.PHONY: prep +prep: + go generate + +.PHONY: compile_linux_wip_amd64 +compile_linux_wip_amd64: + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o asvec-linux-amd64-wip +ifneq (, $(shell which upx)) + upx asvec-linux-amd64-wip +endif + +.PHONY: compile_linux_wip_arm64 +compile_linux_wip_arm64: + env CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o asvec-linux-arm64-wip +ifneq (, $(shell which upx)) + upx asvec-linux-arm64-wip +endif + +.PHONY: compile_linux_amd64 +compile_linux_amd64: + printf "package main\n\nimport _ \"embed\"\n\nvar nLinuxBinaryX64 []byte\n\n//go:embed asvec-linux-arm64-wip\nvar nLinuxBinaryArm64 []byte\n" > embed_linux.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o asvec-linux-amd64 + mv asvec-linux-amd64 $(BIN_DIR)/ + +.PHONY: compile_linux_arm64 +compile_linux_arm64: + printf "package main\n\nimport _ \"embed\"\n\n//go:embed asvec-linux-amd64-wip\nvar nLinuxBinaryX64 []byte\n\nvar nLinuxBinaryArm64 []byte\n" > embed_linux.go + env CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o asvec-linux-arm64 + mv asvec-linux-arm64 $(BIN_DIR)/ + +.PHONY: compile_darwin +compile_darwin: + printf "package main\n\nimport _ \"embed\"\n\n//go:embed asvec-linux-amd64-wip\nvar nLinuxBinaryX64 []byte\n\n//go:embed asvec-linux-arm64-wip\nvar nLinuxBinaryArm64 []byte" > embed_darwin.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o asvec-macos-amd64 + env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o asvec-macos-arm64 + mv asvec-macos-amd64 $(BIN_DIR)/ + mv asvec-macos-arm64 $(BIN_DIR)/ + +.PHONY: compile_darwin_amd64 +compile_darwin_amd64: + printf "package main\n\nimport _ \"embed\"\n\n//go:embed asvec-linux-amd64-wip\nvar nLinuxBinaryX64 []byte\n\n//go:embed asvec-linux-arm64-wip\nvar nLinuxBinaryArm64 []byte" > embed_darwin.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o asvec-macos-amd64 + mv asvec-macos-amd64 $(BIN_DIR)/ + +.PHONY: compile_darwin_arm64 +compile_darwin_arm64: + printf "package main\n\nimport _ \"embed\"\n\n//go:embed asvec-linux-amd64-wip\nvar nLinuxBinaryX64 []byte\n\n//go:embed asvec-linux-arm64-wip\nvar nLinuxBinaryArm64 []byte" > embed_darwin.go + env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o asvec-macos-arm64 + mv asvec-macos-arm64 $(BIN_DIR)/ + +.PHONY: compile_windows +compile_windows: + printf "package main\n\nimport _ \"embed\"\n\n//go:embed asvec-linux-amd64-wip\nvar nLinuxBinaryX64 []byte\n\n//go:embed asvec-linux-arm64-wip\nvar nLinuxBinaryArm64 []byte" > embed_windows.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o asvec-windows-amd64.exe + env CGO_ENABLED=0 GOOS=windows GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o asvec-windows-arm64.exe + mv asvec-windows-amd64.exe $(BIN_DIR)/ + mv asvec-windows-arm64.exe $(BIN_DIR)/ + +.PHONY: compile_windows_amd64 +compile_windows_amd64: + printf "package main\n\nimport _ \"embed\"\n\n//go:embed asvec-linux-amd64-wip\nvar nLinuxBinaryX64 []byte\n\n//go:embed asvec-linux-arm64-wip\nvar nLinuxBinaryArm64 []byte" > embed_windows.go + env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o asvec-windows-amd64.exe + mv asvec-windows-amd64.exe $(BIN_DIR)/ + +.PHONY: compile_windows_arm64 +compile_windows_arm64: + printf "package main\n\nimport _ \"embed\"\n\n//go:embed asvec-linux-amd64-wip\nvar nLinuxBinaryX64 []byte\n\n//go:embed asvec-linux-arm64-wip\nvar nLinuxBinaryArm64 []byte" > embed_windows.go + env CGO_ENABLED=0 GOOS=windows GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o asvec-windows-arm64.exe + mv asvec-windows-arm64.exe $(BIN_DIR)/ + +.PHONY: official +official: prep + printf "" > embed_tail.txt + +.PHONY: prerelease +prerelease: prep + printf -- "-prerelease" > embed_tail.txt + +.PHONY: build-official +build-official: official compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_linux_amd64 compile_linux_arm64 compile_darwin compile_windows reset2 + +.PHONY: build-prerelease +build-prerelease: prerelease compile_linux_wip_amd64 compile_linux_wip_arm64 reset1 compile_linux_amd64 compile_linux_arm64 compile_darwin compile_windows reset2 + +RET := $(shell echo) + +.PHONY: pkg-deb-amd64 +pkg-deb-amd64: + cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec + rm -rf $(BIN_DIR)/deb + mkdir -p $(BIN_DIR)/deb/DEBIAN + mkdir -p $(BIN_DIR)/deb/usr/bin + @ eval "$$amddebscript" + mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/bin/ + sudo dpkg-deb -Zxz -b $(BIN_DIR)/deb + rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.deb + mv $(BIN_DIR)/deb.deb $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.deb + rm -rf $(BIN_DIR)/deb + +.PHONY: pkg-deb-arm64 +pkg-deb-arm64: + cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec + rm -rf $(BIN_DIR)/deb + mkdir -p $(BIN_DIR)/deb/DEBIAN + mkdir -p $(BIN_DIR)/deb/usr/bin + @ eval "$$armdebscript" + mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/bin/ + sudo dpkg-deb -Zxz -b $(BIN_DIR)/deb + rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.deb + mv $(BIN_DIR)/deb.deb $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.deb + rm -rf $(BIN_DIR)/deb + +.PHONY: pkg-deb +pkg-deb: pkg-deb-amd64 pkg-deb-arm64 + +.PHONY: pkg-zip-amd64 +pkg-zip-amd64: + cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec + rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.zip + bash -ce "cd $(BIN_DIR) && zip packages/asvec-linux-amd64-${ver}.zip asvec" + rm -f $(BIN_DIR)/asvec + +.PHONY: pkg-zip-arm64 +pkg-zip-arm64: + cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec + rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.zip + bash -ce "cd $(BIN_DIR) && zip packages/asvec-linux-arm64-${ver}.zip asvec" + rm -f $(BIN_DIR)/asvec + +.PHONY: pkg-windows-zip +pkg-windows-zip: pkg-windows-zip-amd64 pkg-windows-zip-arm64 + +.PHONY: pkg-windows-zip-amd64 +pkg-windows-zip-amd64: + cp $(BIN_DIR)/asvec-windows-amd64.exe $(BIN_DIR)/asvec.exe + rm -f $(BIN_DIR)/packages/asvec-windows-amd64-${ver}.zip + bash -ce "cd $(BIN_DIR) && zip packages/asvec-windows-amd64-${ver}.zip asvec.exe" + rm -f $(BIN_DIR)/asvec.exe + +.PHONY: pkg-windows-zip-arm64 +pkg-windows-zip-arm64: + cp $(BIN_DIR)/asvec-windows-arm64.exe $(BIN_DIR)/asvec.exe + rm -f $(BIN_DIR)/packages/asvec-windows-arm64-${ver}.zip + bash -ce "cd $(BIN_DIR) && zip packages/asvec-windows-arm64-${ver}.zip asvec.exe" + rm -f $(BIN_DIR)/asvec.exe + +.PHONY: pkg-zip +pkg-zip: pkg-zip-amd64 pkg-zip-arm64 + +.PHONY: pkg-rpm-amd64 +pkg-rpm-amd64: + rm -rf $(BIN_DIR)/asvec-rpm-centos + cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos + sed -i.bak "s/VERSIONHERE/${ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec + cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec-rpm-centos/usr/bin/asvec + rm -f $(BIN_DIR)/asvec-linux-x86_64.rpm + bash -ce "cd $(BIN_DIR) && rpmbuild --target=x86_64-redhat-linux --buildroot \$$(pwd)/asvec-rpm-centos -bb asvec-rpm-centos/asvec.spec" + rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.rpm + mv $(BIN_DIR)/asvec-linux-x86_64.rpm $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.rpm + +.PHONY: pkg-rpm-arm64 +pkg-rpm-arm64: + rm -rf $(BIN_DIR)/asvec-rpm-centos + cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos + sed -i.bak "s/VERSIONHERE/${ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec + cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec-rpm-centos/usr/bin/asvec + rm -f $(BIN_DIR)/asvec-linux-arm64.rpm + bash -ce "cd $(BIN_DIR) && rpmbuild --target=arm64-redhat-linux --buildroot \$$(pwd)/asvec-rpm-centos -bb asvec-rpm-centos/asvec.spec" + rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.rpm + mv $(BIN_DIR)/asvec-linux-arm64.rpm $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.rpm + +.PHONY: pkg-rpm +pkg-rpm: pkg-rpm-amd64 pkg-rpm-arm64 + +.PHONY: pkg-linux +pkg-linux: pkg-zip pkg-deb pkg-rpm + +### note - static linking +###go build -ldflags="-extldflags=-static" + +.PHONY: macos-codesign +macos-codesign: +ifeq (exists, $(shell [ -f $(BIN_DIR)/asvec-macos-amd64 ] && echo "exists" || echo "not found")) + codesign --verbose --deep --timestamp --force --options runtime --sign ${SIGNER} $(BIN_DIR)/asvec-macos-amd64 + codesign --verbose --verify $(BIN_DIR)/asvec-macos-amd64 +endif +ifeq (exists, $(shell [ -f $(BIN_DIR)/asvec-macos-arm64 ] && echo "exists" || echo "not found")) + codesign --verbose --deep --timestamp --force --options runtime --sign ${SIGNER} $(BIN_DIR)/asvec-macos-arm64 + codesign --verbose --verify $(BIN_DIR)/asvec-macos-arm64 +endif + +.PHONY: macos-zip-build +macos-zip-build: +ifeq (exists, $(shell [ -f $(BIN_DIR)/asvec-macos-amd64 ] && echo "exists" || echo "not found")) + cp $(BIN_DIR)/asvec-macos-amd64 $(BIN_DIR)/asvec + rm -f $(BIN_DIR)/packages/asvec-macos-amd64-${ver}.zip + bash -ce "cd $(BIN_DIR) && zip packages/asvec-macos-amd64-${ver}.zip asvec" + rm -f $(BIN_DIR)/asvec +endif +ifeq (exists, $(shell [ -f $(BIN_DIR)/asvec-macos-arm64 ] && echo "exists" || echo "not found")) + cp $(BIN_DIR)/asvec-macos-arm64 $(BIN_DIR)/asvec + rm -f $(BIN_DIR)/packages/asvec-macos-arm64-${ver}.zip + bash -ce "cd $(BIN_DIR) && zip packages/asvec-macos-arm64-${ver}.zip asvec" + rm -f $(BIN_DIR)/asvec +endif + +.PHONY: macos-zip-notarize +macos-zip-notarize: +ifeq (exists, $(shell [ -f $(BIN_DIR)/packages/asvec-macos-amd64-${ver}.zip ] && echo "exists" || echo "not found")) + rm -f notarize_result_amd64 + xcrun notarytool submit --apple-id ${APPLEID} --password ${APPLEPW} --team-id ${TEAMID} -f json --wait --timeout 10m $(BIN_DIR)/packages/asvec-macos-amd64-${ver}.zip > notarize_result_amd64 + if [ "$$(cat notarize_result_amd64 |jq -r .status)" != "Accepted" ] ;\ + then \ + echo "ZIP-AMD FAILED TO NOTARIZE" ;\ + cat notarize_result_amd64 ;\ + exit 1 ;\ + else \ + echo "ZIP-AMD NOTARIZE SUCCESS" ;\ + fi +endif +ifeq (exists, $(shell [ -f $(BIN_DIR)/packages/asvec-macos-arm64-${ver}.zip ] && echo "exists" || echo "not found")) + rm -f notarize_result_arm64 + xcrun notarytool submit --apple-id ${APPLEID} --password ${APPLEPW} --team-id ${TEAMID} -f json --wait --timeout 10m $(BIN_DIR)/packages/asvec-macos-arm64-${ver}.zip > notarize_result_arm64 + if [ "$$(cat notarize_result_arm64 |jq -r .status)" != "Accepted" ] ;\ + then \ + echo "ZIP-ARM FAILED TO NOTARIZE" ;\ + cat notarize_result_arm64 ;\ + exit 1 ;\ + else \ + echo "ZIP-ARM NOTARIZE SUCCESS" ;\ + fi +endif + +.PHONY: macos-pkg-build +macos-pkg-build: + cp -a $(BIN_DIR)/asvec-macos-amd64 $(BIN_DIR)/macos-pkg/asvec/ + cp -a $(BIN_DIR)/asvec-macos-arm64 $(BIN_DIR)/macos-pkg/asvec/ + sed "s/ASVECVERSIONHERE/${ver}/g" $(BIN_DIR)/macos-pkg/AsVec-template.pkgproj > $(BIN_DIR)/macos-pkg/AsVec.pkgproj + bash -ce "cd $(BIN_DIR)/macos-pkg && /usr/local/bin/packagesbuild --project AsVec.pkgproj" + mv $(BIN_DIR)/macos-pkg/build/AsVec.pkg $(BIN_DIR)/asvec-macos-${ver}-unsigned.pkg + +.PHONY: macos-pkg-sign +macos-pkg-sign: + productsign --timestamp --sign ${INSTALLSIGNER} $(BIN_DIR)/asvec-macos-${ver}-unsigned.pkg $(BIN_DIR)/packages/asvec-macos-${ver}.pkg + +.PHONY: macos-pkg-notarize +macos-pkg-notarize: + rm -f notarize_result_pkg + xcrun notarytool submit --apple-id ${APPLEID} --password ${APPLEPW} --team-id ${TEAMID} -f json --wait --timeout 10m $(BIN_DIR)/packages/asvec-macos-${ver}.pkg > notarize_result_pkg + if [ "$$(cat notarize_result_pkg |jq -r .status)" != "Accepted" ] ;\ + then \ + echo "PKG FAILED TO NOTARIZE" ;\ + cat notarize_result_pkg ;\ + exit 1 ;\ + else \ + echo "PKG NOTARIZE SUCCESS" ;\ + fi + +### make cleanall && make build-prerelease && make pkg-linux && make pkg-windows-zip && make macos-build-all && make macos-notarize-all +### make cleanall && make build-official && make pkg-linux && make pkg-windows-zip && make macos-build-all && make macos-notarize-all \ No newline at end of file diff --git a/VERSION.md b/VERSION.md new file mode 100644 index 0000000..6c6aa7c --- /dev/null +++ b/VERSION.md @@ -0,0 +1 @@ +0.1.0 \ No newline at end of file diff --git a/bin/asvecrpm/asvec.spec b/bin/asvecrpm/asvec.spec new file mode 100644 index 0000000..2891356 --- /dev/null +++ b/bin/asvecrpm/asvec.spec @@ -0,0 +1,20 @@ +Buildroot: ./ +Name: aerolab +Version: VERSIONHERE +Release: 2 +Summary: Tool for deploying non-prod Aerospike server clusters on docker or in AWS +License: see github.com/aerospike/aerolab +Group: aerospike + +%define _rpmdir ./ +%define _rpmfilename %%{NAME}-linux-%%{ARCH}.rpm +%define _unpackaged_files_terminate_build 0 +%define _binaries_in_noarch_packages_terminate_build 0 + +%description + + +Tool for deploying non-prod Aerospike server clusters on docker or in AWS + +%files +"/usr/bin/aerolab" diff --git a/bin/asvecrpm/usr/bin/.gitignore b/bin/asvecrpm/usr/bin/.gitignore new file mode 100644 index 0000000..29c3f11 --- /dev/null +++ b/bin/asvecrpm/usr/bin/.gitignore @@ -0,0 +1 @@ +aerolab diff --git a/bin/macos-pkg/.gitignore b/bin/macos-pkg/.gitignore new file mode 100644 index 0000000..d0585b7 --- /dev/null +++ b/bin/macos-pkg/.gitignore @@ -0,0 +1,3 @@ +/build +/build/** +/AeroLab.pkgproj \ No newline at end of file diff --git a/bin/macos-pkg/AsVec-template.pkgproj b/bin/macos-pkg/AsVec-template.pkgproj new file mode 100755 index 0000000..8cec69f --- /dev/null +++ b/bin/macos-pkg/AsVec-template.pkgproj @@ -0,0 +1,698 @@ + + + + + PROJECT + + PACKAGE_FILES + + DEFAULT_INSTALL_LOCATION + /Library + HIERARCHY + + CHILDREN + + + CHILDREN + + GID + 80 + PATH + Applications + PATH_TYPE + 0 + PERMISSIONS + 509 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 0 + PATH + asvec + PATH_TYPE + 1 + PERMISSIONS + 493 + TYPE + 3 + UID + 0 + + + CHILDREN + + GID + 80 + PATH + Application Support + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Automator + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Documentation + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Extensions + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Filesystems + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Frameworks + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Input Methods + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Internet Plug-Ins + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Keyboard Layouts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchAgents + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchDaemons + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PreferencePanes + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Preferences + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 80 + PATH + Printers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PrivilegedHelperTools + PATH_TYPE + 0 + PERMISSIONS + 1005 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickLook + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickTime + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Screen Savers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Scripts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Services + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Widgets + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + Library + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 0 + PATH + Shared + PATH_TYPE + 0 + PERMISSIONS + 1023 + TYPE + 1 + UID + 0 + + + GID + 80 + PATH + Users + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + / + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + PAYLOAD_TYPE + 0 + PRESERVE_EXTENDED_ATTRIBUTES + + SHOW_INVISIBLE + + SPLIT_FORKS + + TREAT_MISSING_FILES_AS_WARNING + + VERSION + 5 + + PACKAGE_SCRIPTS + + POSTINSTALL_PATH + + PATH + install.sh + PATH_TYPE + 1 + + PREINSTALL_PATH + + PATH_TYPE + 1 + + RESOURCES + + + PACKAGE_SETTINGS + + AUTHENTICATION + 1 + CONCLUSION_ACTION + 0 + FOLLOW_SYMBOLIC_LINKS + + IDENTIFIER + com.aerospike.asvec + LOCATION + 0 + NAME + + OVERWRITE_PERMISSIONS + + PAYLOAD_SIZE + -1 + REFERENCE_PATH + + RELOCATABLE + + USE_HFS+_COMPRESSION + + VERSION + ASVECVERSIONHERE + + PROJECT_COMMENTS + + NOTES + + + + PROJECT_SETTINGS + + BUILD_PATH + + PATH + build + PATH_TYPE + 1 + + EXCLUDED_FILES + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .DS_Store + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .DS_Store files + PROXY_TOOLTIP + Remove ".DS_Store" files created by the Finder. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .pbdevelopment + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .pbdevelopment files + PROXY_TOOLTIP + Remove ".pbdevelopment" files created by ProjectBuilder or Xcode. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + CVS + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .cvsignore + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .cvspass + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .svn + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .git + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .gitignore + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove SCM metadata + PROXY_TOOLTIP + Remove helper files and folders used by the CVS, SVN or Git Source Code Management systems. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + classes.nib + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + designable.db + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + info.nib + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Optimize nib files + PROXY_TOOLTIP + Remove "classes.nib", "info.nib" and "designable.nib" files within .nib bundles. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + Resources Disabled + TYPE + 1 + + + PROTECTED + + PROXY_NAME + Remove Resources Disabled folders + PROXY_TOOLTIP + Remove "Resources Disabled" folders. + STATE + + + + SEPARATOR + + + + NAME + asvec + PAYLOAD_ONLY + + + + TYPE + 1 + VERSION + 2 + + diff --git a/bin/macos-pkg/AsVec.pkgproj b/bin/macos-pkg/AsVec.pkgproj new file mode 100644 index 0000000..d5cf099 --- /dev/null +++ b/bin/macos-pkg/AsVec.pkgproj @@ -0,0 +1,698 @@ + + + + + PROJECT + + PACKAGE_FILES + + DEFAULT_INSTALL_LOCATION + /Library + HIERARCHY + + CHILDREN + + + CHILDREN + + GID + 80 + PATH + Applications + PATH_TYPE + 0 + PERMISSIONS + 509 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 0 + PATH + asvec + PATH_TYPE + 1 + PERMISSIONS + 493 + TYPE + 3 + UID + 0 + + + CHILDREN + + GID + 80 + PATH + Application Support + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Automator + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Documentation + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Extensions + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Filesystems + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Frameworks + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Input Methods + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Internet Plug-Ins + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Keyboard Layouts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchAgents + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + LaunchDaemons + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PreferencePanes + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Preferences + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 80 + PATH + Printers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + PrivilegedHelperTools + PATH_TYPE + 0 + PERMISSIONS + 1005 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickLook + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + QuickTime + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Screen Savers + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Scripts + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Services + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + GID + 0 + PATH + Widgets + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + Library + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + CHILDREN + + + CHILDREN + + GID + 0 + PATH + Shared + PATH_TYPE + 0 + PERMISSIONS + 1023 + TYPE + 1 + UID + 0 + + + GID + 80 + PATH + Users + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + + GID + 0 + PATH + / + PATH_TYPE + 0 + PERMISSIONS + 493 + TYPE + 1 + UID + 0 + + PAYLOAD_TYPE + 0 + PRESERVE_EXTENDED_ATTRIBUTES + + SHOW_INVISIBLE + + SPLIT_FORKS + + TREAT_MISSING_FILES_AS_WARNING + + VERSION + 5 + + PACKAGE_SCRIPTS + + POSTINSTALL_PATH + + PATH + install.sh + PATH_TYPE + 1 + + PREINSTALL_PATH + + PATH_TYPE + 1 + + RESOURCES + + + PACKAGE_SETTINGS + + AUTHENTICATION + 1 + CONCLUSION_ACTION + 0 + FOLLOW_SYMBOLIC_LINKS + + IDENTIFIER + com.aerospike.asvec + LOCATION + 0 + NAME + + OVERWRITE_PERMISSIONS + + PAYLOAD_SIZE + -1 + REFERENCE_PATH + + RELOCATABLE + + USE_HFS+_COMPRESSION + + VERSION + 0.1.0 + + PROJECT_COMMENTS + + NOTES + + + + PROJECT_SETTINGS + + BUILD_PATH + + PATH + build + PATH_TYPE + 1 + + EXCLUDED_FILES + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .DS_Store + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .DS_Store files + PROXY_TOOLTIP + Remove ".DS_Store" files created by the Finder. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + .pbdevelopment + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove .pbdevelopment files + PROXY_TOOLTIP + Remove ".pbdevelopment" files created by ProjectBuilder or Xcode. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + CVS + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .cvsignore + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .cvspass + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + .svn + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .git + TYPE + 1 + + + REGULAR_EXPRESSION + + STRING + .gitignore + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Remove SCM metadata + PROXY_TOOLTIP + Remove helper files and folders used by the CVS, SVN or Git Source Code Management systems. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + classes.nib + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + designable.db + TYPE + 0 + + + REGULAR_EXPRESSION + + STRING + info.nib + TYPE + 0 + + + PROTECTED + + PROXY_NAME + Optimize nib files + PROXY_TOOLTIP + Remove "classes.nib", "info.nib" and "designable.nib" files within .nib bundles. + STATE + + + + PATTERNS_ARRAY + + + REGULAR_EXPRESSION + + STRING + Resources Disabled + TYPE + 1 + + + PROTECTED + + PROXY_NAME + Remove Resources Disabled folders + PROXY_TOOLTIP + Remove "Resources Disabled" folders. + STATE + + + + SEPARATOR + + + + NAME + asvec + PAYLOAD_ONLY + + + + TYPE + 1 + VERSION + 2 + + diff --git a/bin/macos-pkg/asvec/.gitignore b/bin/macos-pkg/asvec/.gitignore new file mode 100644 index 0000000..a8e9ffa --- /dev/null +++ b/bin/macos-pkg/asvec/.gitignore @@ -0,0 +1 @@ +aerolab-* \ No newline at end of file diff --git a/bin/macos-pkg/install.sh b/bin/macos-pkg/install.sh new file mode 100755 index 0000000..ccd5c06 --- /dev/null +++ b/bin/macos-pkg/install.sh @@ -0,0 +1,11 @@ +#!/bin/bash +mkdir -p /usr/local/bin/ +BIN=asvec-macos-amd64 +uname -p |grep arm && BIN=asvec-macos-arm64 || echo "amd" +uname -m |grep arm && BIN=asvec-macos-arm64 || echo "amd" +chmod 755 /Library/asvec/* +rm -f /usr/local/bin/asvec || echo "first_install" +ln -s /Library/asvec/${BIN} /usr/local/bin/asvec +ln -s /Library/asvec/${BIN} /usr/local/aerospike/bin/asvec +mkdir -p /etc/paths.d || echo "path install will fail" +echo "/usr/local/bin" |tee /etc/paths.d/asvec || echo "path install failed" diff --git a/bin/packages/.gitignore b/bin/packages/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/bin/packages/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/cmd/create.go b/cmd/create.go index a5e613c..5cf7f1b 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -1,12 +1,9 @@ /* Copyright © 2024 NAME HERE - */ package cmd import ( - "fmt" - "github.com/spf13/cobra" ) @@ -20,9 +17,6 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("create called") - }, } func init() { diff --git a/cmd/createIndex.go b/cmd/createIndex.go index a6297c6..d639d60 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -14,15 +14,6 @@ import ( "github.com/spf13/viper" ) -var requiredFlags = []string{ - flagNameNamespace, - flagNameIndexName, - flagNameDimension, - flagNameDistance, -} - -var persistentRequiredFlags = []string{} - const ( flagNameMaxEdges = "hnsw-max-edges" flagNameConstructionEf = "hnsw-ef-construction" @@ -42,38 +33,53 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { + // TODO: likely add to prerun step seed := viper.GetString(flagNameSeeds) port := viper.GetInt(flagNamePort) hostPort := avs.NewHostPort(seed, port, false) namespace := viper.GetString(flagNameNamespace) sets := viper.GetStringSlice(flagNameSets) indexName := viper.GetString(flagNameIndexName) - vectorField := viper.GetString(flagNameVector) + vectorField := viper.GetString(flagNameVectorField) dimension := viper.GetUint32(flagNameDimension) indexMeta := viper.GetStringMapString(flagNameIndexMeta) distanceMetric := viper.GetString(flagNameDistance) + timeout := viper.GetDuration(flagNameTimeout) - logger.Debug("parsed flags", slog.String("seeds", seed), slog.Int("port", port), slog.String("namespace", namespace), slog.Any("sets", sets), slog.String("index-name", indexName), slog.String("vector-field", vectorField), slog.Uint64("dimension", uint64(dimension)), slog.Any("index-meta", indexMeta)) + logger.Debug("parsed flags", + slog.String("seeds", seed), slog.Int("port", port), slog.String("namespace", namespace), + slog.Any("sets", sets), slog.String("index-name", indexName), slog.String("vector-field", vectorField), + slog.Uint64("dimension", uint64(dimension)), slog.Any("index-meta", indexMeta), slog.String("distance-metric", distanceMetric), + slog.Duration("timeout", timeout), + ) - ctx := context.TODO() + ctx := context.Background() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() adminClient, err := avs.NewAdminClient(ctx, []*avs.HostPort{hostPort}, nil, false, logger) if err != nil { logger.Error("failed to create AVS client", slog.Any("error", err)) - view.Printf("Failed to connect to AVS: %v", err) - return + return err } - // TODO: parse cosine + cancel() + defer adminClient.Close() + + ctx, cancel = context.WithTimeout(context.Background(), timeout) + defer cancel() + + // TODO: Add storage type err = adminClient.IndexCreate(ctx, namespace, sets, indexName, vectorField, dimension, protos.VectorDistanceMetric(protos.VectorDistanceMetric_value[distanceMetric]), nil, indexMeta) if err != nil { logger.Error("unable to create index", slog.Any("error", err)) - view.Printf("Unable to create index: %v", err) - return + return err } view.Printf("Successfully created index %s.%s", namespace, indexName) + return nil }, } @@ -93,7 +99,18 @@ func init() { flags.AddDimensionFlag() flags.AddDistanceMetricFlag() flags.AddIndexMetaFlag() + flags.AddTimeoutFlag() + + var requiredFlags = []string{ + flagNameNamespace, + flagNameIndexName, + flagNameVectorField, + flagNameDimension, + flagNameDistance, + } + // TODO: Add custom template for usage to take into account terminal width + // Ex: https://github.com/sigstore/cosign/pull/3011/files flags.Uint32(flagNameMaxEdges, 0, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) flags.Uint32(flagNameConstructionEf, 0, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) flags.Uint32(flagNameEf, 0, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) @@ -104,13 +121,4 @@ func init() { for _, flag := range requiredFlags { createIndexCmd.MarkFlagRequired(flag) } - - for _, flag := range persistentRequiredFlags { - createIndexCmd.MarkPersistentFlagRequired(flag) - } - - // TODO hnsw metadata - viper.BindPFlags(createIndexCmd.PersistentFlags()) - viper.BindPFlags(createIndexCmd.Flags()) - } diff --git a/cmd/deleteIndex.go b/cmd/deleteIndex.go deleted file mode 100644 index 39a6052..0000000 --- a/cmd/deleteIndex.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright © 2024 NAME HERE - -*/ -package cmd - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -// deleteIndexCmd represents the deleteIndex command -var deleteIndexCmd = &cobra.Command{ - Use: "deleteIndex", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("deleteIndex called") - }, -} - -func init() { - deleteCmd.AddCommand(deleteIndexCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // deleteIndexCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // deleteIndexCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/cmd/delete.go b/cmd/drop.go similarity index 66% rename from cmd/delete.go rename to cmd/drop.go index 679dbb1..267021c 100644 --- a/cmd/delete.go +++ b/cmd/drop.go @@ -1,18 +1,15 @@ /* Copyright © 2024 NAME HERE - */ package cmd import ( - "fmt" - "github.com/spf13/cobra" ) -// deleteCmd represents the delete command -var deleteCmd = &cobra.Command{ - Use: "delete", +// dropCmd represents the drop command +var dropCmd = &cobra.Command{ + Use: "drop", Short: "A brief description of your command", Long: `A longer description that spans multiple lines and likely contains examples and usage of using your command. For example: @@ -20,21 +17,18 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("delete called") - }, } func init() { - rootCmd.AddCommand(deleteCmd) + rootCmd.AddCommand(dropCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: - // deleteCmd.PersistentFlags().String("foo", "", "A help for foo") + // dropCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: - // deleteCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + // dropCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go new file mode 100644 index 0000000..f3da4ad --- /dev/null +++ b/cmd/dropIndex.go @@ -0,0 +1,90 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "context" + "log/slog" + + avs "github.com/aerospike/aerospike-proximus-client-go" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var dropIndexRequiredFlags = []string{ + flagNameNamespace, + flagNameIndexName, + flagNameDimension, + flagNameDistance, +} + +// dropIndexCmd represents the dropIndex command +var dropIndexCmd = &cobra.Command{ + Use: "index", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + RunE: func(cmd *cobra.Command, args []string) error { + // TODO: likely add to prerun step + seed := viper.GetString(flagNameSeeds) + port := viper.GetInt(flagNamePort) + hostPort := avs.NewHostPort(seed, port, false) + namespace := viper.GetString(flagNameNamespace) + indexName := viper.GetString(flagNameIndexName) + timeout := viper.GetDuration(flagNameTimeout) + + logger.Debug("parsed flags", + slog.String("seeds", seed), slog.Int("port", port), slog.String("namespace", namespace), + slog.String("index-name", indexName), slog.Duration("timeout", timeout), + ) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + adminClient, err := avs.NewAdminClient(ctx, avs.HostPortSlice{hostPort}, nil, false, logger) + if err != nil { + logger.Error("failed to create AVS client", slog.Any("error", err)) + return err + } + + cancel() + defer adminClient.Close() + + ctx, cancel = context.WithTimeout(context.Background(), timeout) + defer cancel() + + err = adminClient.IndexDrop(ctx, namespace, indexName) + if err != nil { + logger.Error("unable to drop index", slog.Any("error", err)) + return err + } + + view.Printf("Successfully dropped index %s.%s", namespace, indexName) + return nil + }, +} + +func init() { + dropCmd.AddCommand(dropIndexCmd) + + flags := NewFlagSetBuilder(dropIndexCmd.Flags()) + flags.AddSeedFlag() + flags.AddPortFlag() + flags.AddNamespaceFlag() + flags.AddIndexNameFlag() + flags.AddTimeoutFlag() + + var requiredFlags = []string{ + flagNameNamespace, + flagNameIndexName, + } + + for _, flag := range requiredFlags { + dropIndexCmd.MarkFlagRequired(flag) + } +} diff --git a/cmd/flags.go b/cmd/flags.go index a4f2887..47ea14e 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -4,21 +4,24 @@ import ( "fmt" "net" "strings" + "time" "github.com/aerospike/aerospike-proximus-client-go/protos" "github.com/spf13/pflag" ) const ( - flagNameSeeds = "seeds" - flagNamePort = "port" - flagNameNamespace = "namespace" - flagNameSets = "sets" - flagNameIndexName = "index-name" - flagNameVector = "vector-field" - flagNameDimension = "dimension" - flagNameDistance = "distance-metric" - flagNameIndexMeta = "index-meta" + flagNameSeeds = "seeds" + flagNamePort = "port" + flagNameNamespace = "namespace" + flagNameSets = "sets" + flagNameIndexName = "index-name" + flagNameVectorField = "vector-field" + flagNameDimension = "dimension" + flagNameDistance = "distance-metric" + flagNameIndexMeta = "index-meta" + flagNameTimeout = "timeout" + flagNameVerbose = "verbose" ) type FlagSetBuilder struct { @@ -54,7 +57,7 @@ func (fsb *FlagSetBuilder) AddIndexNameFlag() { } func (fsb *FlagSetBuilder) AddVectorFieldFlag() { - fsb.StringP(flagNameVector, "v", "vector-field", "The name of the vector field.") + fsb.StringP(flagNameVectorField, "f", "", "The name of the vector field.") } @@ -72,22 +75,30 @@ func (fsb *FlagSetBuilder) AddIndexMetaFlag() { fsb.StringToStringP(flagNameIndexMeta, "e", nil, "The metadata for the index.") } +func (fsb *FlagSetBuilder) AddTimeoutFlag() { + fsb.DurationP(flagNameTimeout, "t", time.Second*5, "The timeout used for each request.") +} + +func (fsb *FlagSetBuilder) AddVerbose() { + fsb.BoolP(flagNameVerbose, "v", false, "Display extra detail about an index.") +} + type DistanceMetricFlag string // This is just a set of valid VectorDistanceMetrics. The value does not have meaning var distanceMetricSet = protos.VectorDistanceMetric_value -func (mode *DistanceMetricFlag) Set(val string) error { +func (f *DistanceMetricFlag) Set(val string) error { val = strings.ToUpper(val) - if val, ok := distanceMetricSet[val]; ok { - *mode = DistanceMetricFlag(val) + if _, ok := distanceMetricSet[val]; ok { + *f = DistanceMetricFlag(val) return nil } return fmt.Errorf("unrecognized distance metric") } -func (mode *DistanceMetricFlag) Type() string { +func (f *DistanceMetricFlag) Type() string { names := []string{} for key := range distanceMetricSet { @@ -97,6 +108,39 @@ func (mode *DistanceMetricFlag) Type() string { return strings.Join(names, ",") } -func (mode *DistanceMetricFlag) String() string { - return string(*mode) +func (f *DistanceMetricFlag) String() string { + return string(*f) +} + +type LogLevelFlag string + +var logLevelSet = map[string]struct{}{ + "DEBUG": {}, + "INFO": {}, + "WARN": {}, + "ERROR": {}, +} + +func (f *LogLevelFlag) Set(val string) error { + val = strings.ToUpper(val) + if _, ok := logLevelSet[val]; ok { + *f = LogLevelFlag(val) + return nil + } + + return fmt.Errorf("unrecognized log level") +} + +func (f *LogLevelFlag) Type() string { + names := []string{} + + for key := range logLevelSet { + names = append(names, key) + } + + return strings.Join(names, ",") +} + +func (f *LogLevelFlag) String() string { + return string(*f) } diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 250e02b..38e8bb5 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -1,18 +1,26 @@ /* Copyright © 2024 NAME HERE - */ package cmd import ( - "fmt" + "context" + "log/slog" + avs "github.com/aerospike/aerospike-proximus-client-go" + "github.com/aerospike/aerospike-proximus-client-go/protos" "github.com/spf13/cobra" + "github.com/spf13/viper" ) +// type indexInfo struct { +// Definition *protos.IndexDefinition +// Status *protos.IndexStatusResponse +// } + // listIndexCmd represents the listIndex command var listIndexCmd = &cobra.Command{ - Use: "listIndex", + Use: "index", Short: "A brief description of your command", Long: `A longer description that spans multiple lines and likely contains examples and usage of using your command. For example: @@ -20,21 +28,70 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("listIndex called") + RunE: func(cmd *cobra.Command, args []string) error { + seed := viper.GetString(flagNameSeeds) + port := viper.GetInt(flagNamePort) + hostPort := avs.NewHostPort(seed, port, false) + timeout := viper.GetDuration(flagNameTimeout) + verbose := viper.GetBool(flagNameVerbose) + + logger.Debug("parsed flags", + slog.String("seeds", seed), slog.Int("port", port), slog.Duration("timeout", timeout), + ) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + adminClient, err := avs.NewAdminClient(ctx, avs.HostPortSlice{hostPort}, nil, false, logger) + if err != nil { + logger.Error("failed to create AVS client", slog.Any("error", err)) + return err + } + + cancel() + + ctx, cancel = context.WithTimeout(context.Background(), timeout) + defer cancel() + + indexList, err := adminClient.IndexList(ctx) + if err != nil { + logger.Error("failed to list indexes", slog.Any("error", err)) + return err + } + + indexStatusList := make([]*protos.IndexStatusResponse, len(indexList.GetIndices())) + + if verbose { + cancel() + + ctx, cancel = context.WithTimeout(context.Background(), timeout) + defer cancel() + + for i, index := range indexList.GetIndices() { + indexStatus, err := adminClient.IndexGetStatus(ctx, index.Id.Namespace, index.Id.Name) + if err != nil { + logger.ErrorContext(ctx, "failed to get index status", slog.Any("error", err), slog.String("index", index.Id.String())) + continue + } + + indexStatusList[i] = indexStatus + } + + } + + logger.Debug(indexList.String()) + view.PrintIndexes(indexList, indexStatusList, verbose) + + return nil }, } func init() { listCmd.AddCommand(listIndexCmd) - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // listIndexCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // listIndexCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + flags := NewFlagSetBuilder(listIndexCmd.Flags()) + flags.AddSeedFlag() + flags.AddPortFlag() + flags.AddTimeoutFlag() + flags.AddVerbose() } diff --git a/cmd/root.go b/cmd/root.go index 9cec31f..e2bfb5b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,7 @@ Copyright © 2024 NAME HERE package cmd import ( + "context" "log/slog" "os" @@ -12,7 +13,9 @@ import ( "github.com/spf13/viper" ) -var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})) +var lvl = new(slog.LevelVar) +var logLevelFlagName = "log-level" +var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: lvl})) var view = NewView(os.Stdout) // rootCmd represents the base command when called without any subcommands @@ -25,9 +28,21 @@ examples and usage of using your application. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + cmd.SilenceUsage = true + viper.BindPFlags(cmd.PersistentFlags()) + viper.BindPFlags(cmd.Flags()) + + if viper.IsSet(logLevelFlagName) { + level := viper.GetString(logLevelFlagName) + handler := logger.Handler() + lvl.UnmarshalText([]byte(level)) + + handler.Enabled(context.Background(), lvl.Level()) + } else { + lvl.Set(slog.LevelError + 1) // disable all logging + } + }, } // Execute adds all child commands to the root command and sets flags appropriately. @@ -40,15 +55,8 @@ func Execute() { } func init() { - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.asvec.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - // rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + logLevel := LogLevelFlag("disabled") + rootCmd.PersistentFlags().Var(&logLevel, logLevelFlagName, "Log level for additional details and debugging") common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") viper.SetEnvPrefix("AVS") viper.AutomaticEnv() diff --git a/cmd/view.go b/cmd/view.go index 8ba83c2..f6659da 100644 --- a/cmd/view.go +++ b/cmd/view.go @@ -1,8 +1,12 @@ package cmd import ( + // "asvec/cmd/writers" + "asvec/cmd/writers" "fmt" "io" + + "github.com/aerospike/aerospike-proximus-client-go/protos" ) type View struct { @@ -14,15 +18,22 @@ func NewView(writer io.Writer) *View { } func (v *View) Print(a ...any) { - s := fmt.Sprint(a...) - fmt.Print(s) - v.writer.Write([]byte(fmt.Sprint(a...))) + _, err := v.writer.Write([]byte(fmt.Sprint(a...))) + if err != nil { + panic(err) + } + v.Newline() } func (v *View) Printf(f string, a ...any) { s := fmt.Sprintf(f, a...) - v.writer.Write([]byte(s)) + + _, err := v.writer.Write([]byte(s)) + if err != nil { + panic(err) + } + v.Newline() } @@ -32,3 +43,21 @@ func (v *View) Newline() { panic(err) } } + +func (v *View) getIndexListWriter(verbose bool) *writers.IndexTableWriter { + return writers.NewIndexTableWriter(v.writer, verbose) +} + +func (v *View) PrintIndexes(indexList *protos.IndexDefinitionList, indexStatusList []*protos.IndexStatusResponse, verbose bool) { + t := v.getIndexListWriter(verbose) + + for i, index := range indexList.Indices { + if index.Id.Name == "" || index.Id.Namespace == "" { + continue + } + + t.AppendIndexRow(index, indexStatusList[i]) + } + + t.Render() +} diff --git a/cmd/writers/default.go b/cmd/writers/default.go new file mode 100644 index 0000000..dc92c94 --- /dev/null +++ b/cmd/writers/default.go @@ -0,0 +1,20 @@ +package writers + +import ( + "io" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" +) + +func NewDefaultWriter(writer io.Writer) *table.Writer { + t := table.NewWriter() + + t.SetOutputMirror(writer) + t.AppendSeparator() + t.SuppressEmptyColumns() + t.SetStyle(table.StyleRounded) + t.Style().Title.Align = text.AlignCenter + + return &t +} diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go new file mode 100644 index 0000000..50f6ac6 --- /dev/null +++ b/cmd/writers/indexList.go @@ -0,0 +1,65 @@ +package writers + +import ( + "fmt" + "io" + + "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" +) + +type IndexTableWriter struct { + table.Writer + verbose bool +} + +func NewIndexTableWriter(writer io.Writer, verbose bool) *IndexTableWriter { + removeNil := text.Transformer(func(val interface{}) string { + switch v := val.(type) { + case *string: + if v == nil { + return "" + } + + return *v + default: + return fmt.Sprintf("%v", v) + } + }) + + t := IndexTableWriter{*NewDefaultWriter(writer), verbose} + row := table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric"} + + if verbose { + row = append(row, "Unmerged") + } + + t.AppendHeader(row) + + t.SetTitle("Indexes") + t.SetAutoIndex(true) + t.SortBy([]table.SortBy{ + {Name: "Namespace", Mode: table.Asc}, + {Name: "Set", Mode: table.Asc}, + {Name: "Name", Mode: table.Asc}, + }) + t.SetColumnConfigs([]table.ColumnConfig{ + { + Name: "Set", + Transformer: removeNil, + }, + }) + + return &t +} + +func (itw *IndexTableWriter) AppendIndexRow(index *protos.IndexDefinition, status *protos.IndexStatusResponse) { + row := table.Row{index.Id.Name, index.Id.Namespace, index.SetFilter, index.Field, index.Dimensions, index.VectorDistanceMetric} + + if itw.verbose { + row = append(row, status.GetUnmergedRecordCount()) + } + + itw.AppendRow(row) +} diff --git a/cmd/writers/transformers.go b/cmd/writers/transformers.go new file mode 100644 index 0000000..6dfde5b --- /dev/null +++ b/cmd/writers/transformers.go @@ -0,0 +1,20 @@ +package writers + +import ( + "fmt" + + "github.com/jedib0t/go-pretty/v6/text" +) + +var removeNil = text.Transformer(func(val interface{}) string { + switch v := val.(type) { + case *string: + if v == nil { + return "" + } + + return *v + default: + return fmt.Sprintf("%v", v) + } +}) diff --git a/go.mod b/go.mod index 7ff50a1..61f1de7 100644 --- a/go.mod +++ b/go.mod @@ -2,29 +2,33 @@ module asvec go 1.21.7 -replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go +//replace github.com/aerospike/aerospike-proximus-client-go => github.com/aerospike/aerospike-proximus-client-go VEC-155-admin-client +//replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go require ( - github.com/aerospike/aerospike-proximus-client-go v0.0.0 github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 + github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.0 + github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 ) require ( github.com/aerospike/aerospike-client-go/v7 v7.2.1 // indirect + github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index 04dc9dd..380c48d 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,10 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.2.1 h1:4A6CxgJMRlDnHx4ycyJ1a5lUzxMS7u4byG1XlhCrvSg= github.com/aerospike/aerospike-client-go/v7 v7.2.1/go.mod h1:sKfNsnAKgkGtAlYxdgWNOJm3ykm49s/p6xjEB/cX8/k= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240509004238-aa3172c86d65 h1:0ZhYas/nSdMtx8PV0by6qiajOiPgL9JbTrZTXs1gmgE= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240509004238-aa3172c86d65/go.mod h1:HcQ1MSWCBlL8Sk8jTmXCll2ISoKMEvVBMSDeK5Y70mc= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839 h1:FZZ8WtD4roA8UOJetG7KkMFZYPYJv8Dle4NSM/Emo00= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -39,12 +43,16 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU= +github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -64,6 +72,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= From 9a022cfe45201e9e9ee8d053e447688b380ec78f Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 16:31:18 -0700 Subject: [PATCH 04/35] add this branch to pre-release --- .github/workflows/create-prerelease.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index ff8a995..eef131d 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -1,6 +1,9 @@ name: Build and Create Pre-Release on: + push: + branches: VEC-168-init + workflow_dispatch: inputs: addCommit: From 54b9e398bc58cd5e65af156f5276a433283086db Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 16:34:28 -0700 Subject: [PATCH 05/35] fix src dir --- .github/workflows/create-prerelease.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index eef131d..4c75264 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -49,7 +49,7 @@ jobs: buildcmd="build-prerelease" [ "${ADDCOMMIT}" = "false" ] && buildcmd="build-official" export PATH=$PATH:/usr/local/bin:/usr/local/go/bin - cd ~/work/asvec/asvec/src && make cleanall && make ${buildcmd} + cd ~/work/asvec/asvec && make cleanall && make ${buildcmd} - name: "Create linux packages" env: ADDCOMMIT: ${{ inputs.addCommit }} @@ -57,7 +57,7 @@ jobs: buildcmd="build-prerelease" [ "${ADDCOMMIT}" = "false" ] && buildcmd="build-official" export PATH=$PATH:/usr/local/bin:/usr/local/go/bin - cd ~/work/asvec/asvec/src && make pkg-linux + cd ~/work/asvec/asvec && make pkg-linux - name: "Create windows zips" env: ADDCOMMIT: ${{ inputs.addCommit }} @@ -65,9 +65,9 @@ jobs: buildcmd="build-prerelease" [ "${ADDCOMMIT}" = "false" ] && buildcmd="build-official" export PATH=$PATH:/usr/local/bin:/usr/local/go/bin - cd ~/work/asvec/asvec/src && make pkg-windows-zip + cd ~/work/asvec/asvec && make pkg-windows-zip - name: "Print asvec version" - run: cd ~/work/asvec/asvec/src && ../bin/asvec-macos-amd64 version + run: cd ~/work/asvec/asvec && ../bin/asvec-macos-amd64 version - name: "Prepare keychain for signing MacOS" env: keypw: ${{ secrets.APPLEPW }} @@ -99,7 +99,7 @@ jobs: export asvec_signer="${xasvec_signer}" export asvec_installsigner="${xasvec_installsigner}" export asvec_teamid="${xasvec_teamid}" - export PATH=$PATH:/usr/local/bin:/usr/local/go/bin && cd ~/work/asvec/asvec/src && make macos-build-all && make macos-notarize-all + export PATH=$PATH:/usr/local/bin:/usr/local/go/bin && cd ~/work/asvec/asvec && make macos-build-all && make macos-notarize-all - name: "Create a new pre-release" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From dacc4acfacf9298ff16857165b945717c000af1c Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 16:46:14 -0700 Subject: [PATCH 06/35] fix install --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 62b54a7..61b1657 100644 --- a/Makefile +++ b/Makefile @@ -154,7 +154,7 @@ CPU := $(shell uname -m) ver:=$(shell V=$$(git branch --show-current); if [[ $$V == v* ]]; then printf $${V:1} > ./VERSION.md; fi; cat ./VERSION.md) define _amddebscript ver=$(cat ./VERSION.md) -cat < $(BIN_DIR)/deb/DEBIAN/control +cat < ./bin/deb/DEBIAN/control Website: www.aerospike.com Maintainer: Aerospike Name: Aerospike Vector CLI @@ -168,7 +168,7 @@ endef export amddebscript = $(value _amddebscript) define _armdebscript ver=$(cat ./VERSION.md) -cat < $(BIN_DIR)/deb/DEBIAN/control +cat < ./bin/deb/DEBIAN/control Website: www.aerospike.com Maintainer: Aerospike Name: Aerospike Vector CLI From 2b2928bf55ec17f77cf5552a073dfc0f6fa98fcd Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 17:00:08 -0700 Subject: [PATCH 07/35] fix rpm and deb build --- Makefile | 8 ++++---- bin/asvecrpm/asvec.spec | 6 +++--- bin/asvecrpm/usr/bin/.gitignore | 2 +- bin/macos-pkg/asvec/.gitignore | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 61b1657..e0b535c 100644 --- a/Makefile +++ b/Makefile @@ -320,9 +320,9 @@ pkg-deb-amd64: cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec rm -rf $(BIN_DIR)/deb mkdir -p $(BIN_DIR)/deb/DEBIAN - mkdir -p $(BIN_DIR)/deb/usr/bin + mkdir -p $(BIN_DIR)/deb/usr/local/aerospike/bin @ eval "$$amddebscript" - mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/bin/ + mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/local/aerospike/bin/ sudo dpkg-deb -Zxz -b $(BIN_DIR)/deb rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.deb mv $(BIN_DIR)/deb.deb $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.deb @@ -333,9 +333,9 @@ pkg-deb-arm64: cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec rm -rf $(BIN_DIR)/deb mkdir -p $(BIN_DIR)/deb/DEBIAN - mkdir -p $(BIN_DIR)/deb/usr/bin + mkdir -p $(BIN_DIR)/deb/usr/local/aerospike/bin @ eval "$$armdebscript" - mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/bin/ + mv $(BIN_DIR)/asvec $(BIN_DIR)/deb/usr/local/aerospike/bin/ sudo dpkg-deb -Zxz -b $(BIN_DIR)/deb rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.deb mv $(BIN_DIR)/deb.deb $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.deb diff --git a/bin/asvecrpm/asvec.spec b/bin/asvecrpm/asvec.spec index 2891356..d8abe63 100644 --- a/bin/asvecrpm/asvec.spec +++ b/bin/asvecrpm/asvec.spec @@ -1,9 +1,9 @@ Buildroot: ./ -Name: aerolab +Name: asvec Version: VERSIONHERE Release: 2 Summary: Tool for deploying non-prod Aerospike server clusters on docker or in AWS -License: see github.com/aerospike/aerolab +License: see github.com/aerospike/asvec Group: aerospike %define _rpmdir ./ @@ -17,4 +17,4 @@ Group: aerospike Tool for deploying non-prod Aerospike server clusters on docker or in AWS %files -"/usr/bin/aerolab" +"/usr/local/aerospike/bin/asvec" diff --git a/bin/asvecrpm/usr/bin/.gitignore b/bin/asvecrpm/usr/bin/.gitignore index 29c3f11..0d50953 100644 --- a/bin/asvecrpm/usr/bin/.gitignore +++ b/bin/asvecrpm/usr/bin/.gitignore @@ -1 +1 @@ -aerolab +asvec diff --git a/bin/macos-pkg/asvec/.gitignore b/bin/macos-pkg/asvec/.gitignore index a8e9ffa..9d95e7e 100644 --- a/bin/macos-pkg/asvec/.gitignore +++ b/bin/macos-pkg/asvec/.gitignore @@ -1 +1 @@ -aerolab-* \ No newline at end of file +asvec-* \ No newline at end of file From 3400b447bd58ef2af51dddb12d4d2cb7a3b7f636 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 17:06:28 -0700 Subject: [PATCH 08/35] again --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e0b535c..e15f2af 100644 --- a/Makefile +++ b/Makefile @@ -383,7 +383,7 @@ pkg-rpm-amd64: rm -rf $(BIN_DIR)/asvec-rpm-centos cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos sed -i.bak "s/VERSIONHERE/${ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec - cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec-rpm-centos/usr/bin/asvec + cp $(BIN_DIR)/asvec-linux-amd64 $(BIN_DIR)/asvec-rpm-centos/usr/local/aerospike/bin/asvec rm -f $(BIN_DIR)/asvec-linux-x86_64.rpm bash -ce "cd $(BIN_DIR) && rpmbuild --target=x86_64-redhat-linux --buildroot \$$(pwd)/asvec-rpm-centos -bb asvec-rpm-centos/asvec.spec" rm -f $(BIN_DIR)/packages/asvec-linux-amd64-${ver}.rpm @@ -394,7 +394,7 @@ pkg-rpm-arm64: rm -rf $(BIN_DIR)/asvec-rpm-centos cp -a $(BIN_DIR)/asvecrpm $(BIN_DIR)/asvec-rpm-centos sed -i.bak "s/VERSIONHERE/${ver}/g" $(BIN_DIR)/asvec-rpm-centos/asvec.spec - cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec-rpm-centos/usr/bin/asvec + cp $(BIN_DIR)/asvec-linux-arm64 $(BIN_DIR)/asvec-rpm-centos/usr/local/aerospike/bin/asvec rm -f $(BIN_DIR)/asvec-linux-arm64.rpm bash -ce "cd $(BIN_DIR) && rpmbuild --target=arm64-redhat-linux --buildroot \$$(pwd)/asvec-rpm-centos -bb asvec-rpm-centos/asvec.spec" rm -f $(BIN_DIR)/packages/asvec-linux-arm64-${ver}.rpm From 7992963d26d8cfb2a505618d7ef3db5a8a9ac1b5 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 17:15:56 -0700 Subject: [PATCH 09/35] again --- bin/asvecrpm/usr/{ => local/aerospike}/bin/.gitignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bin/asvecrpm/usr/{ => local/aerospike}/bin/.gitignore (100%) diff --git a/bin/asvecrpm/usr/bin/.gitignore b/bin/asvecrpm/usr/local/aerospike/bin/.gitignore similarity index 100% rename from bin/asvecrpm/usr/bin/.gitignore rename to bin/asvecrpm/usr/local/aerospike/bin/.gitignore From 7b3506c74682e8b5df7111ffa6603452ba8da445 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 17:22:14 -0700 Subject: [PATCH 10/35] again --- .github/workflows/create-prerelease.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index 4c75264..c39ea2d 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -67,7 +67,7 @@ jobs: export PATH=$PATH:/usr/local/bin:/usr/local/go/bin cd ~/work/asvec/asvec && make pkg-windows-zip - name: "Print asvec version" - run: cd ~/work/asvec/asvec && ../bin/asvec-macos-amd64 version + run: cd ~/work/asvec/asvec && ./bin/asvec-macos-amd64 version - name: "Prepare keychain for signing MacOS" env: keypw: ${{ secrets.APPLEPW }} From 43f9bfcb3ce809a5eadce3ba0a34782770f38940 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 29 May 2024 17:29:08 -0700 Subject: [PATCH 11/35] again --- .github/workflows/create-prerelease.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index c39ea2d..1f44220 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -67,7 +67,7 @@ jobs: export PATH=$PATH:/usr/local/bin:/usr/local/go/bin cd ~/work/asvec/asvec && make pkg-windows-zip - name: "Print asvec version" - run: cd ~/work/asvec/asvec && ./bin/asvec-macos-amd64 version + run: cd ~/work/asvec/asvec && ./bin/asvec-macos-amd64 --version - name: "Prepare keychain for signing MacOS" env: keypw: ${{ secrets.APPLEPW }} From 6176bc993b130123fd18cd697f66896332e63e82 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Thu, 30 May 2024 09:00:13 -0700 Subject: [PATCH 12/35] change secrets --- .github/workflows/create-prerelease.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index 1f44220..df14589 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -70,7 +70,7 @@ jobs: run: cd ~/work/asvec/asvec && ./bin/asvec-macos-amd64 --version - name: "Prepare keychain for signing MacOS" env: - keypw: ${{ secrets.APPLEPW }} + keypw: ${{ secrets.APPLEUSERPW }} INSTALLERP12: ${{ secrets.INSTALLERP12 }} APPLICATIONP12: ${{ secrets.APPLICATIONP12 }} run: | @@ -88,7 +88,7 @@ jobs: - name: "Sign and build MacOS" env: xasvec_appleid: ${{ secrets.APPLEUSER }} - xasvec_applepw: ${{ secrets.APPLEPW }} + xasvec_applepw: ${{ secrets.APPLEAPPPW }} xasvec_signer: ${{ secrets.APPLESIGNER }} xasvec_installsigner: ${{ secrets.APPLEINSTALLSIGNER }} xasvec_teamid: ${{ secrets.APPLETEAMID }} From 491fb0720b362deb3cf18d3ac46da1efafccf541 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Thu, 30 May 2024 09:09:27 -0700 Subject: [PATCH 13/35] add release file --- RELEASE.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 RELEASE.md diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..e69de29 From c2472b1e805d9d5e434a9b50c6d0185622850b4d Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Thu, 30 May 2024 15:47:21 -0700 Subject: [PATCH 14/35] add verbose list indexs output --- cmd/create.go | 10 --- cmd/createIndex.go | 131 +++++++++++++++++++++++++++++++++------ cmd/dropIndex.go | 16 ++--- cmd/flags.go | 61 +++++++++++++----- cmd/listIndex.go | 13 ++-- cmd/root.go | 4 +- cmd/view.go | 8 ++- cmd/writers/default.go | 5 +- cmd/writers/indexList.go | 97 ++++++++++++++++++++++------- 9 files changed, 254 insertions(+), 91 deletions(-) diff --git a/cmd/create.go b/cmd/create.go index 5cf7f1b..6f5b6d7 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -21,14 +21,4 @@ to quickly create a Cobra application.`, func init() { rootCmd.AddCommand(createCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // createCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // createCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } diff --git a/cmd/createIndex.go b/cmd/createIndex.go index d639d60..ed104f4 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -5,7 +5,10 @@ package cmd import ( "context" + "fmt" "log/slog" + "strconv" + "strings" avs "github.com/aerospike/aerospike-proximus-client-go" "github.com/aerospike/aerospike-proximus-client-go/protos" @@ -15,14 +18,32 @@ import ( ) const ( - flagNameMaxEdges = "hnsw-max-edges" - flagNameConstructionEf = "hnsw-ef-construction" - flagNameEf = "hnsw-ef" - flagNameBatchMaxRecords = "hnsw-batch-max-records" - flagNameBatchInterval = "hnsw-batch-interval" - flagNameBatchDisabled = "hnsw-batch-disabled" + flagNameStorageNamespace = "storage-namespace" + flagNameStorageSet = "storage-set" + flagNameMaxEdges = "hnsw-max-edges" + flagNameConstructionEf = "hnsw-ef-construction" + flagNameEf = "hnsw-ef" + flagNameBatchMaxRecords = "hnsw-batch-max-records" + flagNameBatchInterval = "hnsw-batch-interval" + flagNameBatchEnabled = "hnsw-batch-disabled" ) +func parseHostPort(rawHost string) (*avs.HostPort, error) { + split := strings.SplitN(rawHost, ":", 2) + host := split[0] + port := 5000 + + if len(split) > 1 { + var err error + port, err = strconv.Atoi(split[1]) + + if err != nil { + return nil, fmt.Errorf("unparsable port: %w", err) + } + } + return avs.NewHostPort(host, port, false), nil +} + // createIndexCmd represents the createIndex command var createIndexCmd = &cobra.Command{ Use: "index", @@ -33,33 +54,76 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, + PreRunE: func(cmd *cobra.Command, args []string) error { + if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { + return fmt.Errorf(fmt.Sprintf("only %s or %s allowed", flagNameSeeds, flagNameHost)) + } + + return nil + }, RunE: func(cmd *cobra.Command, args []string) error { // TODO: likely add to prerun step - seed := viper.GetString(flagNameSeeds) - port := viper.GetInt(flagNamePort) - hostPort := avs.NewHostPort(seed, port, false) + isLoadBalancer := false + hosts := avs.HostPortSlice{} + + if viper.IsSet(flagNameHost) { + isLoadBalancer = true + rawHost := viper.GetString(flagNameHost) + hostPort, err := parseHostPort(rawHost) + + if err != nil { + return err + } + + hosts = append(hosts, hostPort) + } else if viper.IsSet(flagNameSeeds) { + rawSeed := viper.GetStringSlice(flagNameSeeds) + + for _, rawHost := range rawSeed { + hostPort, err := parseHostPort(rawHost) + + if err != nil { + return err + } + + hosts = append(hosts, hostPort) + } + } + namespace := viper.GetString(flagNameNamespace) sets := viper.GetStringSlice(flagNameSets) indexName := viper.GetString(flagNameIndexName) vectorField := viper.GetString(flagNameVectorField) dimension := viper.GetUint32(flagNameDimension) indexMeta := viper.GetStringMapString(flagNameIndexMeta) - distanceMetric := viper.GetString(flagNameDistance) + distanceMetric := viper.GetString(flagNameDistanceMetric) timeout := viper.GetDuration(flagNameTimeout) + storageNamespace := viperGetIfSetString(flagNameStorageNamespace) + storageSet := viperGetIfSetString(flagNameSets) + hnswM := viperGetIfSetUint32(flagNameMaxEdges) + hnswEf := viperGetIfSetUint32(flagNameEf) + hnswConEf := viperGetIfSetUint32(flagNameConstructionEf) + hnswBatchMaxConns := viperGetIfSetUint32(flagNameBatchMaxRecords) + hnswBatchInterval := viperGetIfSetUint32(flagNameBatchInterval) + hnswBatchEnabled := viperGetIfSetBool(flagNameBatchEnabled) + + // TODO, fix seeds logger.Debug("parsed flags", - slog.String("seeds", seed), slog.Int("port", port), slog.String("namespace", namespace), + slog.String("namespace", namespace), slog.Any("sets", sets), slog.String("index-name", indexName), slog.String("vector-field", vectorField), slog.Uint64("dimension", uint64(dimension)), slog.Any("index-meta", indexMeta), slog.String("distance-metric", distanceMetric), - slog.Duration("timeout", timeout), + slog.Duration("timeout", timeout), slog.Any("storage-namespace", storageNamespace), slog.Any("storage-set", storageSet), + slog.Any("hnsw-max-edges", hnswM), slog.Any("hnsw-ef", hnswEf), + slog.Any("hnsw-ef-construction", hnswConEf), slog.Any("hnsw-batch-max-records", hnswBatchMaxConns), + slog.Any("hnsw-batch-interval", hnswBatchInterval), slog.Any("hnsw-batch-enabled", hnswBatchEnabled), ) - ctx := context.Background() - ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - adminClient, err := avs.NewAdminClient(ctx, []*avs.HostPort{hostPort}, nil, false, logger) + // TODO listener name + adminClient, err := avs.NewAdminClient(ctx, hosts, nil, isLoadBalancer, logger) if err != nil { logger.Error("failed to create AVS client", slog.Any("error", err)) return err @@ -71,8 +135,31 @@ to quickly create a Cobra application.`, ctx, cancel = context.WithTimeout(context.Background(), timeout) defer cancel() - // TODO: Add storage type - err = adminClient.IndexCreate(ctx, namespace, sets, indexName, vectorField, dimension, protos.VectorDistanceMetric(protos.VectorDistanceMetric_value[distanceMetric]), nil, indexMeta) + // Inverted to make it easier to understand + var hnswBatchDisabled *bool + if hnswBatchEnabled != nil { + *hnswBatchDisabled = !*hnswBatchEnabled + } + + indexStorage := &protos.IndexStorage{ + Namespace: storageNamespace, + Set: storageSet, + } + hnswParams := &protos.HnswParams{ + M: hnswM, + Ef: hnswEf, + EfConstruction: hnswConEf, + BatchingParams: &protos.HnswBatchingParams{ + MaxRecords: hnswBatchMaxConns, + Interval: hnswBatchInterval, + Disabled: hnswBatchDisabled, + }, + } + + err = adminClient.IndexCreate( + ctx, namespace, sets, indexName, vectorField, dimension, + protos.VectorDistanceMetric(protos.VectorDistanceMetric_value[distanceMetric]), + hnswParams, indexMeta, indexStorage) if err != nil { logger.Error("unable to create index", slog.Any("error", err)) return err @@ -90,7 +177,7 @@ func init() { flags := NewFlagSetBuilder(createIndexCmd.Flags()) persistentFlags.AddSeedFlag() - persistentFlags.AddPortFlag() + persistentFlags.AddHostFlag() flags.AddNamespaceFlag() flags.AddSetsFlag() @@ -106,19 +193,23 @@ func init() { flagNameIndexName, flagNameVectorField, flagNameDimension, - flagNameDistance, + flagNameDistanceMetric, } // TODO: Add custom template for usage to take into account terminal width // Ex: https://github.com/sigstore/cosign/pull/3011/files + flags.String(flagNameStorageNamespace, "", commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) + flags.String(flagNameStorageSet, "", commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) flags.Uint32(flagNameMaxEdges, 0, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) flags.Uint32(flagNameConstructionEf, 0, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) flags.Uint32(flagNameEf, 0, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) flags.Uint32(flagNameBatchMaxRecords, 0, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) flags.Uint32(flagNameBatchInterval, 0, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) - flags.Bool(flagNameBatchDisabled, false, commonFlags.DefaultWrapHelpString("Disables batching for index updates. Default is false meaning batching is enabled.")) + flags.Bool(flagNameBatchEnabled, true, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) for _, flag := range requiredFlags { createIndexCmd.MarkFlagRequired(flag) } + + createIndexCmd.MarkFlagsMutuallyExclusive(flagNameHost, flagNameSeeds) } diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index f3da4ad..c332b1f 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -16,7 +16,7 @@ var dropIndexRequiredFlags = []string{ flagNameNamespace, flagNameIndexName, flagNameDimension, - flagNameDistance, + flagNameDistanceMetric, } // dropIndexCmd represents the dropIndex command @@ -32,16 +32,16 @@ to quickly create a Cobra application.`, RunE: func(cmd *cobra.Command, args []string) error { // TODO: likely add to prerun step seed := viper.GetString(flagNameSeeds) - port := viper.GetInt(flagNamePort) - hostPort := avs.NewHostPort(seed, port, false) + // port := viper.GetInt(flagNamePort) + hostPort := avs.NewHostPort(seed, 5002, false) namespace := viper.GetString(flagNameNamespace) indexName := viper.GetString(flagNameIndexName) timeout := viper.GetDuration(flagNameTimeout) - logger.Debug("parsed flags", - slog.String("seeds", seed), slog.Int("port", port), slog.String("namespace", namespace), - slog.String("index-name", indexName), slog.Duration("timeout", timeout), - ) + // logger.Debug("parsed flags", + // slog.String("seeds", seed), slog.Int("port", port), slog.String("namespace", namespace), + // slog.String("index-name", indexName), slog.Duration("timeout", timeout), + // ) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -74,7 +74,7 @@ func init() { flags := NewFlagSetBuilder(dropIndexCmd.Flags()) flags.AddSeedFlag() - flags.AddPortFlag() + // flags.AddPortFlag() flags.AddNamespaceFlag() flags.AddIndexNameFlag() flags.AddTimeoutFlag() diff --git a/cmd/flags.go b/cmd/flags.go index 47ea14e..790f70d 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -2,28 +2,57 @@ package cmd import ( "fmt" - "net" "strings" "time" "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/aerospike/tools-common-go/flags" "github.com/spf13/pflag" + "github.com/spf13/viper" ) const ( - flagNameSeeds = "seeds" - flagNamePort = "port" - flagNameNamespace = "namespace" - flagNameSets = "sets" - flagNameIndexName = "index-name" - flagNameVectorField = "vector-field" - flagNameDimension = "dimension" - flagNameDistance = "distance-metric" - flagNameIndexMeta = "index-meta" - flagNameTimeout = "timeout" - flagNameVerbose = "verbose" + flagNameSeeds = "seeds" + flagNameHost = "host" + flagNameListenerName = "listener-name" + flagNameNamespace = "namespace" + flagNameSets = "sets" + flagNameIndexName = "index-name" + flagNameVectorField = "vector-field" + flagNameDimension = "dimension" + flagNameDistanceMetric = "distance-metric" + flagNameIndexMeta = "index-meta" + flagNameTimeout = "timeout" + flagNameVerbose = "verbose" ) +func viperGetIfSetString(flagName string) *string { + if viper.IsSet(flagName) { + s := viper.GetString(flagName) + return &s + } + + return nil +} + +func viperGetIfSetBool(flagName string) *bool { + if viper.IsSet(flagName) { + s := viper.GetBool(flagName) + return &s + } + + return nil +} + +func viperGetIfSetUint32(flagName string) *uint32 { + if viper.IsSet(flagName) { + s := viper.GetUint32(flagName) + return &s + } + + return nil +} + type FlagSetBuilder struct { *pflag.FlagSet } @@ -36,11 +65,11 @@ func NewFlagSetBuilder(flagSet *pflag.FlagSet) *FlagSetBuilder { // TODO: Should this be a list of IPs? Should we support IP:PORT? func (fsb *FlagSetBuilder) AddSeedFlag() { - fsb.IPP(flagNameSeeds, "h", net.ParseIP("127.0.0.1"), "The AVS seed host for cluster discovery.") + fsb.StringArrayP(flagNameSeeds, "s", []string{}, flags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) } -func (fsb *FlagSetBuilder) AddPortFlag() { - fsb.IntP(flagNamePort, "p", 5000, "The AVS seed port for cluster discovery.") +func (fsb *FlagSetBuilder) AddHostFlag() { + fsb.StringP(flagNameHost, "h", "127.0.0.1:5000", fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds)) } func (fsb *FlagSetBuilder) AddNamespaceFlag() { @@ -68,7 +97,7 @@ func (fsb *FlagSetBuilder) AddDimensionFlag() { func (fsb *FlagSetBuilder) AddDistanceMetricFlag() { distMetric := DistanceMetricFlag("") - fsb.VarP(&distMetric, "distance-metric", "m", "The distance metric for the index.") + fsb.VarP(&distMetric, flagNameDistanceMetric, "m", "The distance metric for the index.") } func (fsb *FlagSetBuilder) AddIndexMetaFlag() { diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 38e8bb5..6242b98 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -30,14 +30,14 @@ This application is a tool to generate the needed files to quickly create a Cobra application.`, RunE: func(cmd *cobra.Command, args []string) error { seed := viper.GetString(flagNameSeeds) - port := viper.GetInt(flagNamePort) - hostPort := avs.NewHostPort(seed, port, false) + // port := viper.GetInt(flagNamePort) + hostPort := avs.NewHostPort(seed, 5002, false) timeout := viper.GetDuration(flagNameTimeout) verbose := viper.GetBool(flagNameVerbose) - logger.Debug("parsed flags", - slog.String("seeds", seed), slog.Int("port", port), slog.Duration("timeout", timeout), - ) + // logger.Debug("parsed flags", + // slog.String("seeds", seed), slog.Int("port", port), slog.Duration("timeout", timeout), + // ) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -49,6 +49,7 @@ to quickly create a Cobra application.`, } cancel() + defer adminClient.Close() ctx, cancel = context.WithTimeout(context.Background(), timeout) defer cancel() @@ -91,7 +92,7 @@ func init() { flags := NewFlagSetBuilder(listIndexCmd.Flags()) flags.AddSeedFlag() - flags.AddPortFlag() + // flags.AddPortFlag() flags.AddTimeoutFlag() flags.AddVerbose() } diff --git a/cmd/root.go b/cmd/root.go index e2bfb5b..ea2eee9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( var lvl = new(slog.LevelVar) var logLevelFlagName = "log-level" var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: lvl})) -var view = NewView(os.Stdout) +var view = NewView(os.Stdout, logger) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ @@ -58,6 +58,6 @@ func init() { logLevel := LogLevelFlag("disabled") rootCmd.PersistentFlags().Var(&logLevel, logLevelFlagName, "Log level for additional details and debugging") common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") - viper.SetEnvPrefix("AVS") + viper.SetEnvPrefix("ASVEC") viper.AutomaticEnv() } diff --git a/cmd/view.go b/cmd/view.go index f6659da..355fe07 100644 --- a/cmd/view.go +++ b/cmd/view.go @@ -5,16 +5,18 @@ import ( "asvec/cmd/writers" "fmt" "io" + "log/slog" "github.com/aerospike/aerospike-proximus-client-go/protos" ) type View struct { writer io.Writer + logger *slog.Logger } -func NewView(writer io.Writer) *View { - return &View{writer: writer} +func NewView(writer io.Writer, logger *slog.Logger) *View { + return &View{writer: writer, logger: logger} } func (v *View) Print(a ...any) { @@ -45,7 +47,7 @@ func (v *View) Newline() { } func (v *View) getIndexListWriter(verbose bool) *writers.IndexTableWriter { - return writers.NewIndexTableWriter(v.writer, verbose) + return writers.NewIndexTableWriter(v.writer, verbose, v.logger) } func (v *View) PrintIndexes(indexList *protos.IndexDefinitionList, indexStatusList []*protos.IndexStatusResponse, verbose bool) { diff --git a/cmd/writers/default.go b/cmd/writers/default.go index dc92c94..6c4ffeb 100644 --- a/cmd/writers/default.go +++ b/cmd/writers/default.go @@ -7,7 +7,7 @@ import ( "github.com/jedib0t/go-pretty/v6/text" ) -func NewDefaultWriter(writer io.Writer) *table.Writer { +func NewDefaultWriter(writer io.Writer) table.Writer { t := table.NewWriter() t.SetOutputMirror(writer) @@ -15,6 +15,7 @@ func NewDefaultWriter(writer io.Writer) *table.Writer { t.SuppressEmptyColumns() t.SetStyle(table.StyleRounded) t.Style().Title.Align = text.AlignCenter + t.Style().Title.Colors = append(t.Style().Title.Colors, text.Bold) - return &t + return t } diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go index 50f6ac6..4a963ed 100644 --- a/cmd/writers/indexList.go +++ b/cmd/writers/indexList.go @@ -1,64 +1,113 @@ package writers import ( - "fmt" "io" + "log/slog" "github.com/aerospike/aerospike-proximus-client-go/protos" "github.com/jedib0t/go-pretty/v6/table" - "github.com/jedib0t/go-pretty/v6/text" ) +var rowConfigAutoMerge = table.RowConfig{AutoMerge: true} + type IndexTableWriter struct { table.Writer verbose bool + logger *slog.Logger } -func NewIndexTableWriter(writer io.Writer, verbose bool) *IndexTableWriter { - removeNil := text.Transformer(func(val interface{}) string { - switch v := val.(type) { - case *string: - if v == nil { - return "" - } - - return *v - default: - return fmt.Sprintf("%v", v) - } - }) - - t := IndexTableWriter{*NewDefaultWriter(writer), verbose} - row := table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric"} +func NewIndexTableWriter(writer io.Writer, verbose bool, logger *slog.Logger) *IndexTableWriter { + t := IndexTableWriter{NewDefaultWriter(writer), verbose, logger} if verbose { - row = append(row, "Unmerged") + t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Storage", "Index Parameters"}, rowConfigAutoMerge) + // t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Storage", "Storage", "HNSW", "HNSW", "HNSW", "HNSW", "HNSW", "HNSW"}, rowConfigAutoMerge) + // t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Namespace", "Set", "Max Edges", "Ef", "Construction Ef", "Batch", "Batch", "Batch"}, rowConfigAutoMerge) + // t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Namespace", "Set", "Max Edges", "Ef", "Construction Ef", "Max Records", "Interval", "Disabled"}) + } else { + t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged"}) } - t.AppendHeader(row) - t.SetTitle("Indexes") + t.Style().Options.SeparateRows = true t.SetAutoIndex(true) t.SortBy([]table.SortBy{ {Name: "Namespace", Mode: table.Asc}, {Name: "Set", Mode: table.Asc}, {Name: "Name", Mode: table.Asc}, }) + t.SetColumnConfigs([]table.ColumnConfig{ { - Name: "Set", + + Number: 3, + // Name: "Set", Transformer: removeNil, + // AutoMerge: true, }, + // { + // Number: 8, + // Name: "Namespace", + // AutoMerge: true, + // }, + // { + // Number: 9, + // Name: "Set", + // AutoMerge: true, + // }, + + // {Number: 1, AutoMerge: true}, + // {Number: 2, AutoMerge: true}, + // {Number: 3, AutoMerge: true}, + // {Number: 4, AutoMerge: true}, + // {Number: 5, AutoMerge: true}, + // {Number: 6, AutoMerge: true}, + // {Number: 7, AutoMerge: true}, + // {Number: 8, AutoMerge: true}, + // {Number: 9, AutoMerge: true}, + // {Number: 10, AutoMerge: true}, + // {Number: 11, AutoMerge: true}, + // {Number: 12, AutoMerge: true}, + // {Number: 13, AutoMerge: true}, + // {Number: 14, AutoMerge: true}, + // {Number: 15, AutoMerge: true}, }) return &t } func (itw *IndexTableWriter) AppendIndexRow(index *protos.IndexDefinition, status *protos.IndexStatusResponse) { - row := table.Row{index.Id.Name, index.Id.Namespace, index.SetFilter, index.Field, index.Dimensions, index.VectorDistanceMetric} + row := table.Row{index.Id.Name, index.Id.Namespace, index.SetFilter, index.Field, + index.Dimensions, index.VectorDistanceMetric, status.GetUnmergedRecordCount()} if itw.verbose { - row = append(row, status.GetUnmergedRecordCount()) + tStorage := NewDefaultWriter(nil) + + tStorage.AppendRow(table.Row{"Namespace", index.Storage.GetNamespace()}) + tStorage.AppendRow(table.Row{"Set", index.Storage.GetSet()}) + + row = append(row, tStorage.Render()) + // row = append(row, status.GetUnmergedRecordCount(), index.Storage.GetNamespace(), index.Storage.GetSet()) + + switch v := index.Params.(type) { + case *protos.IndexDefinition_HnswParams: + tHNSW := NewDefaultWriter(nil) + tHNSW.SetTitle("HNSW") + tHNSW.AppendRows([]table.Row{ + {"Max Edges", v.HnswParams.GetM()}, + {"Ef", v.HnswParams.GetEf()}, + {"Construction Ef", v.HnswParams.GetEfConstruction()}, + {"Batch Max Records", v.HnswParams.BatchingParams.GetMaxRecords()}, + {"Batch Interval", v.HnswParams.BatchingParams.GetInterval()}, + {"Batch Disabled", v.HnswParams.BatchingParams.GetDisabled()}, + }) + // row = append(row, v.HnswParams.GetM(), v.HnswParams.GetEf(), v.HnswParams.GetEfConstruction(), + // v.HnswParams.BatchingParams.GetMaxRecords(), v.HnswParams.BatchingParams.GetInterval(), + // v.HnswParams.BatchingParams.GetDisabled()) + row = append(row, tHNSW.Render()) + default: + itw.logger.Warn("the server returned unrecognized index type params. recognized index param types are: HNSW") + } } itw.AppendRow(row) From 5c7a3bd4910102c9dc6fb326523596eec1632c75 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Fri, 31 May 2024 11:51:22 -0700 Subject: [PATCH 15/35] fix and remove unused code --- cmd/flags.go | 2 +- cmd/writers/indexList.go | 39 ++------------------------------------- 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 790f70d..8410455 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -65,7 +65,7 @@ func NewFlagSetBuilder(flagSet *pflag.FlagSet) *FlagSetBuilder { // TODO: Should this be a list of IPs? Should we support IP:PORT? func (fsb *FlagSetBuilder) AddSeedFlag() { - fsb.StringArrayP(flagNameSeeds, "s", []string{}, flags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) + fsb.StringArray(flagNameSeeds, []string{}, flags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) } func (fsb *FlagSetBuilder) AddHostFlag() { diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go index 4a963ed..60e7e0b 100644 --- a/cmd/writers/indexList.go +++ b/cmd/writers/indexList.go @@ -21,9 +21,6 @@ func NewIndexTableWriter(writer io.Writer, verbose bool, logger *slog.Logger) *I if verbose { t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Storage", "Index Parameters"}, rowConfigAutoMerge) - // t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Storage", "Storage", "HNSW", "HNSW", "HNSW", "HNSW", "HNSW", "HNSW"}, rowConfigAutoMerge) - // t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Namespace", "Set", "Max Edges", "Ef", "Construction Ef", "Batch", "Batch", "Batch"}, rowConfigAutoMerge) - // t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Namespace", "Set", "Max Edges", "Ef", "Construction Ef", "Max Records", "Interval", "Disabled"}) } else { t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged"}) } @@ -40,37 +37,9 @@ func NewIndexTableWriter(writer io.Writer, verbose bool, logger *slog.Logger) *I t.SetColumnConfigs([]table.ColumnConfig{ { - Number: 3, - // Name: "Set", + Number: 3, Transformer: removeNil, - // AutoMerge: true, }, - // { - // Number: 8, - // Name: "Namespace", - // AutoMerge: true, - // }, - // { - // Number: 9, - // Name: "Set", - // AutoMerge: true, - // }, - - // {Number: 1, AutoMerge: true}, - // {Number: 2, AutoMerge: true}, - // {Number: 3, AutoMerge: true}, - // {Number: 4, AutoMerge: true}, - // {Number: 5, AutoMerge: true}, - // {Number: 6, AutoMerge: true}, - // {Number: 7, AutoMerge: true}, - // {Number: 8, AutoMerge: true}, - // {Number: 9, AutoMerge: true}, - // {Number: 10, AutoMerge: true}, - // {Number: 11, AutoMerge: true}, - // {Number: 12, AutoMerge: true}, - // {Number: 13, AutoMerge: true}, - // {Number: 14, AutoMerge: true}, - // {Number: 15, AutoMerge: true}, }) return &t @@ -87,7 +56,6 @@ func (itw *IndexTableWriter) AppendIndexRow(index *protos.IndexDefinition, statu tStorage.AppendRow(table.Row{"Set", index.Storage.GetSet()}) row = append(row, tStorage.Render()) - // row = append(row, status.GetUnmergedRecordCount(), index.Storage.GetNamespace(), index.Storage.GetSet()) switch v := index.Params.(type) { case *protos.IndexDefinition_HnswParams: @@ -99,11 +67,8 @@ func (itw *IndexTableWriter) AppendIndexRow(index *protos.IndexDefinition, statu {"Construction Ef", v.HnswParams.GetEfConstruction()}, {"Batch Max Records", v.HnswParams.BatchingParams.GetMaxRecords()}, {"Batch Interval", v.HnswParams.BatchingParams.GetInterval()}, - {"Batch Disabled", v.HnswParams.BatchingParams.GetDisabled()}, + {"Batch Enabled", !v.HnswParams.BatchingParams.GetDisabled()}, }) - // row = append(row, v.HnswParams.GetM(), v.HnswParams.GetEf(), v.HnswParams.GetEfConstruction(), - // v.HnswParams.BatchingParams.GetMaxRecords(), v.HnswParams.BatchingParams.GetInterval(), - // v.HnswParams.BatchingParams.GetDisabled()) row = append(row, tHNSW.Render()) default: itw.logger.Warn("the server returned unrecognized index type params. recognized index param types are: HNSW") From 2eba0d789585597504d1f14bd7d764d103ff1720 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Fri, 31 May 2024 11:56:52 -0700 Subject: [PATCH 16/35] update client commit --- go.mod | 4 ++-- go.sum | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 61f1de7..ad47495 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module asvec go 1.21.7 //replace github.com/aerospike/aerospike-proximus-client-go => github.com/aerospike/aerospike-proximus-client-go VEC-155-admin-client -//replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go +// replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go require ( github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 @@ -15,7 +15,7 @@ require ( require ( github.com/aerospike/aerospike-client-go/v7 v7.2.1 // indirect - github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839 // indirect + github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240530224737-115c3007bafa // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 380c48d..a76251d 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240509004238-aa3172c8 github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240509004238-aa3172c86d65/go.mod h1:HcQ1MSWCBlL8Sk8jTmXCll2ISoKMEvVBMSDeK5Y70mc= github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839 h1:FZZ8WtD4roA8UOJetG7KkMFZYPYJv8Dle4NSM/Emo00= github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240530224737-115c3007bafa h1:+DUvX+HJvY4ZWEVXjlN14iwFBwPP8br0mDL8XPUO3E0= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240530224737-115c3007bafa/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= From cb01cf4bc04352d470e979cdeb75991f2c391fc0 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Mon, 3 Jun 2024 16:07:12 -0700 Subject: [PATCH 17/35] cleanup flags, add seeds and host flag --- .gitignore | 4 +- cmd/createIndex.go | 330 ++++++++++++++++++++-------------------- cmd/dropIndex.go | 142 ++++++++++------- cmd/flags.go | 372 +++++++++++++++++++++++++++++++++------------ cmd/listIndex.go | 154 ++++++++++++------- cmd/root.go | 50 ++++-- go.mod | 4 +- go.sum | 2 + 8 files changed, 667 insertions(+), 391 deletions(-) diff --git a/.gitignore b/.gitignore index 92c8db2..2655a7e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -./docker/* \ No newline at end of file +/docker/* +/bin/* +embed_*.go \ No newline at end of file diff --git a/cmd/createIndex.go b/cmd/createIndex.go index ed104f4..244021c 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -7,209 +7,203 @@ import ( "context" "fmt" "log/slog" - "strconv" - "strings" + "time" avs "github.com/aerospike/aerospike-proximus-client-go" "github.com/aerospike/aerospike-proximus-client-go/protos" commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/spf13/viper" ) -const ( - flagNameStorageNamespace = "storage-namespace" - flagNameStorageSet = "storage-set" - flagNameMaxEdges = "hnsw-max-edges" - flagNameConstructionEf = "hnsw-ef-construction" - flagNameEf = "hnsw-ef" - flagNameBatchMaxRecords = "hnsw-batch-max-records" - flagNameBatchInterval = "hnsw-batch-interval" - flagNameBatchEnabled = "hnsw-batch-disabled" -) +const () + +type ciFlags struct { + host *HostPortFlag + seeds *SeedsSliceFlag + listenerName StringOptionalFlag + namespace string + sets []string + indexName string + vectorField string + dimensions uint32 + distanceMetric DistanceMetricFlag + indexMeta map[string]string + storageNamespace StringOptionalFlag + storageSet StringOptionalFlag + hnswMaxEdges Uint32OptionalFlag + hnswEf Uint32OptionalFlag + hnswConstructionEf Uint32OptionalFlag + hnswBatchMaxRecords Uint32OptionalFlag + hnswBatchInterval Uint32OptionalFlag + hnswBatchEnabled BoolOptionalFlag + timeout time.Duration +} -func parseHostPort(rawHost string) (*avs.HostPort, error) { - split := strings.SplitN(rawHost, ":", 2) - host := split[0] - port := 5000 +var createIndexFlags = &ciFlags{ + host: NewDefaultHostPortFlag(), + seeds: &SeedsSliceFlag{}, + storageNamespace: StringOptionalFlag{}, + storageSet: StringOptionalFlag{}, + hnswMaxEdges: Uint32OptionalFlag{}, + hnswEf: Uint32OptionalFlag{}, + hnswConstructionEf: Uint32OptionalFlag{}, + hnswBatchMaxRecords: Uint32OptionalFlag{}, + hnswBatchInterval: Uint32OptionalFlag{}, + hnswBatchEnabled: BoolOptionalFlag{}, +} - if len(split) > 1 { - var err error - port, err = strconv.Atoi(split[1]) +func newCreateIndexFlagSet() *pflag.FlagSet { + flagSet := &pflag.FlagSet{} + flagSet.VarP(createIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) + flagSet.Var(createIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) + flagSet.VarP(&createIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) + flagSet.StringVarP(&createIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) + flagSet.StringArrayVarP(&createIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) + flagSet.StringVarP(&createIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) + flagSet.StringVarP(&createIndexFlags.vectorField, flagNameVectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) + flagSet.Uint32VarP(&createIndexFlags.dimensions, flagNameDimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) + flagSet.VarP(&createIndexFlags.distanceMetric, flagNameDistanceMetric, "m", commonFlags.DefaultWrapHelpString("The distance metric for the index.")) + flagSet.StringToStringVar(&createIndexFlags.indexMeta, flagNameIndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) + flagSet.DurationVar(&createIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) + flagSet.Var(&createIndexFlags.storageNamespace, flagNameStorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) + flagSet.Var(&createIndexFlags.storageSet, flagNameStorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) + flagSet.Var(&createIndexFlags.hnswMaxEdges, flagNameMaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) + flagSet.Var(&createIndexFlags.hnswConstructionEf, flagNameConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) + flagSet.Var(&createIndexFlags.hnswEf, flagNameEf, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) + flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flagNameBatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) + flagSet.Var(&createIndexFlags.hnswBatchInterval, flagNameBatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) + flagSet.Var(&createIndexFlags.hnswBatchEnabled, flagNameBatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) + + return flagSet +} - if err != nil { - return nil, fmt.Errorf("unparsable port: %w", err) - } - } - return avs.NewHostPort(host, port, false), nil +var createIndexRequiredFlags = []string{ + flagNameNamespace, + flagNameIndexName, + flagNameVectorField, + flagNameDimension, + flagNameDistanceMetric, } // createIndexCmd represents the createIndex command -var createIndexCmd = &cobra.Command{ - Use: "index", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples +func newCreateIndexCmd() *cobra.Command { + return &cobra.Command{ + Use: "index", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - PreRunE: func(cmd *cobra.Command, args []string) error { - if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf(fmt.Sprintf("only %s or %s allowed", flagNameSeeds, flagNameHost)) - } - - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - // TODO: likely add to prerun step - isLoadBalancer := false - hosts := avs.HostPortSlice{} - - if viper.IsSet(flagNameHost) { - isLoadBalancer = true - rawHost := viper.GetString(flagNameHost) - hostPort, err := parseHostPort(rawHost) + PreRunE: func(cmd *cobra.Command, args []string) error { + if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { + return fmt.Errorf(fmt.Sprintf("only --%s or --%s allowed", flagNameSeeds, flagNameHost)) + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + hosts, isLoadBalancer := parseBothHostSeedsFlag(*createIndexFlags.seeds, *createIndexFlags.host) + + logger.Debug("parsed flags", + slog.String(flagNameHost, createIndexFlags.host.String()), + slog.String(flagNameSeeds, createIndexFlags.seeds.String()), + slog.String(flagNameListenerName, createIndexFlags.listenerName.String()), + slog.String(flagNameNamespace, createIndexFlags.namespace), + slog.Any(flagNameSets, createIndexFlags.sets), + slog.String(flagNameIndexName, createIndexFlags.indexName), + slog.String(flagNameVectorField, createIndexFlags.vectorField), + slog.Uint64(flagNameDimension, uint64(createIndexFlags.dimensions)), + slog.Any(flagNameIndexMeta, createIndexFlags.indexMeta), + slog.String(flagNameDistanceMetric, createIndexFlags.distanceMetric.String()), + slog.Duration(flagNameTimeout, createIndexFlags.timeout), + slog.Any(flagNameStorageNamespace, createIndexFlags.storageNamespace.String()), + slog.Any(flagNameStorageSet, createIndexFlags.storageSet.String()), + slog.Any(flagNameMaxEdges, createIndexFlags.hnswMaxEdges.String()), + slog.Any(flagNameEf, createIndexFlags.hnswEf), + slog.Any(flagNameConstructionEf, createIndexFlags.hnswConstructionEf.String()), + slog.Any(flagNameBatchMaxRecords, createIndexFlags.hnswBatchMaxRecords.String()), + slog.Any(flagNameBatchInterval, createIndexFlags.hnswBatchInterval.String()), + slog.Any(flagNameBatchEnabled, createIndexFlags.hnswBatchEnabled.String()), + ) + + ctx, cancel := context.WithTimeout(context.Background(), createIndexFlags.timeout) + defer cancel() + + adminClient, err := avs.NewAdminClient( + ctx, hosts, createIndexFlags.listenerName.Val, isLoadBalancer, logger, + ) if err != nil { + logger.Error("failed to create AVS client", slog.Any("error", err)) return err } - hosts = append(hosts, hostPort) - } else if viper.IsSet(flagNameSeeds) { - rawSeed := viper.GetStringSlice(flagNameSeeds) + cancel() + defer adminClient.Close() + + ctx, cancel = context.WithTimeout(context.Background(), createIndexFlags.timeout) + defer cancel() - for _, rawHost := range rawSeed { - hostPort, err := parseHostPort(rawHost) + // Inverted to make it easier to understand + var hnswBatchDisabled *bool + if createIndexFlags.hnswBatchEnabled.Val != nil { + bd := !(*createIndexFlags.hnswBatchEnabled.Val) + hnswBatchDisabled = &bd + } + + indexStorage := &protos.IndexStorage{ + Namespace: createIndexFlags.storageNamespace.Val, + Set: createIndexFlags.storageSet.Val, + } - if err != nil { - return err - } + hnswParams := &protos.HnswParams{ + M: createIndexFlags.hnswMaxEdges.Val, + Ef: createIndexFlags.hnswEf.Val, + EfConstruction: createIndexFlags.hnswConstructionEf.Val, + BatchingParams: &protos.HnswBatchingParams{ + MaxRecords: createIndexFlags.hnswBatchMaxRecords.Val, + Interval: createIndexFlags.hnswBatchInterval.Val, + Disabled: hnswBatchDisabled, + }, + } - hosts = append(hosts, hostPort) + err = adminClient.IndexCreate( + ctx, + createIndexFlags.namespace, + createIndexFlags.sets, + createIndexFlags.indexName, + createIndexFlags.vectorField, + createIndexFlags.dimensions, + protos.VectorDistanceMetric(protos.VectorDistanceMetric_value[createIndexFlags.distanceMetric.String()]), + hnswParams, + createIndexFlags.indexMeta, + indexStorage, + ) + if err != nil { + logger.Error("unable to create index", slog.Any("error", err)) + return err } - } - - namespace := viper.GetString(flagNameNamespace) - sets := viper.GetStringSlice(flagNameSets) - indexName := viper.GetString(flagNameIndexName) - vectorField := viper.GetString(flagNameVectorField) - dimension := viper.GetUint32(flagNameDimension) - indexMeta := viper.GetStringMapString(flagNameIndexMeta) - distanceMetric := viper.GetString(flagNameDistanceMetric) - timeout := viper.GetDuration(flagNameTimeout) - - storageNamespace := viperGetIfSetString(flagNameStorageNamespace) - storageSet := viperGetIfSetString(flagNameSets) - hnswM := viperGetIfSetUint32(flagNameMaxEdges) - hnswEf := viperGetIfSetUint32(flagNameEf) - hnswConEf := viperGetIfSetUint32(flagNameConstructionEf) - hnswBatchMaxConns := viperGetIfSetUint32(flagNameBatchMaxRecords) - hnswBatchInterval := viperGetIfSetUint32(flagNameBatchInterval) - hnswBatchEnabled := viperGetIfSetBool(flagNameBatchEnabled) - - // TODO, fix seeds - logger.Debug("parsed flags", - slog.String("namespace", namespace), - slog.Any("sets", sets), slog.String("index-name", indexName), slog.String("vector-field", vectorField), - slog.Uint64("dimension", uint64(dimension)), slog.Any("index-meta", indexMeta), slog.String("distance-metric", distanceMetric), - slog.Duration("timeout", timeout), slog.Any("storage-namespace", storageNamespace), slog.Any("storage-set", storageSet), - slog.Any("hnsw-max-edges", hnswM), slog.Any("hnsw-ef", hnswEf), - slog.Any("hnsw-ef-construction", hnswConEf), slog.Any("hnsw-batch-max-records", hnswBatchMaxConns), - slog.Any("hnsw-batch-interval", hnswBatchInterval), slog.Any("hnsw-batch-enabled", hnswBatchEnabled), - ) - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - // TODO listener name - adminClient, err := avs.NewAdminClient(ctx, hosts, nil, isLoadBalancer, logger) - if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) - return err - } - - cancel() - defer adminClient.Close() - - ctx, cancel = context.WithTimeout(context.Background(), timeout) - defer cancel() - - // Inverted to make it easier to understand - var hnswBatchDisabled *bool - if hnswBatchEnabled != nil { - *hnswBatchDisabled = !*hnswBatchEnabled - } - - indexStorage := &protos.IndexStorage{ - Namespace: storageNamespace, - Set: storageSet, - } - hnswParams := &protos.HnswParams{ - M: hnswM, - Ef: hnswEf, - EfConstruction: hnswConEf, - BatchingParams: &protos.HnswBatchingParams{ - MaxRecords: hnswBatchMaxConns, - Interval: hnswBatchInterval, - Disabled: hnswBatchDisabled, - }, - } - - err = adminClient.IndexCreate( - ctx, namespace, sets, indexName, vectorField, dimension, - protos.VectorDistanceMetric(protos.VectorDistanceMetric_value[distanceMetric]), - hnswParams, indexMeta, indexStorage) - if err != nil { - logger.Error("unable to create index", slog.Any("error", err)) - return err - } - - view.Printf("Successfully created index %s.%s", namespace, indexName) - return nil - }, + + view.Printf("Successfully created index %s.%s", createIndexFlags.namespace, createIndexFlags.indexName) + return nil + }, + } } func init() { + createIndexCmd := newCreateIndexCmd() createCmd.AddCommand(createIndexCmd) - persistentFlags := NewFlagSetBuilder(createIndexCmd.PersistentFlags()) - flags := NewFlagSetBuilder(createIndexCmd.Flags()) - - persistentFlags.AddSeedFlag() - persistentFlags.AddHostFlag() - - flags.AddNamespaceFlag() - flags.AddSetsFlag() - flags.AddIndexNameFlag() - flags.AddVectorFieldFlag() - flags.AddDimensionFlag() - flags.AddDistanceMetricFlag() - flags.AddIndexMetaFlag() - flags.AddTimeoutFlag() - - var requiredFlags = []string{ - flagNameNamespace, - flagNameIndexName, - flagNameVectorField, - flagNameDimension, - flagNameDistanceMetric, - } - // TODO: Add custom template for usage to take into account terminal width // Ex: https://github.com/sigstore/cosign/pull/3011/files - flags.String(flagNameStorageNamespace, "", commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) - flags.String(flagNameStorageSet, "", commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) - flags.Uint32(flagNameMaxEdges, 0, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) - flags.Uint32(flagNameConstructionEf, 0, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) - flags.Uint32(flagNameEf, 0, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) - flags.Uint32(flagNameBatchMaxRecords, 0, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) - flags.Uint32(flagNameBatchInterval, 0, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) - flags.Bool(flagNameBatchEnabled, true, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) - - for _, flag := range requiredFlags { + + flagSet := newCreateIndexFlagSet() + createIndexCmd.Flags().AddFlagSet(flagSet) + + for _, flag := range createIndexRequiredFlags { createIndexCmd.MarkFlagRequired(flag) } - - createIndexCmd.MarkFlagsMutuallyExclusive(flagNameHost, flagNameSeeds) } diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index c332b1f..7136451 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -5,86 +5,114 @@ package cmd import ( "context" + "fmt" "log/slog" + "time" avs "github.com/aerospike/aerospike-proximus-client-go" + commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/spf13/viper" ) +type diFlags struct { + host *HostPortFlag + seeds *SeedsSliceFlag + listenerName StringOptionalFlag + namespace string + sets []string + indexName string + timeout time.Duration +} + +var dropIndexFlags = &diFlags{ + host: NewDefaultHostPortFlag(), + seeds: &SeedsSliceFlag{}, +} + +func newDropIndexFlagSet() *pflag.FlagSet { + flagSet := &pflag.FlagSet{} + flagSet.VarP(dropIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) + flagSet.Var(dropIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) + flagSet.VarP(&dropIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) + flagSet.StringVarP(&dropIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) + flagSet.StringArrayVarP(&dropIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) + flagSet.StringVarP(&dropIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) + flagSet.DurationVar(&dropIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) + + return flagSet +} + var dropIndexRequiredFlags = []string{ flagNameNamespace, flagNameIndexName, - flagNameDimension, - flagNameDistanceMetric, } // dropIndexCmd represents the dropIndex command -var dropIndexCmd = &cobra.Command{ - Use: "index", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples +func newDropIndexCommand() *cobra.Command { + return &cobra.Command{ + Use: "index", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - RunE: func(cmd *cobra.Command, args []string) error { - // TODO: likely add to prerun step - seed := viper.GetString(flagNameSeeds) - // port := viper.GetInt(flagNamePort) - hostPort := avs.NewHostPort(seed, 5002, false) - namespace := viper.GetString(flagNameNamespace) - indexName := viper.GetString(flagNameIndexName) - timeout := viper.GetDuration(flagNameTimeout) - - // logger.Debug("parsed flags", - // slog.String("seeds", seed), slog.Int("port", port), slog.String("namespace", namespace), - // slog.String("index-name", indexName), slog.Duration("timeout", timeout), - // ) - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - adminClient, err := avs.NewAdminClient(ctx, avs.HostPortSlice{hostPort}, nil, false, logger) - if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) - return err - } - - cancel() - defer adminClient.Close() - - ctx, cancel = context.WithTimeout(context.Background(), timeout) - defer cancel() - - err = adminClient.IndexDrop(ctx, namespace, indexName) - if err != nil { - logger.Error("unable to drop index", slog.Any("error", err)) - return err - } - - view.Printf("Successfully dropped index %s.%s", namespace, indexName) - return nil - }, + PreRunE: func(cmd *cobra.Command, args []string) error { + if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { + return fmt.Errorf(fmt.Sprintf("only --%s or --%s allowed", flagNameSeeds, flagNameHost)) + } + + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + logger.Debug("parsed flags", + slog.String(flagNameHost, dropIndexFlags.host.String()), + slog.String(flagNameSeeds, dropIndexFlags.seeds.String()), + slog.String(flagNameListenerName, dropIndexFlags.listenerName.String()), + slog.String(flagNameNamespace, dropIndexFlags.namespace), + slog.Any(flagNameSets, dropIndexFlags.sets), + slog.String(flagNameIndexName, dropIndexFlags.indexName), + slog.Duration(flagNameTimeout, dropIndexFlags.timeout), + ) + + hosts, isLoadBalancer := parseBothHostSeedsFlag(*dropIndexFlags.seeds, *dropIndexFlags.host) + + ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) + defer cancel() + + adminClient, err := avs.NewAdminClient(ctx, hosts, nil, isLoadBalancer, logger) + if err != nil { + logger.Error("failed to create AVS client", slog.Any("error", err)) + return err + } + + cancel() + defer adminClient.Close() + + ctx, cancel = context.WithTimeout(context.Background(), dropIndexFlags.timeout) + defer cancel() + + err = adminClient.IndexDrop(ctx, dropIndexFlags.namespace, dropIndexFlags.indexName) + if err != nil { + logger.Error("unable to drop index", slog.Any("error", err)) + return err + } + + view.Printf("Successfully dropped index %s.%s", dropIndexFlags.namespace, dropIndexFlags.indexName) + return nil + }, + } } func init() { + dropIndexCmd := newDropIndexCommand() dropCmd.AddCommand(dropIndexCmd) + dropIndexCmd.Flags().AddFlagSet(newDropIndexFlagSet()) - flags := NewFlagSetBuilder(dropIndexCmd.Flags()) - flags.AddSeedFlag() - // flags.AddPortFlag() - flags.AddNamespaceFlag() - flags.AddIndexNameFlag() - flags.AddTimeoutFlag() - - var requiredFlags = []string{ - flagNameNamespace, - flagNameIndexName, - } - - for _, flag := range requiredFlags { + for _, flag := range dropIndexRequiredFlags { dropIndexCmd.MarkFlagRequired(flag) } } diff --git a/cmd/flags.go b/cmd/flags.go index 8410455..a2713e1 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -2,174 +2,354 @@ package cmd import ( "fmt" + "regexp" + "strconv" "strings" - "time" + + avs "github.com/aerospike/aerospike-proximus-client-go" "github.com/aerospike/aerospike-proximus-client-go/protos" - "github.com/aerospike/tools-common-go/flags" - "github.com/spf13/pflag" - "github.com/spf13/viper" ) const ( - flagNameSeeds = "seeds" - flagNameHost = "host" - flagNameListenerName = "listener-name" - flagNameNamespace = "namespace" - flagNameSets = "sets" - flagNameIndexName = "index-name" - flagNameVectorField = "vector-field" - flagNameDimension = "dimension" - flagNameDistanceMetric = "distance-metric" - flagNameIndexMeta = "index-meta" - flagNameTimeout = "timeout" - flagNameVerbose = "verbose" + logLevelFlagName = "log-level" + flagNameSeeds = "seeds" + flagNameHost = "host" + flagNameListenerName = "listener-name" + flagNameNamespace = "namespace" + flagNameSets = "sets" + flagNameIndexName = "index-name" + flagNameVectorField = "vector-field" + flagNameDimension = "dimension" + flagNameDistanceMetric = "distance-metric" + flagNameIndexMeta = "index-meta" + flagNameTimeout = "timeout" + flagNameVerbose = "verbose" + flagNameStorageNamespace = "storage-namespace" + flagNameStorageSet = "storage-set" + flagNameMaxEdges = "hnsw-max-edges" + flagNameConstructionEf = "hnsw-ef-construction" + flagNameEf = "hnsw-ef" + flagNameBatchMaxRecords = "hnsw-batch-max-records" + flagNameBatchInterval = "hnsw-batch-interval" + flagNameBatchEnabled = "hnsw-batch-enabled" ) -func viperGetIfSetString(flagName string) *string { - if viper.IsSet(flagName) { - s := viper.GetString(flagName) - return &s +type DistanceMetricFlag string + +// This is just a set of valid VectorDistanceMetrics. The value does not have meaning +var distanceMetricSet = protos.VectorDistanceMetric_value + +func (f *DistanceMetricFlag) Set(val string) error { + val = strings.ToUpper(val) + if _, ok := distanceMetricSet[val]; ok { + *f = DistanceMetricFlag(val) + return nil } - return nil + return fmt.Errorf("unrecognized distance metric") } -func viperGetIfSetBool(flagName string) *bool { - if viper.IsSet(flagName) { - s := viper.GetBool(flagName) - return &s +func (f *DistanceMetricFlag) Type() string { + names := []string{} + + for key := range distanceMetricSet { + names = append(names, key) } - return nil + return strings.Join(names, ",") +} + +func (f *DistanceMetricFlag) String() string { + return string(*f) +} + +type LogLevelFlag string + +var logLevelSet = map[string]struct{}{ + "DEBUG": {}, + "INFO": {}, + "WARN": {}, + "ERROR": {}, +} + +func (f *LogLevelFlag) NotSet() bool { + return *f == "" } -func viperGetIfSetUint32(flagName string) *uint32 { - if viper.IsSet(flagName) { - s := viper.GetUint32(flagName) - return &s +func (f *LogLevelFlag) Set(val string) error { + if val == "" { + *f = LogLevelFlag("") + return nil } - return nil + val = strings.ToUpper(val) + if _, ok := logLevelSet[val]; ok { + *f = LogLevelFlag(val) + return nil + } + + return fmt.Errorf("unrecognized log level") } -type FlagSetBuilder struct { - *pflag.FlagSet +func (f *LogLevelFlag) Type() string { + names := []string{} + + for key := range logLevelSet { + names = append(names, key) + } + + return strings.Join(names, ",") } -func NewFlagSetBuilder(flagSet *pflag.FlagSet) *FlagSetBuilder { - return &FlagSetBuilder{ - flagSet, +func (f *LogLevelFlag) String() string { + return string(*f) +} + +const ( + DefaultIPv4 = "127.0.0.1" + DefaultPort = 5000 +) + +func parseHostPort(v string) (*avs.HostPort, error) { + host := &avs.HostPort{} + ipv6HostPattern := `^\[(?P.*)\]` + hostPattern := `^(?P[^:]*)` // matched ipv4 and hostname + portPattern := `(?P\d+)$` + reIPv6Host := regexp.MustCompile(fmt.Sprintf("%s$", ipv6HostPattern)) + reIPv6HostPort := regexp.MustCompile(fmt.Sprintf("%s:%s", ipv6HostPattern, portPattern)) + reIPv4Host := regexp.MustCompile(fmt.Sprintf("%s$", hostPattern)) + reIPv4HostPort := regexp.MustCompile(fmt.Sprintf("%s:%s", hostPattern, portPattern)) + + regexsAndNames := []struct { + regex *regexp.Regexp + groupNames []string + }{ + // The order is important since the ipv4 pattern also matches ipv6 + {reIPv6HostPort, reIPv6HostPort.SubexpNames()}, + {reIPv6Host, reIPv6Host.SubexpNames()}, + {reIPv4HostPort, reIPv4HostPort.SubexpNames()}, + {reIPv4Host, reIPv4Host.SubexpNames()}, } + + for _, r := range regexsAndNames { + regex := r.regex + groupNames := r.groupNames + + if matchs := regex.FindStringSubmatch(v); matchs != nil { + for idx, match := range matchs { + var err error + + name := groupNames[idx] + + switch { + case name == "host": + host.Host = match + case name == "port": + var intPort int64 + + intPort, err = strconv.ParseInt(match, 0, 0) + + if err == nil { + host.Port = int(intPort) + } + } + + if err != nil { + return host, fmt.Errorf("failed to parse %s : %s", name, err) + } + } + + return host, nil + } + } + + return host, fmt.Errorf("does not match any expected formats") +} + +// A cobra PFlag to parse and display help info for the host[:tls-name][:port] +// input option. It implements the pflag Value and SliceValue interfaces to +// enable automatic parsing by cobra. +type HostPortFlag struct { + HostPort avs.HostPort } -// TODO: Should this be a list of IPs? Should we support IP:PORT? -func (fsb *FlagSetBuilder) AddSeedFlag() { - fsb.StringArray(flagNameSeeds, []string{}, flags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) +func NewDefaultHostPortFlag() *HostPortFlag { + return &HostPortFlag{ + HostPort: avs.HostPort{ + Host: DefaultIPv4, + Port: DefaultPort, + }, + } } -func (fsb *FlagSetBuilder) AddHostFlag() { - fsb.StringP(flagNameHost, "h", "127.0.0.1:5000", fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds)) +func (hp *HostPortFlag) Set(val string) error { + hostPort, err := parseHostPort(val) + if err != nil { + return err + } + + hp.HostPort = *hostPort + + return nil } -func (fsb *FlagSetBuilder) AddNamespaceFlag() { - fsb.StringP(flagNameNamespace, "n", "", "The namespace for the index.") +func (hp *HostPortFlag) Type() string { + return "host[:port]" } -func (fsb *FlagSetBuilder) AddSetsFlag() { - fsb.StringArrayP(flagNameSets, "s", nil, "The sets for the index.") +func (hp *HostPortFlag) String() string { + return hp.HostPort.String() } -func (fsb *FlagSetBuilder) AddIndexNameFlag() { - fsb.StringP(flagNameIndexName, "i", "", "The name of the index.") +type SeedsSliceFlag struct { + Seeds avs.HostPortSlice +} +func NewSeedsSliceFlag() SeedsSliceFlag { + return SeedsSliceFlag{ + Seeds: avs.HostPortSlice{}, + } } -func (fsb *FlagSetBuilder) AddVectorFieldFlag() { - fsb.StringP(flagNameVectorField, "f", "", "The name of the vector field.") +// Append adds the specified value to the end of the flag value list. +func (slice *SeedsSliceFlag) Append(val string) error { + host, err := parseHostPort(val) + + if err != nil { + return err + } + + slice.Seeds = append(slice.Seeds, host) + return nil } -func (fsb *FlagSetBuilder) AddDimensionFlag() { - fsb.IntP(flagNameDimension, "d", 0, "The dimension of the vector field.") +// Replace will fully overwrite any data currently in the flag value list. +func (slice *SeedsSliceFlag) Replace(vals []string) error { + slice.Seeds = avs.HostPortSlice{} + + for _, val := range vals { + if err := slice.Append(val); err != nil { + return err + } + } + return nil } -func (fsb *FlagSetBuilder) AddDistanceMetricFlag() { - distMetric := DistanceMetricFlag("") - fsb.VarP(&distMetric, flagNameDistanceMetric, "m", "The distance metric for the index.") +// GetSlice returns the flag value list as an array of strings. +func (slice *SeedsSliceFlag) GetSlice() []string { + strs := []string{} + + for _, elem := range slice.Seeds { + strs = append(strs, elem.String()) + } + + return strs } -func (fsb *FlagSetBuilder) AddIndexMetaFlag() { - fsb.StringToStringP(flagNameIndexMeta, "e", nil, "The metadata for the index.") +func (slice *SeedsSliceFlag) Set(commaSepVal string) error { + vals := strings.Split(commaSepVal, ",") + + for _, val := range vals { + if err := slice.Append(val); err != nil { + return err + } + } + + return nil } -func (fsb *FlagSetBuilder) AddTimeoutFlag() { - fsb.DurationP(flagNameTimeout, "t", time.Second*5, "The timeout used for each request.") +func (slice *SeedsSliceFlag) Type() string { + return "seed[:port][,...]" } -func (fsb *FlagSetBuilder) AddVerbose() { - fsb.BoolP(flagNameVerbose, "v", false, "Display extra detail about an index.") +func (slice *SeedsSliceFlag) String() string { + return slice.Seeds.String() } -type DistanceMetricFlag string +func parseBothHostSeedsFlag(seeds SeedsSliceFlag, host HostPortFlag) (avs.HostPortSlice, bool) { -// This is just a set of valid VectorDistanceMetrics. The value does not have meaning -var distanceMetricSet = protos.VectorDistanceMetric_value + isLoadBalancer := false + hosts := avs.HostPortSlice{} -func (f *DistanceMetricFlag) Set(val string) error { - val = strings.ToUpper(val) - if _, ok := distanceMetricSet[val]; ok { - *f = DistanceMetricFlag(val) - return nil + if len(seeds.Seeds) > 0 { + logger.Debug("seeds is set") + + hosts = append(hosts, seeds.Seeds...) + } else { + logger.Debug("hosts is set") + + isLoadBalancer = true + hosts = append(hosts, &host.HostPort) } - return fmt.Errorf("unrecognized distance metric") + return hosts, isLoadBalancer } -func (f *DistanceMetricFlag) Type() string { - names := []string{} +type StringOptionalFlag struct { + Val *string +} - for key := range distanceMetricSet { - names = append(names, key) +func (f *StringOptionalFlag) Set(val string) error { + f.Val = &val + return nil +} + +func (f *StringOptionalFlag) Type() string { + return "string" +} + +func (f *StringOptionalFlag) String() string { + if f.Val != nil { + return *f.Val } - return strings.Join(names, ",") + return "" } -func (f *DistanceMetricFlag) String() string { - return string(*f) +type Uint32OptionalFlag struct { + Val *uint32 } -type LogLevelFlag string +func (f *Uint32OptionalFlag) Set(val string) error { + v, err := strconv.ParseUint(val, 0, 32) + u32Val := uint32(v) + f.Val = &u32Val + return err +} -var logLevelSet = map[string]struct{}{ - "DEBUG": {}, - "INFO": {}, - "WARN": {}, - "ERROR": {}, +func (f *Uint32OptionalFlag) Type() string { + return "uint32" } -func (f *LogLevelFlag) Set(val string) error { - val = strings.ToUpper(val) - if _, ok := logLevelSet[val]; ok { - *f = LogLevelFlag(val) - return nil +func (f *Uint32OptionalFlag) String() string { + if f.Val != nil { + return strconv.FormatUint(uint64(*f.Val), 10) } - return fmt.Errorf("unrecognized log level") + return "" } -func (f *LogLevelFlag) Type() string { - names := []string{} +type BoolOptionalFlag struct { + Val *bool +} - for key := range logLevelSet { - names = append(names, key) - } +func (f *BoolOptionalFlag) Set(val string) error { + v, err := strconv.ParseBool(val) + f.Val = &v + return err +} - return strings.Join(names, ",") +func (f *BoolOptionalFlag) Type() string { + return "bool" } -func (f *LogLevelFlag) String() string { - return string(*f) +func (f *BoolOptionalFlag) String() string { + if f.Val != nil { + return strconv.FormatBool(*f.Val) + } + + return "" } diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 6242b98..46e3c05 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -5,94 +5,140 @@ package cmd import ( "context" + "fmt" "log/slog" + "sync" + "time" avs "github.com/aerospike/aerospike-proximus-client-go" "github.com/aerospike/aerospike-proximus-client-go/protos" + commonFlags "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/spf13/viper" ) -// type indexInfo struct { -// Definition *protos.IndexDefinition -// Status *protos.IndexStatusResponse -// } +type liFlags struct { + host *HostPortFlag + seeds *SeedsSliceFlag + listenerName StringOptionalFlag + verbose bool + timeout time.Duration +} + +var listIndexFlags = &liFlags{ + host: NewDefaultHostPortFlag(), + seeds: &SeedsSliceFlag{}, +} + +func newListIndexFlagSet() *pflag.FlagSet { + flagSet := &pflag.FlagSet{} + flagSet.VarP(listIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) + flagSet.Var(listIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) + flagSet.VarP(&listIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) + flagSet.BoolVarP(&listIndexFlags.verbose, flagNameVerbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) + flagSet.DurationVar(&listIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) + + return flagSet +} + +var listIndexRequiredFlags = []string{ + flagNameNamespace, + flagNameIndexName, +} // listIndexCmd represents the listIndex command -var listIndexCmd = &cobra.Command{ - Use: "index", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples +func newListIndexCmd() *cobra.Command { + return &cobra.Command{Use: "index", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - RunE: func(cmd *cobra.Command, args []string) error { - seed := viper.GetString(flagNameSeeds) - // port := viper.GetInt(flagNamePort) - hostPort := avs.NewHostPort(seed, 5002, false) - timeout := viper.GetDuration(flagNameTimeout) - verbose := viper.GetBool(flagNameVerbose) + PreRunE: func(cmd *cobra.Command, args []string) error { + if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { + return fmt.Errorf(fmt.Sprintf("only --%s or --%s allowed", flagNameSeeds, flagNameHost)) + } - // logger.Debug("parsed flags", - // slog.String("seeds", seed), slog.Int("port", port), slog.Duration("timeout", timeout), - // ) + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + logger.Debug("parsed flags", + slog.String(flagNameHost, listIndexFlags.host.String()), + slog.String(flagNameSeeds, listIndexFlags.seeds.String()), + slog.String(flagNameListenerName, listIndexFlags.listenerName.String()), + slog.Bool(flagNameVerbose, listIndexFlags.verbose), + slog.Duration(flagNameTimeout, listIndexFlags.timeout), + ) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() + hosts, isLoadBalancer := parseBothHostSeedsFlag(*listIndexFlags.seeds, *listIndexFlags.host) - adminClient, err := avs.NewAdminClient(ctx, avs.HostPortSlice{hostPort}, nil, false, logger) - if err != nil { - logger.Error("failed to create AVS client", slog.Any("error", err)) - return err - } + ctx, cancel := context.WithTimeout(context.Background(), listIndexFlags.timeout) + defer cancel() - cancel() - defer adminClient.Close() + adminClient, err := avs.NewAdminClient(ctx, hosts, listIndexFlags.listenerName.Val, isLoadBalancer, logger) + if err != nil { + logger.Error("failed to create AVS client", slog.Any("error", err)) + return err + } - ctx, cancel = context.WithTimeout(context.Background(), timeout) - defer cancel() + cancel() + defer adminClient.Close() - indexList, err := adminClient.IndexList(ctx) - if err != nil { - logger.Error("failed to list indexes", slog.Any("error", err)) - return err - } + ctx, cancel = context.WithTimeout(context.Background(), listIndexFlags.timeout) + defer cancel() - indexStatusList := make([]*protos.IndexStatusResponse, len(indexList.GetIndices())) + indexList, err := adminClient.IndexList(ctx) + if err != nil { + logger.Error("failed to list indexes", slog.Any("error", err)) + return err + } - if verbose { - cancel() + indexStatusList := make([]*protos.IndexStatusResponse, len(indexList.GetIndices())) - ctx, cancel = context.WithTimeout(context.Background(), timeout) - defer cancel() + if listIndexFlags.verbose { + cancel() + + ctx, cancel = context.WithTimeout(context.Background(), listIndexFlags.timeout) + defer cancel() - for i, index := range indexList.GetIndices() { - indexStatus, err := adminClient.IndexGetStatus(ctx, index.Id.Namespace, index.Id.Name) - if err != nil { - logger.ErrorContext(ctx, "failed to get index status", slog.Any("error", err), slog.String("index", index.Id.String())) - continue + wg := sync.WaitGroup{} + for i, index := range indexList.GetIndices() { + wg.Add(1) + go func(i int, index *protos.IndexDefinition) { + defer wg.Done() + indexStatus, err := adminClient.IndexGetStatus(ctx, index.Id.Namespace, index.Id.Name) + if err != nil { + logger.ErrorContext(ctx, "failed to get index status", slog.Any("error", err), slog.String("index", index.Id.String())) + return + } + + indexStatusList[i] = indexStatus + logger.Debug("server index status", slog.Int("index", i), slog.Any("response", indexStatus)) + }(i, index) } - indexStatusList[i] = indexStatus + wg.Wait() } - } + logger.Debug("server index list", slog.String("response", indexList.String())) - logger.Debug(indexList.String()) - view.PrintIndexes(indexList, indexStatusList, verbose) + view.PrintIndexes(indexList, indexStatusList, listIndexFlags.verbose) - return nil - }, + return nil + }, + } } func init() { + listIndexCmd := newListIndexCmd() + listCmd.AddCommand(listIndexCmd) + listIndexCmd.Flags().AddFlagSet(newListIndexFlagSet()) - flags := NewFlagSetBuilder(listIndexCmd.Flags()) - flags.AddSeedFlag() - // flags.AddPortFlag() - flags.AddTimeoutFlag() - flags.AddVerbose() + for _, flag := range listIndexRequiredFlags { + listIndexCmd.MarkFlagRequired(flag) + } } diff --git a/cmd/root.go b/cmd/root.go index ea2eee9..8bded26 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,19 +5,26 @@ package cmd import ( "context" + "fmt" "log/slog" "os" common "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/spf13/viper" ) var lvl = new(slog.LevelVar) -var logLevelFlagName = "log-level" var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: lvl})) var view = NewView(os.Stdout, logger) +type rootFlags_ struct { + logLevel LogLevelFlag +} + +var rootFlags = &rootFlags_{} + // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "asvec", @@ -28,20 +35,37 @@ examples and usage of using your application. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - PersistentPreRun: func(cmd *cobra.Command, args []string) { - cmd.SilenceUsage = true - viper.BindPFlags(cmd.PersistentFlags()) - viper.BindPFlags(cmd.Flags()) - - if viper.IsSet(logLevelFlagName) { - level := viper.GetString(logLevelFlagName) + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if rootFlags.logLevel.NotSet() { + lvl.Set(slog.LevelError + 1) // disable all logging + } else { + level := rootFlags.logLevel handler := logger.Handler() lvl.UnmarshalText([]byte(level)) handler.Enabled(context.Background(), lvl.Level()) - } else { - lvl.Set(slog.LevelError + 1) // disable all logging } + + cmd.SilenceUsage = true + viper.BindPFlags(cmd.PersistentFlags()) + viper.BindPFlags(cmd.Flags()) + + var persistedErr error + flags := cmd.Flags() + + flags.VisitAll(func(f *pflag.Flag) { + val := viper.GetString(f.Name) + + // Apply the viper config value to the flag when viper has a value + if viper.IsSet(f.Name) && !f.Changed { + if err := f.Value.Set(val); err != nil { + persistedErr = fmt.Errorf("failed to parse flag %s: %s", f.Name, err) + } + } + }) + + return persistedErr + }, } @@ -55,9 +79,9 @@ func Execute() { } func init() { - logLevel := LogLevelFlag("disabled") - rootCmd.PersistentFlags().Var(&logLevel, logLevelFlagName, "Log level for additional details and debugging") + rootCmd.PersistentFlags().Var(&rootFlags.logLevel, logLevelFlagName, "Log level for additional details and debugging") common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") viper.SetEnvPrefix("ASVEC") - viper.AutomaticEnv() + viper.BindEnv(flagNameHost) + viper.BindEnv(flagNameSeeds) } diff --git a/go.mod b/go.mod index ad47495..1b5a952 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module asvec go 1.21.7 //replace github.com/aerospike/aerospike-proximus-client-go => github.com/aerospike/aerospike-proximus-client-go VEC-155-admin-client -// replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go +//replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go require ( github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 @@ -15,7 +15,7 @@ require ( require ( github.com/aerospike/aerospike-client-go/v7 v7.2.1 // indirect - github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240530224737-115c3007bafa // indirect + github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index a76251d..8973132 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9 github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240530224737-115c3007bafa h1:+DUvX+HJvY4ZWEVXjlN14iwFBwPP8br0mDL8XPUO3E0= github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240530224737-115c3007bafa/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 h1:qVpPCrbp0pNNmP1CPqln6HkzhVmFmOOVZYLq4IDlidI= +github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= From e597fd0850052cd29d41407ddb2a9e11191ec0a5 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Mon, 3 Jun 2024 16:42:40 -0700 Subject: [PATCH 18/35] clean up lint errors --- .golangci.yml | 10 +++++-- cmd/createIndex.go | 58 ++++++++++++++++++++-------------------- cmd/drop.go | 10 ------- cmd/dropIndex.go | 32 +++++++++++----------- cmd/flags.go | 12 ++++++--- cmd/list.go | 16 ----------- cmd/listIndex.go | 33 +++++++++++++---------- cmd/root.go | 34 ++++++++++++++++------- cmd/view.go | 9 ++++--- cmd/writers/default.go | 4 ++- cmd/writers/indexList.go | 9 ++++--- main.go | 1 - 12 files changed, 120 insertions(+), 108 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index f41cb1c..32c6591 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,7 +11,7 @@ linters-settings: - performance - style govet: - check-shadowing: true + shadow: true enable: - fieldalignment nolintlint: @@ -22,7 +22,13 @@ linters-settings: Main: allow: - $gostd - - github.com/aerospike/aerospike-proximus-client-go/protos + - github.com/aerospike/tools-common-go + - github.com/aerospike/aerospike-proximus-client-go + - asvec/cmd + - github.com/spf13/cobra + - github.com/spf13/viper + - github.com/spf13/pflag + - github.com/jedib0t/go-pretty linters: disable-all: true diff --git a/cmd/createIndex.go b/cmd/createIndex.go index 244021c..53eaed9 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -17,9 +17,8 @@ import ( "github.com/spf13/viper" ) -const () - -type ciFlags struct { +//nolint:govet // Padding not a concern for a CLI +var createIndexFlags = &struct { host *HostPortFlag seeds *SeedsSliceFlag listenerName StringOptionalFlag @@ -39,9 +38,7 @@ type ciFlags struct { hnswBatchInterval Uint32OptionalFlag hnswBatchEnabled BoolOptionalFlag timeout time.Duration -} - -var createIndexFlags = &ciFlags{ +}{ host: NewDefaultHostPortFlag(), seeds: &SeedsSliceFlag{}, storageNamespace: StringOptionalFlag{}, @@ -56,25 +53,25 @@ var createIndexFlags = &ciFlags{ func newCreateIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.VarP(createIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) - flagSet.Var(createIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) - flagSet.VarP(&createIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) - flagSet.StringVarP(&createIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) - flagSet.StringArrayVarP(&createIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) - flagSet.StringVarP(&createIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) - flagSet.StringVarP(&createIndexFlags.vectorField, flagNameVectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) - flagSet.Uint32VarP(&createIndexFlags.dimensions, flagNameDimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) - flagSet.VarP(&createIndexFlags.distanceMetric, flagNameDistanceMetric, "m", commonFlags.DefaultWrapHelpString("The distance metric for the index.")) - flagSet.StringToStringVar(&createIndexFlags.indexMeta, flagNameIndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) - flagSet.DurationVar(&createIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) - flagSet.Var(&createIndexFlags.storageNamespace, flagNameStorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) - flagSet.Var(&createIndexFlags.storageSet, flagNameStorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) - flagSet.Var(&createIndexFlags.hnswMaxEdges, flagNameMaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) - flagSet.Var(&createIndexFlags.hnswConstructionEf, flagNameConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) - flagSet.Var(&createIndexFlags.hnswEf, flagNameEf, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) - flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flagNameBatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) - flagSet.Var(&createIndexFlags.hnswBatchInterval, flagNameBatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) - flagSet.Var(&createIndexFlags.hnswBatchEnabled, flagNameBatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) + flagSet.VarP(createIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability + flagSet.Var(createIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability + flagSet.VarP(&createIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability + flagSet.StringArrayVarP(&createIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability + flagSet.StringVarP(&createIndexFlags.vectorField, flagNameVectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability + flagSet.Uint32VarP(&createIndexFlags.dimensions, flagNameDimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability + flagSet.VarP(&createIndexFlags.distanceMetric, flagNameDistanceMetric, "m", commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.StringToStringVar(&createIndexFlags.indexMeta, flagNameIndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.DurationVar(&createIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.storageNamespace, flagNameStorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) //nolint:lll // For readability //nolint:lll // For readability + flagSet.Var(&createIndexFlags.storageSet, flagNameStorageSet, commonFlags.DefaultWrapHelpString("Optional storage set where the index is stored. Defaults to the index name.")) //nolint:lll // For readability //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswMaxEdges, flagNameMaxEdges, commonFlags.DefaultWrapHelpString("Maximum number bi-directional links per HNSW vertex. Greater values of 'm' in general provide better recall for data with high dimensionality, while lower values work well for data with lower dimensionality. The storage space required for the index increases proportionally with 'm'. The default value is 16.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswConstructionEf, flagNameConstructionEf, commonFlags.DefaultWrapHelpString("The number of candidate nearest neighbors shortlisted during index creation. Larger values provide better recall at the cost of longer index update times. The default is 100.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswEf, flagNameEf, commonFlags.DefaultWrapHelpString("The default number of candidate nearest neighbors shortlisted during search. Larger values provide better recall at the cost of longer search times. The default is 100.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchMaxRecords, flagNameBatchMaxRecords, commonFlags.DefaultWrapHelpString("Maximum number of records to fit in a batch. The default value is 10000.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchInterval, flagNameBatchInterval, commonFlags.DefaultWrapHelpString("The maximum amount of time in milliseconds to wait before finalizing a batch. The default value is 10000.")) //nolint:lll // For readability + flagSet.Var(&createIndexFlags.hnswBatchEnabled, flagNameBatchEnabled, commonFlags.DefaultWrapHelpString("Enables batching for index updates. Default is true meaning batching is enabled.")) //nolint:lll // For readability return flagSet } @@ -98,14 +95,14 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - PreRunE: func(cmd *cobra.Command, args []string) error { + PreRunE: func(_ *cobra.Command, _ []string) error { if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf(fmt.Sprintf("only --%s or --%s allowed", flagNameSeeds, flagNameHost)) + return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) } return nil }, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { hosts, isLoadBalancer := parseBothHostSeedsFlag(*createIndexFlags.seeds, *createIndexFlags.host) logger.Debug("parsed flags", @@ -204,6 +201,9 @@ func init() { createIndexCmd.Flags().AddFlagSet(flagSet) for _, flag := range createIndexRequiredFlags { - createIndexCmd.MarkFlagRequired(flag) + err := createIndexCmd.MarkFlagRequired(flag) + if err != nil { + panic(err) + } } } diff --git a/cmd/drop.go b/cmd/drop.go index 267021c..bf5b956 100644 --- a/cmd/drop.go +++ b/cmd/drop.go @@ -21,14 +21,4 @@ to quickly create a Cobra application.`, func init() { rootCmd.AddCommand(dropCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // dropCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // dropCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index 7136451..daa389c 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -16,7 +16,8 @@ import ( "github.com/spf13/viper" ) -type diFlags struct { +//nolint:govet // Padding not a concern for a CLI +var dropIndexFlags = &struct { host *HostPortFlag seeds *SeedsSliceFlag listenerName StringOptionalFlag @@ -24,22 +25,20 @@ type diFlags struct { sets []string indexName string timeout time.Duration -} - -var dropIndexFlags = &diFlags{ +}{ host: NewDefaultHostPortFlag(), seeds: &SeedsSliceFlag{}, } func newDropIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.VarP(dropIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) - flagSet.Var(dropIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) - flagSet.VarP(&dropIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) - flagSet.StringVarP(&dropIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) - flagSet.StringArrayVarP(&dropIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) - flagSet.StringVarP(&dropIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) - flagSet.DurationVar(&dropIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) + flagSet.VarP(dropIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability + flagSet.Var(dropIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability + flagSet.VarP(&dropIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability + flagSet.StringVarP(&dropIndexFlags.namespace, flagNameNamespace, "n", "", commonFlags.DefaultWrapHelpString("The namespace for the index.")) //nolint:lll // For readability + flagSet.StringArrayVarP(&dropIndexFlags.sets, flagNameSets, "s", nil, commonFlags.DefaultWrapHelpString("The sets for the index.")) //nolint:lll // For readability + flagSet.StringVarP(&dropIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability + flagSet.DurationVar(&dropIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability return flagSet } @@ -60,14 +59,14 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - PreRunE: func(cmd *cobra.Command, args []string) error { + PreRunE: func(_ *cobra.Command, _ []string) error { if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf(fmt.Sprintf("only --%s or --%s allowed", flagNameSeeds, flagNameHost)) + return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) } return nil }, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { logger.Debug("parsed flags", slog.String(flagNameHost, dropIndexFlags.host.String()), slog.String(flagNameSeeds, dropIndexFlags.seeds.String()), @@ -113,6 +112,9 @@ func init() { dropIndexCmd.Flags().AddFlagSet(newDropIndexFlagSet()) for _, flag := range dropIndexRequiredFlags { - dropIndexCmd.MarkFlagRequired(flag) + err := dropIndexCmd.MarkFlagRequired(flag) + if err != nil { + panic(err) + } } } diff --git a/cmd/flags.go b/cmd/flags.go index a2713e1..5bfbf33 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -270,7 +270,6 @@ func (slice *SeedsSliceFlag) String() string { } func parseBothHostSeedsFlag(seeds SeedsSliceFlag, host HostPortFlag) (avs.HostPortSlice, bool) { - isLoadBalancer := false hosts := avs.HostPortSlice{} @@ -282,12 +281,15 @@ func parseBothHostSeedsFlag(seeds SeedsSliceFlag, host HostPortFlag) (avs.HostPo logger.Debug("hosts is set") isLoadBalancer = true + hosts = append(hosts, &host.HostPort) } return hosts, isLoadBalancer } +const optionalEmptyString = "" + type StringOptionalFlag struct { Val *string } @@ -306,7 +308,7 @@ func (f *StringOptionalFlag) String() string { return *f.Val } - return "" + return optionalEmptyString } type Uint32OptionalFlag struct { @@ -317,6 +319,7 @@ func (f *Uint32OptionalFlag) Set(val string) error { v, err := strconv.ParseUint(val, 0, 32) u32Val := uint32(v) f.Val = &u32Val + return err } @@ -329,7 +332,7 @@ func (f *Uint32OptionalFlag) String() string { return strconv.FormatUint(uint64(*f.Val), 10) } - return "" + return optionalEmptyString } type BoolOptionalFlag struct { @@ -339,6 +342,7 @@ type BoolOptionalFlag struct { func (f *BoolOptionalFlag) Set(val string) error { v, err := strconv.ParseBool(val) f.Val = &v + return err } @@ -351,5 +355,5 @@ func (f *BoolOptionalFlag) String() string { return strconv.FormatBool(*f.Val) } - return "" + return optionalEmptyString } diff --git a/cmd/list.go b/cmd/list.go index b8e09c2..8ab8abf 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -1,12 +1,9 @@ /* Copyright © 2024 NAME HERE - */ package cmd import ( - "fmt" - "github.com/spf13/cobra" ) @@ -20,21 +17,8 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("list called") - }, } func init() { rootCmd.AddCommand(listCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // listCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // listCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 46e3c05..260904d 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -18,26 +18,24 @@ import ( "github.com/spf13/viper" ) -type liFlags struct { +var listIndexFlags = &struct { host *HostPortFlag seeds *SeedsSliceFlag listenerName StringOptionalFlag verbose bool timeout time.Duration -} - -var listIndexFlags = &liFlags{ +}{ host: NewDefaultHostPortFlag(), seeds: &SeedsSliceFlag{}, } func newListIndexFlagSet() *pflag.FlagSet { flagSet := &pflag.FlagSet{} - flagSet.VarP(listIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) - flagSet.Var(listIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) - flagSet.VarP(&listIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) - flagSet.BoolVarP(&listIndexFlags.verbose, flagNameVerbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) - flagSet.DurationVar(&listIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) + flagSet.VarP(listIndexFlags.host, flagNameHost, "h", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS host to connect to. If cluster discovery is needed use --%s", flagNameSeeds))) //nolint:lll // For readability + flagSet.Var(listIndexFlags.seeds, flagNameSeeds, commonFlags.DefaultWrapHelpString(fmt.Sprintf("The AVS seeds to use for cluster discovery. If no cluster discovery is needed (i.e. load-balancer) then use --%s", flagNameHost))) //nolint:lll // For readability + flagSet.VarP(&listIndexFlags.listenerName, flagNameListenerName, "l", commonFlags.DefaultWrapHelpString("The listener to ask the AVS server for as configured in the AVS server. Likely required for cloud deployments.")) //nolint:lll // For readability + flagSet.BoolVarP(&listIndexFlags.verbose, flagNameVerbose, "v", false, commonFlags.DefaultWrapHelpString("Print detailed index information.")) //nolint:lll // For readability + flagSet.DurationVar(&listIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability return flagSet } @@ -57,14 +55,14 @@ and usage of using your command. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - PreRunE: func(cmd *cobra.Command, args []string) error { + PreRunE: func(_ *cobra.Command, _ []string) error { if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { - return fmt.Errorf(fmt.Sprintf("only --%s or --%s allowed", flagNameSeeds, flagNameHost)) + return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) } return nil }, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { logger.Debug("parsed flags", slog.String(flagNameHost, listIndexFlags.host.String()), slog.String(flagNameSeeds, listIndexFlags.seeds.String()), @@ -111,7 +109,11 @@ to quickly create a Cobra application.`, defer wg.Done() indexStatus, err := adminClient.IndexGetStatus(ctx, index.Id.Namespace, index.Id.Name) if err != nil { - logger.ErrorContext(ctx, "failed to get index status", slog.Any("error", err), slog.String("index", index.Id.String())) + logger.ErrorContext(ctx, + "failed to get index status", + slog.Any("error", err), + slog.String("index", index.Id.String()), + ) return } @@ -139,6 +141,9 @@ func init() { listIndexCmd.Flags().AddFlagSet(newListIndexFlagSet()) for _, flag := range listIndexRequiredFlags { - listIndexCmd.MarkFlagRequired(flag) + err := listIndexCmd.MarkFlagRequired(flag) + if err != nil { + panic(err) + } } } diff --git a/cmd/root.go b/cmd/root.go index 8bded26..965c5ff 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,11 +19,9 @@ var lvl = new(slog.LevelVar) var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: lvl})) var view = NewView(os.Stdout, logger) -type rootFlags_ struct { +var rootFlags = &struct { logLevel LogLevelFlag -} - -var rootFlags = &rootFlags_{} +}{} // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ @@ -35,20 +33,30 @@ examples and usage of using your application. For example: Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { if rootFlags.logLevel.NotSet() { lvl.Set(slog.LevelError + 1) // disable all logging } else { level := rootFlags.logLevel handler := logger.Handler() - lvl.UnmarshalText([]byte(level)) + + err := lvl.UnmarshalText([]byte(level)) + if err != nil { + return err + } handler.Enabled(context.Background(), lvl.Level()) } cmd.SilenceUsage = true - viper.BindPFlags(cmd.PersistentFlags()) - viper.BindPFlags(cmd.Flags()) + + if err := viper.BindPFlags(cmd.PersistentFlags()); err != nil { + return err + } + + if err := viper.BindPFlags(cmd.Flags()); err != nil { + return err + } var persistedErr error flags := cmd.Flags() @@ -82,6 +90,12 @@ func init() { rootCmd.PersistentFlags().Var(&rootFlags.logLevel, logLevelFlagName, "Log level for additional details and debugging") common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") viper.SetEnvPrefix("ASVEC") - viper.BindEnv(flagNameHost) - viper.BindEnv(flagNameSeeds) + + if err := viper.BindEnv(flagNameHost); err != nil { + logger.Error("failed to bind environment variable", slog.Any("error", err)) + } + + if err := viper.BindEnv(flagNameSeeds); err != nil { + logger.Error("failed to bind environment variable", slog.Any("error", err)) + } } diff --git a/cmd/view.go b/cmd/view.go index 355fe07..c3cbb14 100644 --- a/cmd/view.go +++ b/cmd/view.go @@ -1,7 +1,6 @@ package cmd import ( - // "asvec/cmd/writers" "asvec/cmd/writers" "fmt" "io" @@ -20,7 +19,7 @@ func NewView(writer io.Writer, logger *slog.Logger) *View { } func (v *View) Print(a ...any) { - _, err := v.writer.Write([]byte(fmt.Sprint(a...))) + _, err := fmt.Fprint(v.writer, a...) if err != nil { panic(err) } @@ -50,7 +49,11 @@ func (v *View) getIndexListWriter(verbose bool) *writers.IndexTableWriter { return writers.NewIndexTableWriter(v.writer, verbose, v.logger) } -func (v *View) PrintIndexes(indexList *protos.IndexDefinitionList, indexStatusList []*protos.IndexStatusResponse, verbose bool) { +func (v *View) PrintIndexes( + indexList *protos.IndexDefinitionList, + indexStatusList []*protos.IndexStatusResponse, + verbose bool, +) { t := v.getIndexListWriter(verbose) for i, index := range indexList.Indices { diff --git a/cmd/writers/default.go b/cmd/writers/default.go index 6c4ffeb..73fbc11 100644 --- a/cmd/writers/default.go +++ b/cmd/writers/default.go @@ -14,8 +14,10 @@ func NewDefaultWriter(writer io.Writer) table.Writer { t.AppendSeparator() t.SuppressEmptyColumns() t.SetStyle(table.StyleRounded) - t.Style().Title.Align = text.AlignCenter + t.Style().Title.Colors = append(t.Style().Title.Colors, text.Bold) + t.Style().Title.Align = text.AlignCenter + return t } diff --git a/cmd/writers/indexList.go b/cmd/writers/indexList.go index 60e7e0b..da07d78 100644 --- a/cmd/writers/indexList.go +++ b/cmd/writers/indexList.go @@ -10,6 +10,7 @@ import ( var rowConfigAutoMerge = table.RowConfig{AutoMerge: true} +//nolint:govet // Padding not a concern for a CLI type IndexTableWriter struct { table.Writer verbose bool @@ -20,20 +21,19 @@ func NewIndexTableWriter(writer io.Writer, verbose bool, logger *slog.Logger) *I t := IndexTableWriter{NewDefaultWriter(writer), verbose, logger} if verbose { - t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged", "Storage", "Index Parameters"}, rowConfigAutoMerge) + t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", + "Distance Metric", "Unmerged", "Storage", "Index Parameters"}, rowConfigAutoMerge) } else { t.AppendHeader(table.Row{"Name", "Namespace", "Set", "Field", "Dimensions", "Distance Metric", "Unmerged"}) } t.SetTitle("Indexes") - t.Style().Options.SeparateRows = true t.SetAutoIndex(true) t.SortBy([]table.SortBy{ {Name: "Namespace", Mode: table.Asc}, {Name: "Set", Mode: table.Asc}, {Name: "Name", Mode: table.Asc}, }) - t.SetColumnConfigs([]table.ColumnConfig{ { @@ -42,6 +42,8 @@ func NewIndexTableWriter(writer io.Writer, verbose bool, logger *slog.Logger) *I }, }) + t.Style().Options.SeparateRows = true + return &t } @@ -69,6 +71,7 @@ func (itw *IndexTableWriter) AppendIndexRow(index *protos.IndexDefinition, statu {"Batch Interval", v.HnswParams.BatchingParams.GetInterval()}, {"Batch Enabled", !v.HnswParams.BatchingParams.GetDisabled()}, }) + row = append(row, tHNSW.Render()) default: itw.logger.Warn("the server returned unrecognized index type params. recognized index param types are: HNSW") diff --git a/main.go b/main.go index bb0c559..23be779 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,5 @@ /* Copyright © 2024 NAME HERE - */ package main From 6b3b60004aa8c2b591f14b5f7bdebceaa4e02fbf Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 4 Jun 2024 10:44:59 -0700 Subject: [PATCH 19/35] add link check action --- .github/workflows/readme-check-links.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/readme-check-links.yml diff --git a/.github/workflows/readme-check-links.yml b/.github/workflows/readme-check-links.yml new file mode 100644 index 0000000..d2fb5a4 --- /dev/null +++ b/.github/workflows/readme-check-links.yml @@ -0,0 +1,14 @@ +name: Check Markdown links + +on: push + +jobs: + markdown-link-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: yes + file-path: './**/*.md, ./LICENSE, ./**/*.go' + \ No newline at end of file From d60063bc51d42894e653d3db2f75f6e17700cdf1 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 4 Jun 2024 10:46:37 -0700 Subject: [PATCH 20/35] fix action --- .github/workflows/readme-check-links.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/readme-check-links.yml b/.github/workflows/readme-check-links.yml index d2fb5a4..5237359 100644 --- a/.github/workflows/readme-check-links.yml +++ b/.github/workflows/readme-check-links.yml @@ -10,5 +10,9 @@ jobs: - uses: gaurav-nelson/github-action-markdown-link-check@v1 with: use-quiet-mode: yes - file-path: './**/*.md, ./LICENSE, ./**/*.go' + file-extension: .md + - uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: yes + file-extension: .go \ No newline at end of file From d8fede72c72e31b53ec98d13f776bf997f07cfa1 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 4 Jun 2024 10:48:09 -0700 Subject: [PATCH 21/35] update name --- .github/workflows/readme-check-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/readme-check-links.yml b/.github/workflows/readme-check-links.yml index 5237359..e5fe33d 100644 --- a/.github/workflows/readme-check-links.yml +++ b/.github/workflows/readme-check-links.yml @@ -1,4 +1,4 @@ -name: Check Markdown links +name: Check Markdown and Help Text Links on: push From 4795b13577e4a3705ea14f325918ac69a2a54cd6 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 4 Jun 2024 12:36:10 -0700 Subject: [PATCH 22/35] add help text and fill out readme --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++-- cmd/create.go | 13 ++++++------- cmd/createIndex.go | 19 ++++++++++++------- cmd/drop.go | 13 ++++++------- cmd/dropIndex.go | 15 ++++++++------- cmd/flags.go | 5 ++++- cmd/list.go | 13 ++++++------- cmd/listIndex.go | 19 ++++++++----------- cmd/root.go | 13 +++++-------- 9 files changed, 98 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 3a61969..f478c12 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,43 @@ -# asvec -Aerospike Vector CLI +# Aerospike Vector Search CLI Tool + +## Overview + +The Aerospike Vector Search (AVS) CLI Tool is designed to simplify the +management of AVS deployments. It offers a features aimed at enhancing +efficiency and productivity for users getting started with vector search. + +## Getting Started + +### Prerequisites + +Ensure you have an AVS instance up and running for `asvec` to connect to. +Checkout out the [AVS documentation](https://aerospike.com/docs/vector) for +options on getting started. + +### Installation + +Download the latest release from [GitHub Releases](https://github.com/aerospike/asvec/releases). + +### Basic Usage + +To verify the installation and view available commands, execute: +```bash +asvec --help +``` + +## Features + +> [!NOTE] +> More features are in the works. Don't worry! + +- **Index Management**: Streamline the management of AVS deployments. + +## Issues + +If you encounter an issue feel free to open a GitHub issue or discussion. +Otherwise, if you are an enterprise customer, please [contact support](https://aerospike.com/support/) + + +## License + +This project is licensed under the MIT License. See the [LICENSE.md](LICENSE.md) file for details. diff --git a/cmd/create.go b/cmd/create.go index 6f5b6d7..aee73bc 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -10,13 +10,12 @@ import ( // createCmd represents the create command var createCmd = &cobra.Command{ Use: "create", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "A parent command for creating resources", + Long: `A parent command for creating resources. It currently only supports creating indexes. + For example: + export ASVEC_HOST=:5000 + asvec create index -i myindex -n test -s testset -f vector-field -d 256 -m COSINE + `, } func init() { diff --git a/cmd/createIndex.go b/cmd/createIndex.go index 53eaed9..cf7f962 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -88,13 +88,18 @@ var createIndexRequiredFlags = []string{ func newCreateIndexCmd() *cobra.Command { return &cobra.Command{ Use: "index", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "A command for creating indexes", + Long: `A command for creating indexes. An index is required to enable vector + search on your data. The index tells AVS where your data is located, + what your vectors look like, and how vectors should be compared to each other. + You can optionally tweak where your index is stored, and how the HNSW algorithm + behaves. + + For example: + export ASVEC_HOST=:5000 + asvec create index -i myindex -n test -s testset -d 256 -m COSINE --vector-field vector \ + --storage-namespace test --hnsw-batch-enabled false + `, PreRunE: func(_ *cobra.Command, _ []string) error { if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) diff --git a/cmd/drop.go b/cmd/drop.go index bf5b956..a5648d9 100644 --- a/cmd/drop.go +++ b/cmd/drop.go @@ -10,13 +10,12 @@ import ( // dropCmd represents the drop command var dropCmd = &cobra.Command{ Use: "drop", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "A parent command for dropping resources", + Long: `A parent command for dropping resources. It currently only supports dropping indexes. + For example: + export ASVEC_HOST=:5000 + asvec drop index -i myindex -n test + `, } func init() { diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index daa389c..1bdb024 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -52,13 +52,14 @@ var dropIndexRequiredFlags = []string{ func newDropIndexCommand() *cobra.Command { return &cobra.Command{ Use: "index", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "A command for dropping indexes", + Long: `A command for dropping indexes. Deleting an index will free up + storage but will also disable vector search on your data. + + For example: + export ASVEC_HOST=:5000 + asvec drop index -i myindex -n test + `, PreRunE: func(_ *cobra.Command, _ []string) error { if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) diff --git a/cmd/flags.go b/cmd/flags.go index 5bfbf33..5b91e94 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -149,7 +149,6 @@ func parseHostPort(v string) (*avs.HostPort, error) { var intPort int64 intPort, err = strconv.ParseInt(match, 0, 0) - if err == nil { host.Port = int(intPort) } @@ -160,6 +159,10 @@ func parseHostPort(v string) (*avs.HostPort, error) { } } + if host.Port == 0 { + host.Port = DefaultPort + } + return host, nil } } diff --git a/cmd/list.go b/cmd/list.go index 8ab8abf..7f649df 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -10,13 +10,12 @@ import ( // listCmd represents the list command var listCmd = &cobra.Command{ Use: "list", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "A parent command for listing resources", + Long: `A parent command for listings resources. It currently only supports listing indexes. + For example: + export ASVEC_HOST=:5000 + asvec list index + `, } func init() { diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 260904d..130080f 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -40,21 +40,18 @@ func newListIndexFlagSet() *pflag.FlagSet { return flagSet } -var listIndexRequiredFlags = []string{ - flagNameNamespace, - flagNameIndexName, -} +var listIndexRequiredFlags = []string{} // listIndexCmd represents the listIndex command func newListIndexCmd() *cobra.Command { return &cobra.Command{Use: "index", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "A command for listing indexes", + Long: fmt.Sprintf(`A command for displaying useful information about AVS indexes. To display additional + index information use the --%s flag. + For example: + export ASVEC_HOST=:5000 + asvec list index + `, flagNameVerbose), PreRunE: func(_ *cobra.Command, _ []string) error { if viper.IsSet(flagNameSeeds) && viper.IsSet(flagNameHost) { return fmt.Errorf("only --%s or --%s allowed", flagNameSeeds, flagNameHost) diff --git a/cmd/root.go b/cmd/root.go index 965c5ff..049d46e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,13 +26,11 @@ var rootFlags = &struct { // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "asvec", - Short: "A brief description of your application", - Long: `A longer description that spans multiple lines and likely contains -examples and usage of using your application. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "Aerospike Vector Search CLI", + Long: `Welcome to the AVS Deployment Manager CLI Tool! + To start using this tool, please consult the detailed documentation available at https://aerospike.com/docs/vector. + Should you encounter any issues or have questions, feel free to report them via GitHub issues. + Enterprise customers requiring support should contact Aerospike Support directly at https://aerospike.com/support.`, PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { if rootFlags.logLevel.NotSet() { lvl.Set(slog.LevelError + 1) // disable all logging @@ -73,7 +71,6 @@ to quickly create a Cobra application.`, }) return persistedErr - }, } From 260145e25e3e7454f99002910f70d103115dcb39 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 5 Jun 2024 12:15:10 -0700 Subject: [PATCH 23/35] separate out flags into own module, add flags tests --- .gitignore | 3 +- cmd/cmd_test.go | 142 ++++++++++++ cmd/constants.go | 25 +++ cmd/createIndex.go | 45 ++-- cmd/dropIndex.go | 11 +- cmd/flags.go | 362 ------------------------------- cmd/flags/distanceMetric.go | 40 ++++ cmd/flags/distanceMetric_test.go | 79 +++++++ cmd/flags/hostPort.go | 176 +++++++++++++++ cmd/flags/hostPort_test.go | 301 +++++++++++++++++++++++++ cmd/flags/logLevel.go | 53 +++++ cmd/flags/logLevel_test.go | 44 ++++ cmd/flags/optionals.go | 73 +++++++ cmd/flags/optionals_test.go | 73 +++++++ cmd/listIndex.go | 11 +- cmd/root.go | 5 +- cmd/utils.go | 26 +++ go.mod | 28 ++- go.sum | 82 +++++-- 19 files changed, 1167 insertions(+), 412 deletions(-) create mode 100644 cmd/cmd_test.go create mode 100644 cmd/constants.go delete mode 100644 cmd/flags.go create mode 100644 cmd/flags/distanceMetric.go create mode 100644 cmd/flags/distanceMetric_test.go create mode 100644 cmd/flags/hostPort.go create mode 100644 cmd/flags/hostPort_test.go create mode 100644 cmd/flags/logLevel.go create mode 100644 cmd/flags/logLevel_test.go create mode 100644 cmd/flags/optionals.go create mode 100644 cmd/flags/optionals_test.go create mode 100644 cmd/utils.go diff --git a/.gitignore b/.gitignore index 2655a7e..ff43f2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /docker/* /bin/* -embed_*.go \ No newline at end of file +embed_*.go +/tmp \ No newline at end of file diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go new file mode 100644 index 0000000..d338b64 --- /dev/null +++ b/cmd/cmd_test.go @@ -0,0 +1,142 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "strconv" + "strings" + "testing" + "time" + + "github.com/aerospike/tools-common-go/testutils" + "github.com/stretchr/testify/suite" +) + +var wd, _ = os.Getwd() + +type CmdTestSuite struct { + suite.Suite + app string + coverFile string + coverFileCounter int + host string + port int +} + +func TestDistanceMetricFlagSuite(t *testing.T) { + suite.Run(t, new(CmdTestSuite)) +} + +func (suite *CmdTestSuite) SetupSuite() { + suite.app = path.Join(wd, "app.test") + suite.coverFile = path.Join(wd, "../coverage/cmd-coverage.cov") + suite.coverFileCounter = 0 + suite.host = "127.0.0.1" + suite.port = 10000 + + err := docker_compose_up() + if err != nil { + suite.FailNowf("unable to start docker compose up", "%v", err) + } + + os.Chdir("..") + goArgs := []string{"test", "-coverpkg", "./...", "-tags=integration", "-o", suite.app} + + // Compile test binary + compileCmd := exec.Command("go", goArgs...) + err = compileCmd.Run() + suite.Assert().NoError(err) +} + +func (suite *CmdTestSuite) TearDownSuite() { + docker_compose_down() + err := os.Remove(suite.app) + suite.Assert().NoError(err) + time.Sleep(time.Second * 5) + err = testutils.Stop() + suite.Assert().NoError(err) +} + +func (suite *CmdTestSuite) runCmd(asvecCmd ...string) ([]string, error) { + strs := strings.Split(suite.coverFile, ".") + file := strs[len(strs)-2] + "-" + strconv.Itoa(suite.coverFileCounter) + "." + strs[len(strs)-1] + suite.coverFileCounter++ + var args []string + args = []string{"-test.coverprofile=" + file} + args = append(args, asvecCmd...) + + cmd := exec.Command(suite.app, args...) + stdout, err := cmd.Output() + // fmt.Printf("stdout: %v", string(stdout)) + + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + return []string{string(ee.Stderr)}, err + } + return []string{string(stdout)}, err + } + + lines := strings.Split(string(stdout), "\n") + + return lines, nil +} + +func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { + testCases := []struct { + name string + cmd string + }{ + { + "", + "", + }, + } + + for _, tc := range testCases { + suite.Run(tc.name, func() { + lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) + suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + }) + } +} + +func docker_compose_up() error { + fmt.Println("Starting docker containers") + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + // docker/docker-compose.yml + cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-f /../../docker/docker-compose.yml"), "up", "-d") + output, err := cmd.CombinedOutput() + + fmt.Printf("docker compose up output: %s\n", string(output)) + + if err != nil { + if _, ok := err.(*exec.ExitError); ok { + return err + } + return err + } + + return nil +} + +func docker_compose_down() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-f /../../docker/docker-compose.yml"), "down", "-d") + _, err := cmd.Output() + + if err != nil { + if _, ok := err.(*exec.ExitError); ok { + return err + } + return err + } + + return nil +} diff --git a/cmd/constants.go b/cmd/constants.go new file mode 100644 index 0000000..0b6ee36 --- /dev/null +++ b/cmd/constants.go @@ -0,0 +1,25 @@ +package cmd + +const ( + logLevelFlagName = "log-level" + flagNameSeeds = "seeds" + flagNameHost = "host" + flagNameListenerName = "listener-name" + flagNameNamespace = "namespace" + flagNameSets = "sets" + flagNameIndexName = "index-name" + flagNameVectorField = "vector-field" + flagNameDimension = "dimension" + flagNameDistanceMetric = "distance-metric" + flagNameIndexMeta = "index-meta" + flagNameTimeout = "timeout" + flagNameVerbose = "verbose" + flagNameStorageNamespace = "storage-namespace" + flagNameStorageSet = "storage-set" + flagNameMaxEdges = "hnsw-max-edges" + flagNameConstructionEf = "hnsw-ef-construction" + flagNameEf = "hnsw-ef" + flagNameBatchMaxRecords = "hnsw-batch-max-records" + flagNameBatchInterval = "hnsw-batch-interval" + flagNameBatchEnabled = "hnsw-batch-enabled" +) diff --git a/cmd/createIndex.go b/cmd/createIndex.go index cf7f962..fe388a9 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -4,6 +4,7 @@ Copyright © 2024 NAME HERE package cmd import ( + "asvec/cmd/flags" "context" "fmt" "log/slog" @@ -19,36 +20,36 @@ import ( //nolint:govet // Padding not a concern for a CLI var createIndexFlags = &struct { - host *HostPortFlag - seeds *SeedsSliceFlag - listenerName StringOptionalFlag + host *flags.HostPortFlag + seeds *flags.SeedsSliceFlag + listenerName flags.StringOptionalFlag namespace string sets []string indexName string vectorField string dimensions uint32 - distanceMetric DistanceMetricFlag + distanceMetric flags.DistanceMetricFlag indexMeta map[string]string - storageNamespace StringOptionalFlag - storageSet StringOptionalFlag - hnswMaxEdges Uint32OptionalFlag - hnswEf Uint32OptionalFlag - hnswConstructionEf Uint32OptionalFlag - hnswBatchMaxRecords Uint32OptionalFlag - hnswBatchInterval Uint32OptionalFlag - hnswBatchEnabled BoolOptionalFlag + storageNamespace flags.StringOptionalFlag + storageSet flags.StringOptionalFlag + hnswMaxEdges flags.Uint32OptionalFlag + hnswEf flags.Uint32OptionalFlag + hnswConstructionEf flags.Uint32OptionalFlag + hnswBatchMaxRecords flags.Uint32OptionalFlag + hnswBatchInterval flags.Uint32OptionalFlag + hnswBatchEnabled flags.BoolOptionalFlag timeout time.Duration }{ - host: NewDefaultHostPortFlag(), - seeds: &SeedsSliceFlag{}, - storageNamespace: StringOptionalFlag{}, - storageSet: StringOptionalFlag{}, - hnswMaxEdges: Uint32OptionalFlag{}, - hnswEf: Uint32OptionalFlag{}, - hnswConstructionEf: Uint32OptionalFlag{}, - hnswBatchMaxRecords: Uint32OptionalFlag{}, - hnswBatchInterval: Uint32OptionalFlag{}, - hnswBatchEnabled: BoolOptionalFlag{}, + host: flags.NewDefaultHostPortFlag(), + seeds: &flags.SeedsSliceFlag{}, + storageNamespace: flags.StringOptionalFlag{}, + storageSet: flags.StringOptionalFlag{}, + hnswMaxEdges: flags.Uint32OptionalFlag{}, + hnswEf: flags.Uint32OptionalFlag{}, + hnswConstructionEf: flags.Uint32OptionalFlag{}, + hnswBatchMaxRecords: flags.Uint32OptionalFlag{}, + hnswBatchInterval: flags.Uint32OptionalFlag{}, + hnswBatchEnabled: flags.BoolOptionalFlag{}, } func newCreateIndexFlagSet() *pflag.FlagSet { diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index 1bdb024..39907df 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -4,6 +4,7 @@ Copyright © 2024 NAME HERE package cmd import ( + "asvec/cmd/flags" "context" "fmt" "log/slog" @@ -18,16 +19,16 @@ import ( //nolint:govet // Padding not a concern for a CLI var dropIndexFlags = &struct { - host *HostPortFlag - seeds *SeedsSliceFlag - listenerName StringOptionalFlag + host *flags.HostPortFlag + seeds *flags.SeedsSliceFlag + listenerName flags.StringOptionalFlag namespace string sets []string indexName string timeout time.Duration }{ - host: NewDefaultHostPortFlag(), - seeds: &SeedsSliceFlag{}, + host: flags.NewDefaultHostPortFlag(), + seeds: &flags.SeedsSliceFlag{}, } func newDropIndexFlagSet() *pflag.FlagSet { diff --git a/cmd/flags.go b/cmd/flags.go deleted file mode 100644 index 5b91e94..0000000 --- a/cmd/flags.go +++ /dev/null @@ -1,362 +0,0 @@ -package cmd - -import ( - "fmt" - "regexp" - "strconv" - "strings" - - avs "github.com/aerospike/aerospike-proximus-client-go" - - "github.com/aerospike/aerospike-proximus-client-go/protos" -) - -const ( - logLevelFlagName = "log-level" - flagNameSeeds = "seeds" - flagNameHost = "host" - flagNameListenerName = "listener-name" - flagNameNamespace = "namespace" - flagNameSets = "sets" - flagNameIndexName = "index-name" - flagNameVectorField = "vector-field" - flagNameDimension = "dimension" - flagNameDistanceMetric = "distance-metric" - flagNameIndexMeta = "index-meta" - flagNameTimeout = "timeout" - flagNameVerbose = "verbose" - flagNameStorageNamespace = "storage-namespace" - flagNameStorageSet = "storage-set" - flagNameMaxEdges = "hnsw-max-edges" - flagNameConstructionEf = "hnsw-ef-construction" - flagNameEf = "hnsw-ef" - flagNameBatchMaxRecords = "hnsw-batch-max-records" - flagNameBatchInterval = "hnsw-batch-interval" - flagNameBatchEnabled = "hnsw-batch-enabled" -) - -type DistanceMetricFlag string - -// This is just a set of valid VectorDistanceMetrics. The value does not have meaning -var distanceMetricSet = protos.VectorDistanceMetric_value - -func (f *DistanceMetricFlag) Set(val string) error { - val = strings.ToUpper(val) - if _, ok := distanceMetricSet[val]; ok { - *f = DistanceMetricFlag(val) - return nil - } - - return fmt.Errorf("unrecognized distance metric") -} - -func (f *DistanceMetricFlag) Type() string { - names := []string{} - - for key := range distanceMetricSet { - names = append(names, key) - } - - return strings.Join(names, ",") -} - -func (f *DistanceMetricFlag) String() string { - return string(*f) -} - -type LogLevelFlag string - -var logLevelSet = map[string]struct{}{ - "DEBUG": {}, - "INFO": {}, - "WARN": {}, - "ERROR": {}, -} - -func (f *LogLevelFlag) NotSet() bool { - return *f == "" -} - -func (f *LogLevelFlag) Set(val string) error { - if val == "" { - *f = LogLevelFlag("") - return nil - } - - val = strings.ToUpper(val) - if _, ok := logLevelSet[val]; ok { - *f = LogLevelFlag(val) - return nil - } - - return fmt.Errorf("unrecognized log level") -} - -func (f *LogLevelFlag) Type() string { - names := []string{} - - for key := range logLevelSet { - names = append(names, key) - } - - return strings.Join(names, ",") -} - -func (f *LogLevelFlag) String() string { - return string(*f) -} - -const ( - DefaultIPv4 = "127.0.0.1" - DefaultPort = 5000 -) - -func parseHostPort(v string) (*avs.HostPort, error) { - host := &avs.HostPort{} - ipv6HostPattern := `^\[(?P.*)\]` - hostPattern := `^(?P[^:]*)` // matched ipv4 and hostname - portPattern := `(?P\d+)$` - reIPv6Host := regexp.MustCompile(fmt.Sprintf("%s$", ipv6HostPattern)) - reIPv6HostPort := regexp.MustCompile(fmt.Sprintf("%s:%s", ipv6HostPattern, portPattern)) - reIPv4Host := regexp.MustCompile(fmt.Sprintf("%s$", hostPattern)) - reIPv4HostPort := regexp.MustCompile(fmt.Sprintf("%s:%s", hostPattern, portPattern)) - - regexsAndNames := []struct { - regex *regexp.Regexp - groupNames []string - }{ - // The order is important since the ipv4 pattern also matches ipv6 - {reIPv6HostPort, reIPv6HostPort.SubexpNames()}, - {reIPv6Host, reIPv6Host.SubexpNames()}, - {reIPv4HostPort, reIPv4HostPort.SubexpNames()}, - {reIPv4Host, reIPv4Host.SubexpNames()}, - } - - for _, r := range regexsAndNames { - regex := r.regex - groupNames := r.groupNames - - if matchs := regex.FindStringSubmatch(v); matchs != nil { - for idx, match := range matchs { - var err error - - name := groupNames[idx] - - switch { - case name == "host": - host.Host = match - case name == "port": - var intPort int64 - - intPort, err = strconv.ParseInt(match, 0, 0) - if err == nil { - host.Port = int(intPort) - } - } - - if err != nil { - return host, fmt.Errorf("failed to parse %s : %s", name, err) - } - } - - if host.Port == 0 { - host.Port = DefaultPort - } - - return host, nil - } - } - - return host, fmt.Errorf("does not match any expected formats") -} - -// A cobra PFlag to parse and display help info for the host[:tls-name][:port] -// input option. It implements the pflag Value and SliceValue interfaces to -// enable automatic parsing by cobra. -type HostPortFlag struct { - HostPort avs.HostPort -} - -func NewDefaultHostPortFlag() *HostPortFlag { - return &HostPortFlag{ - HostPort: avs.HostPort{ - Host: DefaultIPv4, - Port: DefaultPort, - }, - } -} - -func (hp *HostPortFlag) Set(val string) error { - hostPort, err := parseHostPort(val) - if err != nil { - return err - } - - hp.HostPort = *hostPort - - return nil -} - -func (hp *HostPortFlag) Type() string { - return "host[:port]" -} - -func (hp *HostPortFlag) String() string { - return hp.HostPort.String() -} - -type SeedsSliceFlag struct { - Seeds avs.HostPortSlice -} - -func NewSeedsSliceFlag() SeedsSliceFlag { - return SeedsSliceFlag{ - Seeds: avs.HostPortSlice{}, - } -} - -// Append adds the specified value to the end of the flag value list. -func (slice *SeedsSliceFlag) Append(val string) error { - host, err := parseHostPort(val) - - if err != nil { - return err - } - - slice.Seeds = append(slice.Seeds, host) - - return nil -} - -// Replace will fully overwrite any data currently in the flag value list. -func (slice *SeedsSliceFlag) Replace(vals []string) error { - slice.Seeds = avs.HostPortSlice{} - - for _, val := range vals { - if err := slice.Append(val); err != nil { - return err - } - } - - return nil -} - -// GetSlice returns the flag value list as an array of strings. -func (slice *SeedsSliceFlag) GetSlice() []string { - strs := []string{} - - for _, elem := range slice.Seeds { - strs = append(strs, elem.String()) - } - - return strs -} - -func (slice *SeedsSliceFlag) Set(commaSepVal string) error { - vals := strings.Split(commaSepVal, ",") - - for _, val := range vals { - if err := slice.Append(val); err != nil { - return err - } - } - - return nil -} - -func (slice *SeedsSliceFlag) Type() string { - return "seed[:port][,...]" -} - -func (slice *SeedsSliceFlag) String() string { - return slice.Seeds.String() -} - -func parseBothHostSeedsFlag(seeds SeedsSliceFlag, host HostPortFlag) (avs.HostPortSlice, bool) { - isLoadBalancer := false - hosts := avs.HostPortSlice{} - - if len(seeds.Seeds) > 0 { - logger.Debug("seeds is set") - - hosts = append(hosts, seeds.Seeds...) - } else { - logger.Debug("hosts is set") - - isLoadBalancer = true - - hosts = append(hosts, &host.HostPort) - } - - return hosts, isLoadBalancer -} - -const optionalEmptyString = "" - -type StringOptionalFlag struct { - Val *string -} - -func (f *StringOptionalFlag) Set(val string) error { - f.Val = &val - return nil -} - -func (f *StringOptionalFlag) Type() string { - return "string" -} - -func (f *StringOptionalFlag) String() string { - if f.Val != nil { - return *f.Val - } - - return optionalEmptyString -} - -type Uint32OptionalFlag struct { - Val *uint32 -} - -func (f *Uint32OptionalFlag) Set(val string) error { - v, err := strconv.ParseUint(val, 0, 32) - u32Val := uint32(v) - f.Val = &u32Val - - return err -} - -func (f *Uint32OptionalFlag) Type() string { - return "uint32" -} - -func (f *Uint32OptionalFlag) String() string { - if f.Val != nil { - return strconv.FormatUint(uint64(*f.Val), 10) - } - - return optionalEmptyString -} - -type BoolOptionalFlag struct { - Val *bool -} - -func (f *BoolOptionalFlag) Set(val string) error { - v, err := strconv.ParseBool(val) - f.Val = &v - - return err -} - -func (f *BoolOptionalFlag) Type() string { - return "bool" -} - -func (f *BoolOptionalFlag) String() string { - if f.Val != nil { - return strconv.FormatBool(*f.Val) - } - - return optionalEmptyString -} diff --git a/cmd/flags/distanceMetric.go b/cmd/flags/distanceMetric.go new file mode 100644 index 0000000..2f07e5e --- /dev/null +++ b/cmd/flags/distanceMetric.go @@ -0,0 +1,40 @@ +package flags + +import ( + "fmt" + "slices" + "strings" + + "github.com/aerospike/aerospike-proximus-client-go/protos" +) + +type DistanceMetricFlag string + +// This is just a set of valid VectorDistanceMetrics. The value does not have meaning +var distanceMetricSet = protos.VectorDistanceMetric_value + +func (f *DistanceMetricFlag) Set(val string) error { + val = strings.ToUpper(val) + if _, ok := distanceMetricSet[val]; ok { + *f = DistanceMetricFlag(val) + return nil + } + + return fmt.Errorf("unrecognized distance metric") +} + +func (f *DistanceMetricFlag) Type() string { + names := []string{} + + for key := range distanceMetricSet { + names = append(names, key) + } + + slices.Sort(names) + + return strings.Join(names, ",") +} + +func (f *DistanceMetricFlag) String() string { + return string(*f) +} diff --git a/cmd/flags/distanceMetric_test.go b/cmd/flags/distanceMetric_test.go new file mode 100644 index 0000000..2f1ec1a --- /dev/null +++ b/cmd/flags/distanceMetric_test.go @@ -0,0 +1,79 @@ +package flags + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type DistanceMetricFlagTestSuite struct { + suite.Suite +} + +func (suite *DistanceMetricFlagTestSuite) TestSet() { + tests := []struct { + input string + expect_err bool + expected DistanceMetricFlag + }{ + { + input: "SQUARED_EUCLIDEAN", + expected: DistanceMetricFlag("SQUARED_EUCLIDEAN"), + }, + { + input: "cosine", + expected: DistanceMetricFlag("COSINE"), + }, + { + input: "manhattan", + expected: DistanceMetricFlag("MANHATTAN"), + }, + { + input: "dot_product", + expected: DistanceMetricFlag("DOT_PRODUCT"), + }, + { + input: "HAMming", + expected: DistanceMetricFlag("HAMMING"), + }, + { + input: "unknown", + expect_err: true, + // You can set the expected value to nil or any other appropriate value + expected: DistanceMetricFlag(""), + }, + } + + for _, test := range tests { + suite.Run(test.input, func() { + flag := DistanceMetricFlag("") + err := flag.Set(test.input) + if test.expect_err { + suite.Error(err) + } else { + suite.NoError(err) + suite.Equal(test.expected, flag) + } + }) + } +} + +func (suite *DistanceMetricFlagTestSuite) TestType() { + flag := DistanceMetricFlag("") + suite.Equal("COSINE,DOT_PRODUCT,HAMMING,MANHATTAN,SQUARED_EUCLIDEAN", flag.Type()) +} + +func (suite *DistanceMetricFlagTestSuite) TestString() { + flag := DistanceMetricFlag("euclidean") + suite.Equal("euclidean", flag.String()) + + flag = DistanceMetricFlag("cosine") + suite.Equal("cosine", flag.String()) + + flag = DistanceMetricFlag("manhattan") + suite.Equal("manhattan", flag.String()) +} + +func TestDistanceMetricFlagSuite(t *testing.T) { + suite.Run(t, new(DistanceMetricFlagTestSuite)) +} diff --git a/cmd/flags/hostPort.go b/cmd/flags/hostPort.go new file mode 100644 index 0000000..638dadf --- /dev/null +++ b/cmd/flags/hostPort.go @@ -0,0 +1,176 @@ +package flags + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + avs "github.com/aerospike/aerospike-proximus-client-go" +) + +const ( + DefaultIPv4 = "127.0.0.1" + DefaultPort = 5000 +) + +func parseHostPort(v string) (*avs.HostPort, error) { + host := &avs.HostPort{} + ipv6HostPattern := `^\[(?P.*)\]` + hostPattern := `^(?P[^:]*)` // matched ipv4 and hostname + portPattern := `(?P\d+)$` + reIPv6Host := regexp.MustCompile(fmt.Sprintf("%s$", ipv6HostPattern)) + reIPv6HostPort := regexp.MustCompile(fmt.Sprintf("%s:%s", ipv6HostPattern, portPattern)) + reIPv4Host := regexp.MustCompile(fmt.Sprintf("%s$", hostPattern)) + reIPv4HostPort := regexp.MustCompile(fmt.Sprintf("%s:%s", hostPattern, portPattern)) + + regexsAndNames := []struct { + regex *regexp.Regexp + groupNames []string + }{ + // The order is important since the ipv4 pattern also matches ipv6 + {reIPv6HostPort, reIPv6HostPort.SubexpNames()}, + {reIPv6Host, reIPv6Host.SubexpNames()}, + {reIPv4HostPort, reIPv4HostPort.SubexpNames()}, + {reIPv4Host, reIPv4Host.SubexpNames()}, + } + + for _, r := range regexsAndNames { + regex := r.regex + groupNames := r.groupNames + + if matchs := regex.FindStringSubmatch(v); matchs != nil { + for idx, match := range matchs { + var err error + + name := groupNames[idx] + + switch { + case name == "host": + host.Host = match + case name == "port": + var intPort int64 + + intPort, err = strconv.ParseInt(match, 0, 0) + if err == nil { + host.Port = int(intPort) + } + } + + if err != nil { + return host, fmt.Errorf("failed to parse %s : %s", name, err) + } + } + + if host.Port == 0 { + host.Port = DefaultPort + } + + return host, nil + } + } + + return host, fmt.Errorf("does not match any expected formats") +} + +// A cobra PFlag to parse and display help info for the host[:tls-name][:port] +// input option. It implements the pflag Value and SliceValue interfaces to +// enable automatic parsing by cobra. +type HostPortFlag struct { + HostPort avs.HostPort +} + +func NewDefaultHostPortFlag() *HostPortFlag { + return &HostPortFlag{ + HostPort: avs.HostPort{ + Host: DefaultIPv4, + Port: DefaultPort, + }, + } +} + +func (hp *HostPortFlag) Set(val string) error { + hostPort, err := parseHostPort(val) + if err != nil { + return err + } + + hp.HostPort = *hostPort + + return nil +} + +func (hp *HostPortFlag) Type() string { + return "host[:port]" +} + +func (hp *HostPortFlag) String() string { + return hp.HostPort.String() +} + +type SeedsSliceFlag struct { + Seeds avs.HostPortSlice +} + +func NewSeedsSliceFlag() SeedsSliceFlag { + return SeedsSliceFlag{ + Seeds: avs.HostPortSlice{}, + } +} + +// Append adds the specified value to the end of the flag value list. +func (slice *SeedsSliceFlag) Append(val string) error { + host, err := parseHostPort(val) + + if err != nil { + return err + } + + slice.Seeds = append(slice.Seeds, host) + + return nil +} + +// Replace will fully overwrite any data currently in the flag value list. +func (slice *SeedsSliceFlag) Replace(vals []string) error { + slice.Seeds = avs.HostPortSlice{} + + for _, val := range vals { + if err := slice.Append(val); err != nil { + return err + } + } + + return nil +} + +// GetSlice returns the flag value list as an array of strings. +func (slice *SeedsSliceFlag) GetSlice() []string { + strs := []string{} + + for _, elem := range slice.Seeds { + strs = append(strs, elem.String()) + } + + return strs +} + +func (slice *SeedsSliceFlag) Set(commaSepVal string) error { + vals := strings.Split(commaSepVal, ",") + + for _, val := range vals { + if err := slice.Append(val); err != nil { + return err + } + } + + return nil +} + +func (slice *SeedsSliceFlag) Type() string { + return "seed[:port][,...]" +} + +func (slice *SeedsSliceFlag) String() string { + return slice.Seeds.String() +} diff --git a/cmd/flags/hostPort_test.go b/cmd/flags/hostPort_test.go new file mode 100644 index 0000000..a02d02a --- /dev/null +++ b/cmd/flags/hostPort_test.go @@ -0,0 +1,301 @@ +package flags + +import ( + "testing" + + avs "github.com/aerospike/aerospike-proximus-client-go" + + "github.com/stretchr/testify/suite" +) + +type HostPortFlagTestSuite struct { + suite.Suite +} + +func (suite *HostPortFlagTestSuite) TestHostPortSetGet() { + testCases := []struct { + input string + output *HostPortFlag + expect_err bool + expected_str string + }{ + { + "127.0.0.1", + &HostPortFlag{ + HostPort: avs.HostPort{ + Host: "127.0.0.1", + Port: 5000, + }, + }, + false, + "127.0.0.1:5000", + }, + { + "127.0.0.2:3002", + &HostPortFlag{ + HostPort: avs.HostPort{ + Host: "127.0.0.2", + Port: 3002, + }, + }, + false, + "127.0.0.2:3002", + }, + { + "127.0.0.2:3002:", + nil, + true, + "", + }, + } + + for _, tc := range testCases { + suite.T().Run(tc.input, func(t *testing.T) { + actual := NewDefaultHostPortFlag() + + if tc.expect_err { + suite.Error(actual.Set(tc.input)) + } else { + suite.NoError(actual.Set(tc.input)) + suite.Equal(tc.output, actual) + suite.Equal(tc.expected_str, actual.String()) + } + }) + } +} + +func (suite *HostPortFlagTestSuite) TestSeedsSliceSetGet() { + testCases := []struct { + input string + output SeedsSliceFlag + slice []string + }{ + { + "127.0.0.1", + SeedsSliceFlag{ + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.1", + Port: 5000, + }, + }, + }, + []string{"127.0.0.1:5000"}, + }, + { + "127.0.0.1,127.0.0.2", + SeedsSliceFlag{ + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.1", + Port: 5000, + }, + { + Host: "127.0.0.2", + Port: 5000, + }, + }, + }, + []string{"127.0.0.1:5000", "127.0.0.2:5000"}, + }, + { + "127.0.0.2:3002", + SeedsSliceFlag{ + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.2", + Port: 3002, + }, + }, + }, + []string{"127.0.0.2:3002"}, + }, + { + "127.0.0.2:3002,127.0.0.3:3003", + SeedsSliceFlag{ + + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.2", + Port: 3002, + }, + { + Host: "127.0.0.3", + Port: 3003, + }, + }, + }, + []string{"127.0.0.2:3002", "127.0.0.3:3003"}, + }, + { + "127.0.0.3:3003", + SeedsSliceFlag{ + + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.3", + + Port: 3003, + }, + }, + }, + []string{"127.0.0.3:3003"}, + }, + { + "127.0.0.3:3003,127.0.0.4:3004", + SeedsSliceFlag{ + + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.3", + Port: 3003, + }, + { + Host: "127.0.0.4", + Port: 3004, + }, + }, + }, + []string{"127.0.0.3:3003", "127.0.0.4:3004"}, + }, + { + "127.0.0.3:3003,127.0.0.4:3004", + SeedsSliceFlag{ + + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.3", + Port: 3003, + }, + { + Host: "127.0.0.4", + Port: 3004, + }, + }, + }, + []string{"127.0.0.3:3003", "127.0.0.4:3004"}, + }, + // Not supported yet, so not testing + // { + // "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + // SeedsSliceFlag{ + + // Seeds: avs.HostPortSlice{ + // { + // Host: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + // }, + // }, + // }, + // []string{"2001:0db8:85a3:0000:0000:8a2e:0370:7334"}, + // }, + // { + // "[fe80::1ff:fe23:4567:890a]:3002", + // SeedsSliceFlag{ + + // Seeds: avs.HostPortSlice{ + // { + // Host: "fe80::1ff:fe23:4567:890a", + // Port: 3002, + // }, + // }, + // }, + // []string{"fe80::1ff:fe23:4567:890a:3002"}, + // }, + // { + // "[100::]:3003", + // SeedsSliceFlag{ + + // Seeds: avs.HostPortSlice{ + // { + // Host: "100::", + + // Port: 3003, + // }, + // }, + // }, + // []string{"100:::3003"}, + // }, + } + + for _, tc := range testCases { + suite.T().Run(tc.input, func(t *testing.T) { + actual := NewSeedsSliceFlag() + + suite.NoError(actual.Set(tc.input)) + suite.Equal(tc.output, actual) + suite.Equal(tc.slice, actual.GetSlice()) + }) + } +} + +func (suite *HostPortFlagTestSuite) TestHostPortAppend() { + testCases := []struct { + input string + append string + output SeedsSliceFlag + }{ + { + "127.0.0.1", + "127.0.0.2", + SeedsSliceFlag{ + + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.1", + Port: 5000, + }, + { + Host: "127.0.0.2", + Port: 5000, + }, + }, + }, + }, + } + + for _, tc := range testCases { + suite.T().Run(tc.input, func(t *testing.T) { + actual := NewSeedsSliceFlag() + + suite.NoError(actual.Set(tc.input)) + suite.NoError(actual.Set(tc.append)) + suite.Equal(tc.output, actual) + }) + } +} + +func (suite *HostPortFlagTestSuite) TestHostPortString() { + testCases := []struct { + input SeedsSliceFlag + output string + }{ + { + SeedsSliceFlag{ + + Seeds: avs.HostPortSlice{ + { + Host: "127.0.0.1", + Port: 3000, + }, + { + Host: "127.0.0.2", + Port: 3002, + }, + }, + }, + "[127.0.0.1:3000, 127.0.0.2:3002]", + }, + } + + for _, tc := range testCases { + suite.T().Run(tc.output, func(t *testing.T) { + suite.Equal(tc.output, tc.input.String()) + }) + } +} + +// In order for 'go test' to run this suite, we need to create +// a normal test function and pass our suite to suite.Run +func TestRunHostTestSuite(t *testing.T) { + suite.Run(t, new(HostPortFlagTestSuite)) +} diff --git a/cmd/flags/logLevel.go b/cmd/flags/logLevel.go new file mode 100644 index 0000000..72cb804 --- /dev/null +++ b/cmd/flags/logLevel.go @@ -0,0 +1,53 @@ +package flags + +import ( + "fmt" + "sort" + "strings" +) + +type LogLevelFlag string + +var logLevelSet = map[string]int{ + "DEBUG": 0, + "INFO": 1, + "WARN": 2, + "ERROR": 3, +} + +func (f *LogLevelFlag) NotSet() bool { + return *f == "" +} + +func (f *LogLevelFlag) Set(val string) error { + if val == "" { + *f = LogLevelFlag("") + return nil + } + + val = strings.ToUpper(val) + if _, ok := logLevelSet[val]; ok { + *f = LogLevelFlag(val) + return nil + } + + return fmt.Errorf("unrecognized log level") +} + +func (f *LogLevelFlag) Type() string { + names := []string{} + + for key := range logLevelSet { + names = append(names, key) + } + + sort.Slice(names, func(i, j int) bool { + return logLevelSet[names[i]] < logLevelSet[names[j]] + }) + + return strings.Join(names, ",") +} + +func (f *LogLevelFlag) String() string { + return string(*f) +} diff --git a/cmd/flags/logLevel_test.go b/cmd/flags/logLevel_test.go new file mode 100644 index 0000000..dd134c6 --- /dev/null +++ b/cmd/flags/logLevel_test.go @@ -0,0 +1,44 @@ +package flags + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type LogLevelSuite struct { + suite.Suite +} + +func TestLogLevelSuite(t *testing.T) { + suite.Run(t, new(LogLevelSuite)) +} + +func (suite *LogLevelSuite) TestNotSet() { + flag := LogLevelFlag("") + suite.True(flag.NotSet()) + + flag = LogLevelFlag("DEBUG") + suite.False(flag.NotSet()) +} + +func (suite *LogLevelSuite) TestSet() { + flag := LogLevelFlag("") + err := flag.Set("DEBUG") + suite.NoError(err) + suite.Equal(LogLevelFlag("DEBUG"), flag) + + err = flag.Set("INVALID") + suite.Error(err) + suite.Equal(LogLevelFlag("DEBUG"), flag) +} + +func (suite *LogLevelSuite) TestType() { + flag := LogLevelFlag("") + suite.Equal("DEBUG,INFO,WARN,ERROR", flag.Type()) +} + +func (suite *LogLevelSuite) TestString() { + flag := LogLevelFlag("DEBUG") + suite.Equal("DEBUG", flag.String()) +} diff --git a/cmd/flags/optionals.go b/cmd/flags/optionals.go new file mode 100644 index 0000000..5168134 --- /dev/null +++ b/cmd/flags/optionals.go @@ -0,0 +1,73 @@ +package flags + +import "strconv" + +const optionalEmptyString = "" + +type StringOptionalFlag struct { + Val *string +} + +func (f *StringOptionalFlag) Set(val string) error { + f.Val = &val + return nil +} + +func (f *StringOptionalFlag) Type() string { + return "string" +} + +func (f *StringOptionalFlag) String() string { + if f.Val != nil { + return *f.Val + } + + return optionalEmptyString +} + +type Uint32OptionalFlag struct { + Val *uint32 +} + +func (f *Uint32OptionalFlag) Set(val string) error { + v, err := strconv.ParseUint(val, 0, 32) + u32Val := uint32(v) + f.Val = &u32Val + + return err +} + +func (f *Uint32OptionalFlag) Type() string { + return "uint32" +} + +func (f *Uint32OptionalFlag) String() string { + if f.Val != nil { + return strconv.FormatUint(uint64(*f.Val), 10) + } + + return optionalEmptyString +} + +type BoolOptionalFlag struct { + Val *bool +} + +func (f *BoolOptionalFlag) Set(val string) error { + v, err := strconv.ParseBool(val) + f.Val = &v + + return err +} + +func (f *BoolOptionalFlag) Type() string { + return "bool" +} + +func (f *BoolOptionalFlag) String() string { + if f.Val != nil { + return strconv.FormatBool(*f.Val) + } + + return optionalEmptyString +} diff --git a/cmd/flags/optionals_test.go b/cmd/flags/optionals_test.go new file mode 100644 index 0000000..e21a715 --- /dev/null +++ b/cmd/flags/optionals_test.go @@ -0,0 +1,73 @@ +package flags + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type OptionalFlagSuite struct { + suite.Suite +} + +func TestOptionalFlagSuite(t *testing.T) { + suite.Run(t, new(OptionalFlagSuite)) +} + +func (suite *OptionalFlagSuite) TestBoolOptionalFlag() { + f := &BoolOptionalFlag{} + + err := f.Set("true") + if err != nil { + suite.T().Errorf("Unexpected error: %v", err) + } + + if f.Val == nil || *f.Val != true { + suite.T().Errorf("Expected true, got %v", f.Val) + } + + err = f.Set("not a bool") + if err == nil { + suite.T().Errorf("Expected error, got nil") + } +} + +func (suite *OptionalFlagSuite) TestStringOptionalFlag() { + f := &StringOptionalFlag{} + + err := f.Set("hello") + if err != nil { + suite.T().Errorf("Unexpected error: %v", err) + } + + if f.Val == nil || *f.Val != "hello" { + suite.T().Errorf("Expected 'hello', got %v", f.Val) + } + + err = f.Set("") + if err != nil { + suite.T().Errorf("Unexpected error: %v", err) + } + + if f.Val == nil { + suite.T().Errorf("Expected not nil") + } +} + +func (suite *OptionalFlagSuite) TestUint32OptionalFlag() { + f := &Uint32OptionalFlag{} + + err := f.Set("42") + if err != nil { + suite.T().Errorf("Unexpected error: %v", err) + } + + if f.Val == nil || *f.Val != 42 { + suite.T().Errorf("Expected 42, got %v", f.Val) + } + + err = f.Set("not a number") + if err == nil { + suite.T().Errorf("Expected error, got nil") + } +} diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 130080f..c2e7782 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -4,6 +4,7 @@ Copyright © 2024 NAME HERE package cmd import ( + "asvec/cmd/flags" "context" "fmt" "log/slog" @@ -19,14 +20,14 @@ import ( ) var listIndexFlags = &struct { - host *HostPortFlag - seeds *SeedsSliceFlag - listenerName StringOptionalFlag + host *flags.HostPortFlag + seeds *flags.SeedsSliceFlag + listenerName flags.StringOptionalFlag verbose bool timeout time.Duration }{ - host: NewDefaultHostPortFlag(), - seeds: &SeedsSliceFlag{}, + host: flags.NewDefaultHostPortFlag(), + seeds: &flags.SeedsSliceFlag{}, } func newListIndexFlagSet() *pflag.FlagSet { diff --git a/cmd/root.go b/cmd/root.go index 049d46e..f588952 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,7 @@ Copyright © 2024 NAME HERE package cmd import ( + "asvec/cmd/flags" "context" "fmt" "log/slog" @@ -20,7 +21,7 @@ var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: var view = NewView(os.Stdout, logger) var rootFlags = &struct { - logLevel LogLevelFlag + logLevel flags.LogLevelFlag }{} // rootCmd represents the base command when called without any subcommands @@ -85,7 +86,7 @@ func Execute() { func init() { rootCmd.PersistentFlags().Var(&rootFlags.logLevel, logLevelFlagName, "Log level for additional details and debugging") - common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") + common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") // TODO: Handle version viper.SetEnvPrefix("ASVEC") if err := viper.BindEnv(flagNameHost); err != nil { diff --git a/cmd/utils.go b/cmd/utils.go new file mode 100644 index 0000000..6a22f95 --- /dev/null +++ b/cmd/utils.go @@ -0,0 +1,26 @@ +package cmd + +import ( + "asvec/cmd/flags" + + avs "github.com/aerospike/aerospike-proximus-client-go" +) + +func parseBothHostSeedsFlag(seeds flags.SeedsSliceFlag, host flags.HostPortFlag) (avs.HostPortSlice, bool) { + isLoadBalancer := false + hosts := avs.HostPortSlice{} + + if len(seeds.Seeds) > 0 { + logger.Debug("seeds is set") + + hosts = append(hosts, seeds.Seeds...) + } else { + logger.Debug("hosts is set") + + isLoadBalancer = true + + hosts = append(hosts, &host.HostPort) + } + + return hosts, isLoadBalancer +} diff --git a/go.mod b/go.mod index 1b5a952..b0be57f 100644 --- a/go.mod +++ b/go.mod @@ -6,24 +6,43 @@ go 1.21.7 //replace github.com/aerospike/aerospike-proximus-client-go => /Users/jesseschmidt/Developer/aerospike-proximus-client-go require ( + github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 + github.com/stretchr/testify v1.9.0 ) require ( + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/aerospike/aerospike-client-go/v7 v7.2.1 // indirect - github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v26.1.0+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -31,6 +50,13 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect + go.opentelemetry.io/otel v1.27.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/otel/sdk v1.27.0 // indirect + go.opentelemetry.io/otel/trace v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/net v0.24.0 // indirect diff --git a/go.sum b/go.sum index 8973132..b99d82d 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,17 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/aerospike/aerospike-client-go/v7 v7.2.1 h1:4A6CxgJMRlDnHx4ycyJ1a5lUzxMS7u4byG1XlhCrvSg= github.com/aerospike/aerospike-client-go/v7 v7.2.1/go.mod h1:sKfNsnAKgkGtAlYxdgWNOJm3ykm49s/p6xjEB/cX8/k= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240509004238-aa3172c86d65 h1:0ZhYas/nSdMtx8PV0by6qiajOiPgL9JbTrZTXs1gmgE= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240509004238-aa3172c86d65/go.mod h1:HcQ1MSWCBlL8Sk8jTmXCll2ISoKMEvVBMSDeK5Y70mc= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839 h1:FZZ8WtD4roA8UOJetG7KkMFZYPYJv8Dle4NSM/Emo00= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240529232503-489d9ca9e839/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240530224737-115c3007bafa h1:+DUvX+HJvY4ZWEVXjlN14iwFBwPP8br0mDL8XPUO3E0= -github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240530224737-115c3007bafa/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 h1:qVpPCrbp0pNNmP1CPqln6HkzhVmFmOOVZYLq4IDlidI= github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -31,6 +31,7 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -43,12 +44,16 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU= github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -61,6 +66,10 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= @@ -78,13 +87,15 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -108,30 +119,71 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= @@ -146,3 +198,5 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= From f7bfa33a44e9e65688513fdfcee13158959375e2 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Wed, 5 Jun 2024 17:13:36 -0700 Subject: [PATCH 24/35] add e2e tests --- cmd/cmd_test.go | 142 ---------------- e2e_test.go | 421 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+), 142 deletions(-) delete mode 100644 cmd/cmd_test.go create mode 100644 e2e_test.go diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go deleted file mode 100644 index d338b64..0000000 --- a/cmd/cmd_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "os" - "os/exec" - "path" - "strconv" - "strings" - "testing" - "time" - - "github.com/aerospike/tools-common-go/testutils" - "github.com/stretchr/testify/suite" -) - -var wd, _ = os.Getwd() - -type CmdTestSuite struct { - suite.Suite - app string - coverFile string - coverFileCounter int - host string - port int -} - -func TestDistanceMetricFlagSuite(t *testing.T) { - suite.Run(t, new(CmdTestSuite)) -} - -func (suite *CmdTestSuite) SetupSuite() { - suite.app = path.Join(wd, "app.test") - suite.coverFile = path.Join(wd, "../coverage/cmd-coverage.cov") - suite.coverFileCounter = 0 - suite.host = "127.0.0.1" - suite.port = 10000 - - err := docker_compose_up() - if err != nil { - suite.FailNowf("unable to start docker compose up", "%v", err) - } - - os.Chdir("..") - goArgs := []string{"test", "-coverpkg", "./...", "-tags=integration", "-o", suite.app} - - // Compile test binary - compileCmd := exec.Command("go", goArgs...) - err = compileCmd.Run() - suite.Assert().NoError(err) -} - -func (suite *CmdTestSuite) TearDownSuite() { - docker_compose_down() - err := os.Remove(suite.app) - suite.Assert().NoError(err) - time.Sleep(time.Second * 5) - err = testutils.Stop() - suite.Assert().NoError(err) -} - -func (suite *CmdTestSuite) runCmd(asvecCmd ...string) ([]string, error) { - strs := strings.Split(suite.coverFile, ".") - file := strs[len(strs)-2] + "-" + strconv.Itoa(suite.coverFileCounter) + "." + strs[len(strs)-1] - suite.coverFileCounter++ - var args []string - args = []string{"-test.coverprofile=" + file} - args = append(args, asvecCmd...) - - cmd := exec.Command(suite.app, args...) - stdout, err := cmd.Output() - // fmt.Printf("stdout: %v", string(stdout)) - - if err != nil { - if ee, ok := err.(*exec.ExitError); ok { - return []string{string(ee.Stderr)}, err - } - return []string{string(stdout)}, err - } - - lines := strings.Split(string(stdout), "\n") - - return lines, nil -} - -func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { - testCases := []struct { - name string - cmd string - }{ - { - "", - "", - }, - } - - for _, tc := range testCases { - suite.Run(tc.name, func() { - lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) - suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) - }) - } -} - -func docker_compose_up() error { - fmt.Println("Starting docker containers") - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - // docker/docker-compose.yml - cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-f /../../docker/docker-compose.yml"), "up", "-d") - output, err := cmd.CombinedOutput() - - fmt.Printf("docker compose up output: %s\n", string(output)) - - if err != nil { - if _, ok := err.(*exec.ExitError); ok { - return err - } - return err - } - - return nil -} - -func docker_compose_down() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-f /../../docker/docker-compose.yml"), "down", "-d") - _, err := cmd.Output() - - if err != nil { - if _, ok := err.(*exec.ExitError); ok { - return err - } - return err - } - - return nil -} diff --git a/e2e_test.go b/e2e_test.go new file mode 100644 index 0000000..d91286a --- /dev/null +++ b/e2e_test.go @@ -0,0 +1,421 @@ +package main_test + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/exec" + "path" + "strconv" + "strings" + "testing" + "time" + + avs "github.com/aerospike/aerospike-proximus-client-go" + "github.com/aerospike/aerospike-proximus-client-go/protos" + "github.com/stretchr/testify/suite" +) + +var wd, _ = os.Getwd() +var logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})) + +var ( + testNamespace = "test" + testSet = "testset" + barNamespace = "bar" +) + +type CmdTestSuite struct { + suite.Suite + app string + coverFile string + coverFileCounter int + avsIP string + avsPort int + avsHostPort *avs.HostPort + avsClient *avs.AdminClient +} + +func TestDistanceMetricFlagSuite(t *testing.T) { + suite.Run(t, new(CmdTestSuite)) +} + +func (suite *CmdTestSuite) SetupSuite() { + suite.app = path.Join(wd, "app.test") + suite.coverFile = path.Join(wd, "../coverage/cmd-coverage.cov") + suite.coverFileCounter = 0 + suite.avsIP = "127.0.0.1" + suite.avsPort = 10000 + suite.avsHostPort = avs.NewHostPort(suite.avsIP, suite.avsPort, false) + // var err error + + err := docker_compose_up() + if err != nil { + suite.FailNowf("unable to start docker compose up", "%v", err) + } + + goArgs := []string{"build", "-cover", "-coverpkg", "./...", "-o", suite.app} + + // Compile test binary + compileCmd := exec.Command("go", goArgs...) + out, err := compileCmd.CombinedOutput() + + if err != nil { + fmt.Printf("Couldn't compile test bin stdout/err: %v\n", string(out)) + } + + suite.Assert().NoError(err) + + // Connect avs client + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + for { + suite.avsClient, err = avs.NewAdminClient(ctx, avs.HostPortSlice{suite.avsHostPort}, nil, true, logger) + + if err != nil { + fmt.Printf("unable to create avs client %v", err) + + if ctx.Err() != nil { + suite.FailNowf("unable to create avs client", "%v", err) + } + + time.Sleep(time.Second) + } else { + break + } + } + +} + +func (suite *CmdTestSuite) TearDownSuite() { + err := os.Remove(suite.app) + suite.Assert().NoError(err) + time.Sleep(time.Second * 5) + suite.Assert().NoError(err) + suite.avsClient.Close() + + err = docker_compose_down() + if err != nil { + fmt.Println("unable to stop docker compose down") + } +} + +func (suite *CmdTestSuite) runCmd(asvecCmd ...string) ([]string, error) { + strs := strings.Split(suite.coverFile, ".") + file := strs[len(strs)-2] + "-" + strconv.Itoa(suite.coverFileCounter) + "." + strs[len(strs)-1] + suite.coverFileCounter++ + var args []string + args = []string{"-test.coverprofile=" + file} + args = append(args, asvecCmd...) + + cmd := exec.Command(suite.app, args...) + stdout, err := cmd.Output() + // fmt.Printf("stdout: %v", string(stdout)) + + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + return []string{string(ee.Stderr)}, err + } + return []string{string(stdout)}, err + } + + lines := strings.Split(string(stdout), "\n") + + return lines, nil +} + +func getStrPtr(str string) *string { + ptr := str + return &ptr +} + +func getUint32Ptr(i int) *uint32 { + ptr := uint32(i) + return &ptr +} + +func getBoolPtr(b bool) *bool { + ptr := b + return &ptr +} + +type IndexDefinitionBuilder struct { + indexName string + namespace string + dimension int + vectorDistanceMetric protos.VectorDistanceMetric + vectorField string + storageNamespace *string + storageSet *string + hnsfM *uint32 + hnsfEfC *uint32 + hnsfEf *uint32 + hnsfBatchingMaxRecord *uint32 + hnsfBatchingInterval *uint32 + hnsfBatchingDisabled *bool +} + +func NewIndexDefinitionBuilder( + indexName, + namespace string, + dimension int, + distanceMetric protos.VectorDistanceMetric, + vectorField string, +) *IndexDefinitionBuilder { + return &IndexDefinitionBuilder{ + indexName: indexName, + namespace: namespace, + dimension: dimension, + vectorDistanceMetric: distanceMetric, + vectorField: vectorField, + } +} + +func (idb *IndexDefinitionBuilder) WithStorageNamespace(storageNamespace string) *IndexDefinitionBuilder { + idb.storageNamespace = &storageNamespace + return idb +} + +func (idb *IndexDefinitionBuilder) WithStorageSet(storageSet string) *IndexDefinitionBuilder { + idb.storageSet = &storageSet + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswM(m uint32) *IndexDefinitionBuilder { + idb.hnsfM = &m + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswEf(ef uint32) *IndexDefinitionBuilder { + idb.hnsfEf = &ef + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswEfConstruction(efConstruction uint32) *IndexDefinitionBuilder { + idb.hnsfEfC = &efConstruction + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingMaxRecord(maxRecord uint32) *IndexDefinitionBuilder { + idb.hnsfBatchingMaxRecord = &maxRecord + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingInterval(interval uint32) *IndexDefinitionBuilder { + idb.hnsfBatchingInterval = &interval + return idb +} + +func (idb *IndexDefinitionBuilder) WithHnswBatchingDisabled(disabled bool) *IndexDefinitionBuilder { + idb.hnsfBatchingDisabled = &disabled + return idb +} + +func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { + indexDef := &protos.IndexDefinition{ + Id: &protos.IndexId{ + Name: idb.indexName, + Namespace: idb.namespace, + }, + Dimensions: uint32(idb.dimension), + VectorDistanceMetric: idb.vectorDistanceMetric, + Field: idb.vectorField, + Type: protos.IndexType_HNSW, + Storage: &protos.IndexStorage{ + Namespace: &idb.namespace, + Set: &idb.indexName, + }, + Params: &protos.IndexDefinition_HnswParams{ + HnswParams: &protos.HnswParams{ + M: getUint32Ptr(16), + EfConstruction: getUint32Ptr(100), + Ef: getUint32Ptr(100), + BatchingParams: &protos.HnswBatchingParams{ + MaxRecords: getUint32Ptr(100000), + Interval: getUint32Ptr(30000), + Disabled: getBoolPtr(false), + }, + }, + }, + } + + if idb.storageNamespace != nil { + indexDef.Storage.Namespace = idb.storageNamespace + } + + if idb.storageSet != nil { + indexDef.Storage.Set = idb.storageSet + } + + if idb.hnsfM != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.M = idb.hnsfM + } + if idb.hnsfEf != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.Ef = idb.hnsfEf + } + if idb.hnsfEfC != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.EfConstruction = idb.hnsfEfC + } + if idb.hnsfBatchingMaxRecord != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.MaxRecords = idb.hnsfBatchingMaxRecord + } + if idb.hnsfBatchingInterval != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Interval = idb.hnsfBatchingInterval + } + if idb.hnsfBatchingDisabled != nil { + indexDef.Params.(*protos.IndexDefinition_HnswParams).HnswParams.BatchingParams.Disabled = idb.hnsfBatchingDisabled + } + + return indexDef +} + +func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { + testCases := []struct { + name string + indexName string + indexNamespace string + cmd string + expected_index *protos.IndexDefinition + }{ + { + "test with storage config", + "index1", + "test", + fmt.Sprintf("create index --seeds %s -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), + NewIndexDefinitionBuilder("index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1"). + WithStorageNamespace("bar"). + WithStorageSet("testbar"). + Build(), + }, + { + "test with hnsw params", + "index2", + "test", + fmt.Sprintf("create index --timeout 10s --seeds %s -n test -i index2 -d 256 -m HAMMING --vector-field vector2 --hnsw-max-edges 10 --hnsw-ef 11 --hnsw-ef-construction 12", suite.avsHostPort.String()), + NewIndexDefinitionBuilder("index2", "test", 256, protos.VectorDistanceMetric_HAMMING, "vector2"). + WithHnswM(10). + WithHnswEf(11). + WithHnswEfConstruction(12). + Build(), + }, + { + "test with hnsw batch params", + "index3", + "test", + fmt.Sprintf("create index --timeout 10s --seeds %s -n test -i index3 -d 256 -m COSINE --vector-field vector3 --hnsw-batch-enabled false --hnsw-batch-interval 50 --hnsw-batch-max-records 100", suite.avsHostPort.String()), + NewIndexDefinitionBuilder("index3", "test", 256, protos.VectorDistanceMetric_COSINE, "vector3"). + WithHnswBatchingMaxRecord(100). + WithHnswBatchingInterval(50). + WithHnswBatchingDisabled(true). + Build(), + }, + // { + // "test every arg", + // "index1", + // "test", + // fmt.Sprintf("create index --host %s -n test -i index2 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector2 --hnsw-batch-enabled false --storage-namespace bar", suite.avsHostPort.String()), + // NewIndexDefinitionBuilder("index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1"). + // WithStorageNamespace("bar"). + // WithHnswBatchingDisabled(true). + // Build(), + // }, + } + + for _, tc := range testCases { + suite.Run(tc.name, func() { + lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) + suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + + actual, err := suite.avsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) + + time.Sleep(time.Second) + + if err != nil { + suite.FailNowf("unable to get index", "%v", err) + } + + suite.EqualExportedValues(tc.expected_index, actual) + }) + } +} + +func (suite *CmdTestSuite) TestCreateIndexFailsAlreadyExistsCmd() { + + lines, err := suite.runCmd(strings.Split(fmt.Sprintf("create index --seeds %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) + suite.Assert().NoError(err, "index should have NOT existed on first call. error: %s, stdout/err: %s", err, lines) + + lines, err = suite.runCmd(strings.Split(fmt.Sprintf("create index --seeds %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) + suite.Assert().Error(err, "index should HAVE existed on first call. error: %s, stdout/err: %s", err, lines) + + suite.Assert().Contains(lines[0], "AlreadyExists") +} + +func (suite *CmdTestSuite) TestFailInvalidArgCreateIndexCmd() { + testCases := []struct { + name string + cmd string + errStr string + }{ + { + "test with storage config", + fmt.Sprintf("create index --seeds %s --host 1.1.1.1:3001 -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), + "Error: only --seeds or --host allowed", + }, + { + "test with storage config", + "create index --host 1.1.1.1:3001 -n test -i index1 -d -1 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", + "Error: invalid argument \"-1\" for \"-d, --dimension\"", + }, + } + + for _, tc := range testCases { + suite.Run(tc.name, func() { + lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) + + suite.Assert().Error(err, "error: %s, stdout/err: %s", err, lines) + suite.Assert().Contains(lines[0], tc.errStr) + }) + } +} + +func docker_compose_up() error { + fmt.Println("Starting docker containers") + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + // docker/docker-compose.yml + cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-fdocker/docker-compose.yml"), "up", "-d") + output, err := cmd.CombinedOutput() + + fmt.Printf("docker compose up output: %s\n", string(output)) + + if err != nil { + if _, ok := err.(*exec.ExitError); ok { + return err + } + return err + } + + return nil +} + +func docker_compose_down() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cmd := exec.CommandContext(ctx, "docker", "compose", fmt.Sprintf("-fdocker/docker-compose.yml"), "down") + _, err := cmd.Output() + + if err != nil { + if _, ok := err.(*exec.ExitError); ok { + return err + } + return err + } + + return nil +} + +// func Index From af6c217c5aa42841846b05bcf6628d62aa94abe2 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Mon, 10 Jun 2024 13:55:52 -0700 Subject: [PATCH 25/35] add test workflow --- .github/workflows/tests.yml | 33 ++++++++ .gitignore | 4 +- Makefile | 31 ++++++- cmd/createIndex.go | 3 +- cmd/flags/distanceMetric.go | 14 ++-- cmd/flags/distanceMetric_test.go | 8 +- cmd/flags/hostPort_test.go | 2 + cmd/flags/logLevel.go | 14 ++-- cmd/flags/logLevel_test.go | 8 +- cmd/flags/optionals_test.go | 2 + cmd/root.go | 3 +- e2e_test.go | 136 +++++++++++++++++++++++++------ go.mod | 40 +++------ go.sum | 103 +++++------------------ 14 files changed, 250 insertions(+), 151 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..35da671 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,33 @@ +name: Test +on: + push: + branches-ignore: + - main + workflow_call: + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Get go version from go.mod + run: | + echo "GO_VERSION=$(grep '^go ' go.mod | cut -d " " -f 2)" >> $GITHUB_ENV + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - name: Write feature keys + env: + FEATURES_CONF : ${{secrets.FEATURES_CONF}} + + run: | + echo "$FEATURES_CONF" > docker/config/features.conf + - name: Run tests + run: | + make coverage + # - name: Upload coverage to Codecov // Add when public + # uses: codecov/codecov-action@v3 + # with: + # token: ${{secrets.CODECOV_TOKEN}} + # files: testdata/coverage/total.cov + # verbose: false diff --git a/.gitignore b/.gitignore index ff43f2a..985e7dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /docker/* /bin/* embed_*.go -/tmp \ No newline at end of file +/tmp +/vendor +/coverage \ No newline at end of file diff --git a/Makefile b/Makefile index e15f2af..e93b729 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ .NOTPARALLEL: +# TODO: move build/pkg related makefile jobs to pkg/ + ## requirements: make dpkg rpmbuild upx golang zip wget jq ## macos pkg requirement: https://docker-laptop.s3.eu-west-1.amazonaws.com/Packages.pkg @@ -9,7 +11,11 @@ INSTALLSIGNER := "$(asvec_installsigner)" APPLEID := "$(asvec_appleid)" APPLEPW := "$(asvec_applepw)" TEAMID := "$(asvec_teamid)" +ROOT_DIR = $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) BIN_DIR = ./bin +COVERAGE_DIR = $(ROOT_DIR)/coverage +COV_UNIT_DIR = $(COVERAGE_DIR)/unit +COV_INTEGRATION_DIR = $(COVERAGE_DIR)/integration ## available make commands .PHONY: help @@ -488,4 +494,27 @@ macos-pkg-notarize: fi ### make cleanall && make build-prerelease && make pkg-linux && make pkg-windows-zip && make macos-build-all && make macos-notarize-all -### make cleanall && make build-official && make pkg-linux && make pkg-windows-zip && make macos-build-all && make macos-notarize-all \ No newline at end of file +### make cleanall && make build-official && make pkg-linux && make pkg-windows-zip && make macos-build-all && make macos-notarize-all + +.PHONY: test +test: integration unit + +.PHONY: integration +integration: + mkdir -p $(COV_INTEGRATION_DIR) || true + COVERAGE_DIR=$(COV_INTEGRATION_DIR) go test -tags=integration -timeout 30m + +.PHONY: unit +unit: + mkdir -p $(COV_UNIT_DIR) || true + go test -tags=unit -cover ./... -args -test.gocoverdir=$(COV_UNIT_DIR) + +.PHONY: coverage +coverage: test + go tool covdata textfmt -i="$(COV_INTEGRATION_DIR),$(COV_UNIT_DIR)" -o=$(COVERAGE_DIR)/total.cov + go tool cover -func=$(COVERAGE_DIR)/total.cov + + +PHONY: view-coverage +view-coverage: $(COVERAGE_DIR)/total.cov + go tool cover -html=$(COVERAGE_DIR)/total.cov diff --git a/cmd/createIndex.go b/cmd/createIndex.go index fe388a9..20ed67f 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "log/slog" + "strings" "time" avs "github.com/aerospike/aerospike-proximus-client-go" @@ -62,7 +63,7 @@ func newCreateIndexFlagSet() *pflag.FlagSet { flagSet.StringVarP(&createIndexFlags.indexName, flagNameIndexName, "i", "", commonFlags.DefaultWrapHelpString("The name of the index.")) //nolint:lll // For readability flagSet.StringVarP(&createIndexFlags.vectorField, flagNameVectorField, "f", "", commonFlags.DefaultWrapHelpString("The name of the vector field.")) //nolint:lll // For readability flagSet.Uint32VarP(&createIndexFlags.dimensions, flagNameDimension, "d", 0, commonFlags.DefaultWrapHelpString("The dimension of the vector field.")) //nolint:lll // For readability - flagSet.VarP(&createIndexFlags.distanceMetric, flagNameDistanceMetric, "m", commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability + flagSet.VarP(&createIndexFlags.distanceMetric, flagNameDistanceMetric, "m", commonFlags.DefaultWrapHelpString(fmt.Sprintf("The distance metric for the index. Valid values: %s", strings.Join(flags.DistanceMetricEnum(), ", ")))) //nolint:lll // For readability flagSet.StringToStringVar(&createIndexFlags.indexMeta, flagNameIndexMeta, nil, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability flagSet.DurationVar(&createIndexFlags.timeout, flagNameTimeout, time.Second*5, commonFlags.DefaultWrapHelpString("The distance metric for the index.")) //nolint:lll // For readability flagSet.Var(&createIndexFlags.storageNamespace, flagNameStorageNamespace, commonFlags.DefaultWrapHelpString("Optional storage namespace where the index is stored. Defaults to the index namespace.")) //nolint:lll // For readability //nolint:lll // For readability diff --git a/cmd/flags/distanceMetric.go b/cmd/flags/distanceMetric.go index 2f07e5e..78d332e 100644 --- a/cmd/flags/distanceMetric.go +++ b/cmd/flags/distanceMetric.go @@ -24,6 +24,14 @@ func (f *DistanceMetricFlag) Set(val string) error { } func (f *DistanceMetricFlag) Type() string { + return "enum" +} + +func (f *DistanceMetricFlag) String() string { + return string(*f) +} + +func DistanceMetricEnum() []string { names := []string{} for key := range distanceMetricSet { @@ -32,9 +40,5 @@ func (f *DistanceMetricFlag) Type() string { slices.Sort(names) - return strings.Join(names, ",") -} - -func (f *DistanceMetricFlag) String() string { - return string(*f) + return names } diff --git a/cmd/flags/distanceMetric_test.go b/cmd/flags/distanceMetric_test.go index 2f1ec1a..5d1444a 100644 --- a/cmd/flags/distanceMetric_test.go +++ b/cmd/flags/distanceMetric_test.go @@ -1,3 +1,5 @@ +//go:build unit + package flags import ( @@ -60,7 +62,7 @@ func (suite *DistanceMetricFlagTestSuite) TestSet() { func (suite *DistanceMetricFlagTestSuite) TestType() { flag := DistanceMetricFlag("") - suite.Equal("COSINE,DOT_PRODUCT,HAMMING,MANHATTAN,SQUARED_EUCLIDEAN", flag.Type()) + suite.Equal("enum", flag.Type()) } func (suite *DistanceMetricFlagTestSuite) TestString() { @@ -74,6 +76,10 @@ func (suite *DistanceMetricFlagTestSuite) TestString() { suite.Equal("manhattan", flag.String()) } +func (suite *DistanceMetricFlagTestSuite) TestDistanceMetricEnum() { + suite.Equal([]string{"COSINE", "DOT_PRODUCT", "HAMMING", "MANHATTAN", "SQUARED_EUCLIDEAN"}, DistanceMetricEnum()) +} + func TestDistanceMetricFlagSuite(t *testing.T) { suite.Run(t, new(DistanceMetricFlagTestSuite)) } diff --git a/cmd/flags/hostPort_test.go b/cmd/flags/hostPort_test.go index a02d02a..1b14406 100644 --- a/cmd/flags/hostPort_test.go +++ b/cmd/flags/hostPort_test.go @@ -1,3 +1,5 @@ +//go:build unit + package flags import ( diff --git a/cmd/flags/logLevel.go b/cmd/flags/logLevel.go index 72cb804..bdb622c 100644 --- a/cmd/flags/logLevel.go +++ b/cmd/flags/logLevel.go @@ -35,6 +35,14 @@ func (f *LogLevelFlag) Set(val string) error { } func (f *LogLevelFlag) Type() string { + return "enum" +} + +func (f *LogLevelFlag) String() string { + return string(*f) +} + +func LogLevelEnum() []string { names := []string{} for key := range logLevelSet { @@ -45,9 +53,5 @@ func (f *LogLevelFlag) Type() string { return logLevelSet[names[i]] < logLevelSet[names[j]] }) - return strings.Join(names, ",") -} - -func (f *LogLevelFlag) String() string { - return string(*f) + return names } diff --git a/cmd/flags/logLevel_test.go b/cmd/flags/logLevel_test.go index dd134c6..44b3a4b 100644 --- a/cmd/flags/logLevel_test.go +++ b/cmd/flags/logLevel_test.go @@ -1,3 +1,5 @@ +//go:build unit + package flags import ( @@ -35,10 +37,14 @@ func (suite *LogLevelSuite) TestSet() { func (suite *LogLevelSuite) TestType() { flag := LogLevelFlag("") - suite.Equal("DEBUG,INFO,WARN,ERROR", flag.Type()) + suite.Equal("enum", flag.Type()) } func (suite *LogLevelSuite) TestString() { flag := LogLevelFlag("DEBUG") suite.Equal("DEBUG", flag.String()) } + +func (suite *LogLevelSuite) TestLogLevelEnum() { + suite.Equal([]string{"DEBUG", "INFO", "WARN", "ERROR"}, LogLevelEnum()) +} diff --git a/cmd/flags/optionals_test.go b/cmd/flags/optionals_test.go index e21a715..12bc5eb 100644 --- a/cmd/flags/optionals_test.go +++ b/cmd/flags/optionals_test.go @@ -1,3 +1,5 @@ +//go:build unit + package flags import ( diff --git a/cmd/root.go b/cmd/root.go index f588952..017228e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,6 +9,7 @@ import ( "fmt" "log/slog" "os" + "strings" common "github.com/aerospike/tools-common-go/flags" "github.com/spf13/cobra" @@ -85,7 +86,7 @@ func Execute() { } func init() { - rootCmd.PersistentFlags().Var(&rootFlags.logLevel, logLevelFlagName, "Log level for additional details and debugging") + rootCmd.PersistentFlags().Var(&rootFlags.logLevel, logLevelFlagName, common.DefaultWrapHelpString(fmt.Sprintf("Log level for additional details and debugging. Valid values: %s", strings.Join(flags.LogLevelEnum(), ", ")))) common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") // TODO: Handle version viper.SetEnvPrefix("ASVEC") diff --git a/e2e_test.go b/e2e_test.go index d91286a..50989cf 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main_test import ( @@ -7,7 +9,6 @@ import ( "os" "os/exec" "path" - "strconv" "strings" "testing" "time" @@ -103,14 +104,8 @@ func (suite *CmdTestSuite) TearDownSuite() { } func (suite *CmdTestSuite) runCmd(asvecCmd ...string) ([]string, error) { - strs := strings.Split(suite.coverFile, ".") - file := strs[len(strs)-2] + "-" + strconv.Itoa(suite.coverFileCounter) + "." + strs[len(strs)-1] - suite.coverFileCounter++ - var args []string - args = []string{"-test.coverprofile=" + file} - args = append(args, asvecCmd...) - - cmd := exec.Command(suite.app, args...) + cmd := exec.Command(suite.app, asvecCmd...) + cmd.Env = []string{"GOCOVERDIR=" + os.Getenv("COVERAGE_DIR")} stdout, err := cmd.Output() // fmt.Printf("stdout: %v", string(stdout)) @@ -274,7 +269,7 @@ func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { testCases := []struct { name string - indexName string + indexName string // index names must be unique for the suite indexNamespace string cmd string expected_index *protos.IndexDefinition @@ -311,16 +306,6 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { WithHnswBatchingDisabled(true). Build(), }, - // { - // "test every arg", - // "index1", - // "test", - // fmt.Sprintf("create index --host %s -n test -i index2 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector2 --hnsw-batch-enabled false --storage-namespace bar", suite.avsHostPort.String()), - // NewIndexDefinitionBuilder("index1", "test", 256, protos.VectorDistanceMetric_SQUARED_EUCLIDEAN, "vector1"). - // WithStorageNamespace("bar"). - // WithHnswBatchingDisabled(true). - // Build(), - // }, } for _, tc := range testCases { @@ -342,7 +327,6 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { } func (suite *CmdTestSuite) TestCreateIndexFailsAlreadyExistsCmd() { - lines, err := suite.runCmd(strings.Split(fmt.Sprintf("create index --seeds %s -n test -i exists -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), " ")...) suite.Assert().NoError(err, "index should have NOT existed on first call. error: %s, stdout/err: %s", err, lines) @@ -352,22 +336,126 @@ func (suite *CmdTestSuite) TestCreateIndexFailsAlreadyExistsCmd() { suite.Assert().Contains(lines[0], "AlreadyExists") } -func (suite *CmdTestSuite) TestFailInvalidArgCreateIndexCmd() { +func (suite *CmdTestSuite) TestSuccessfulDropIndexCmd() { + testCases := []struct { + name string + indexName string // index names must be unique for the suite + indexNamespace string + indexSet []string + cmd string + }{ + { + "test with just namespace", + "indexdrop1", + "test", + nil, + fmt.Sprintf("drop index --seeds %s -n test -i indexdrop1 --timeout 10s", suite.avsHostPort.String()), + }, + { + "test with set", + "indexdrop2", + "test", + []string{ + "testset", + }, + fmt.Sprintf("drop index --seeds %s -n test -s testset -i indexdrop2 --timeout 10s", suite.avsHostPort.String()), + }, + } + + for _, tc := range testCases { + suite.Run(tc.name, func() { + err := suite.avsClient.IndexCreate(context.Background(), tc.indexNamespace, tc.indexSet, tc.indexName, "vector", 1, protos.VectorDistanceMetric_COSINE, nil, nil, nil) + if err != nil { + suite.FailNowf("unable to create index", "%v", err) + } + + lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) + suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + + _, err = suite.avsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) + + time.Sleep(time.Second) + + if err == nil { + suite.FailNow("err is nil, that means the index still exists") + } + }) + } +} + +func (suite *CmdTestSuite) TestDropIndexFailsDoesNotExistCmd() { + lines, err := suite.runCmd(strings.Split(fmt.Sprintf("drop index --seeds %s -n test -i DNE --timeout 10s", suite.avsHostPort.String()), " ")...) + + suite.Assert().Error(err, "index should have NOT existed. stdout/err: %s", lines) + suite.Assert().Contains(lines[0], "server error") +} + +func (suite *CmdTestSuite) TestFailInvalidArg() { testCases := []struct { name string cmd string errStr string }{ { - "test with storage config", + "use seeds and hosts together", fmt.Sprintf("create index --seeds %s --host 1.1.1.1:3001 -n test -i index1 -d 256 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", suite.avsHostPort.String()), "Error: only --seeds or --host allowed", }, { - "test with storage config", + "use seeds and hosts together", + fmt.Sprintf("list index --seeds %s --host 1.1.1.1:3001", suite.avsHostPort.String()), + "Error: only --seeds or --host allowed", + }, + { + "use seeds and hosts together", + fmt.Sprintf("drop index --seeds %s --host 1.1.1.1:3001 -n test -i index1", suite.avsHostPort.String()), + "Error: only --seeds or --host allowed", + }, + { + "test with bad dimension", "create index --host 1.1.1.1:3001 -n test -i index1 -d -1 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", "Error: invalid argument \"-1\" for \"-d, --dimension\"", }, + { + "test with bad distance metric", + "create index --host 1.1.1.1:3001 -n test -i index1 -d 10 -m BAD --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10s", + "Error: invalid argument \"BAD\" for \"-m, --distance-metric\"", + }, + { + "test with bad timeout", + "create index --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10", + "Error: invalid argument \"10\" for \"--timeout\"", + }, + { + "test with bad hnsw-batch-enabled", + "create index --hnsw-batch-enabled foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10", + "Error: invalid argument \"foo\" for \"--hnsw-batch-enabled\"", + }, + { + "test with bad hnsw-batch-interval", + "create index --hnsw-batch-interval foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10", + "Error: invalid argument \"foo\" for \"--hnsw-batch-interval\"", + }, + { + "test with bad hnsw-batch-max-records", + "create index --hnsw-batch-max-records foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10", + "Error: invalid argument \"foo\" for \"--hnsw-batch-max-records\"", + }, + { + "test with bad hnsw-ef", + "create index --hnsw-ef foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10", + "Error: invalid argument \"foo\" for \"--hnsw-ef\"", + }, + { + "test with bad hnsw-ef-construction", + "create index --hnsw-ef-construction foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10", + "Error: invalid argument \"foo\" for \"--hnsw-ef-construction\"", + }, + { + "test with bad hnsw-max-edges", + "create index --hnsw-max-edges foo --host 1.1.1.1:3001 -n test -i index1 -d 10 -m SQUARED_EUCLIDEAN --vector-field vector1 --storage-namespace bar --storage-set testbar --timeout 10", + "Error: invalid argument \"foo\" for \"--hnsw-max-edges\"", + }, } for _, tc := range testCases { diff --git a/go.mod b/go.mod index b0be57f..5f42fff 100644 --- a/go.mod +++ b/go.mod @@ -11,60 +11,40 @@ require ( github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.18.2 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 ) require ( - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/aerospike/aerospike-client-go/v7 v7.2.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/aerospike/aerospike-client-go/v7 v7.4.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v26.1.0+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect - go.opentelemetry.io/otel v1.27.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/trace v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/net v0.24.0 // indirect + golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/grpc v1.63.2 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index b99d82d..a31c1ab 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,11 @@ -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/aerospike/aerospike-client-go/v7 v7.2.1 h1:4A6CxgJMRlDnHx4ycyJ1a5lUzxMS7u4byG1XlhCrvSg= -github.com/aerospike/aerospike-client-go/v7 v7.2.1/go.mod h1:sKfNsnAKgkGtAlYxdgWNOJm3ykm49s/p6xjEB/cX8/k= +github.com/aerospike/aerospike-client-go/v7 v7.4.0 h1:g8/7v8RHhQhTArhW3C7Au7o+u8j8x5eySZL6MXfpHKU= +github.com/aerospike/aerospike-client-go/v7 v7.4.0/go.mod h1:pPKnWiS8VDJcH4IeB1b8SA2TWnkjcVLHwAAJ+BHfGK8= github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9 h1:qVpPCrbp0pNNmP1CPqln6HkzhVmFmOOVZYLq4IDlidI= github.com/aerospike/aerospike-proximus-client-go v0.0.0-20240603230632-86a0ebaa8aa9/go.mod h1:N0kxd4FoYDbLOEwm8vWH6wKUkoR5v0Wp/v0+tUqoUMg= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926 h1:CqkNasGC/7x5JvYjCSuAVX/rG+nUgRQtXfxIURXo5OE= github.com/aerospike/tools-common-go v0.0.0-20240425222921-596724ec5926/go.mod h1:Ig1lRynXx0tXNOY3MdtanTsKz1ifG/2AyDFMXn3RMTc= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -31,7 +25,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -44,16 +37,12 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU= github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -66,10 +55,6 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= @@ -85,17 +70,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -106,8 +90,8 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= -github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -119,75 +103,34 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -198,5 +141,3 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= From 81290cb783a65cb07b0d214ebaf508b9f6cc261f Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Mon, 10 Jun 2024 13:57:34 -0700 Subject: [PATCH 26/35] add docker config for tests --- .gitignore | 2 +- docker/config/aerospike-proximus.yml | 73 +++++++++++++++++++++++++ docker/config/aerospike.conf | 82 ++++++++++++++++++++++++++++ docker/docker-compose.yml | 23 ++++++++ 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 docker/config/aerospike-proximus.yml create mode 100644 docker/config/aerospike.conf create mode 100644 docker/docker-compose.yml diff --git a/.gitignore b/.gitignore index 985e7dc..d87c693 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/docker/* +/docker/config/features.conf /bin/* embed_*.go /tmp diff --git a/docker/config/aerospike-proximus.yml b/docker/config/aerospike-proximus.yml new file mode 100644 index 0000000..ac73f06 --- /dev/null +++ b/docker/config/aerospike-proximus.yml @@ -0,0 +1,73 @@ +# Change the configuration for your use case. +cluster: + # Custom node-id. It will be auto-generated if not specified. + # node-id: a1 + + # Unique identifier for this cluster. + cluster-name: prism-image-search + +# The Proximus service listening ports, TLS and network interface. +service: + ports: + 10000: {} + advertised-listeners: + default: + address: 127.0.0.1 + port: 10000 + +# Management API listening ports, TLS and network interface. +manage: + ports: + 5040: {} + +# Intra cluster interconnect listening ports, TLS and network interface. +interconnect: + ports: + 5001: {} + +#heartbeat: +# seeds: +# - address: localhost +# port: 6001 + +# Target Aerospike cluster +aerospike: + seeds: + - aerospike: + port: 3000 + +# File based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: file +# credentials-file: samples/credentials.yml +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# Vault based credentials store only if security should be enabled. +#security: +# credentials-store: +# type: vault +# url: https://vault:8200 +# secrets-path: /secret/aerospike/aerodb1 +# tls: +# key-store: +# store-type: PEM +# store-file: key.pem +# store-password-file: keypass.txt # Password protecting key.pem. +# certificate-chain-files: certchain.pem +# trust-store: +# store-type: PEM +# certificate-files: cacert.pem +# auth-token: +# private-key: samples/auth/private_key.pem +# public-key: samples/auth/public_key.pem + +# The logging properties. +logging: + #format: json + #file: /var/log/aerospike-proximus/aerospike-proximus.log + enable-console-logging: true + levels: + metrics-ticker: off diff --git a/docker/config/aerospike.conf b/docker/config/aerospike.conf new file mode 100644 index 0000000..a23c052 --- /dev/null +++ b/docker/config/aerospike.conf @@ -0,0 +1,82 @@ +# Aerospike database configuration file for use with systemd. + +service { + cluster-name prism-demo + proto-fd-max 15000 +} + + +logging { + file /var/log/aerospike/aerospike.log { + context any info + } + + # Send log messages to stdout + console { + context any info + context query critical + } +} + +network { + service { + address any + port 3000 + } + + heartbeat { + mode multicast + multicast-group 239.1.99.222 + port 9918 + + # To use unicast-mesh heartbeats, remove the 3 lines above, and see + # aerospike_mesh.conf for alternative. + + interval 150 + timeout 10 + } + + fabric { + port 3001 + } + + info { + port 3003 + } +} + +namespace test { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace bar { + replication-factor 1 + nsup-period 60 + + storage-engine memory { + data-size 1G + } +} + +namespace proximus-meta { + replication-factor 1 + nsup-period 100 + + storage-engine memory { + data-size 1G + } + + # To use file storage backing, comment out the line above and use the + # following lines instead. +# storage-engine device { +# file /opt/aerospike/data/bar.dat +# filesize 16G +# data-in-memory true # Store data in memory in addition to file. +# } +} + diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..9870a90 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,23 @@ +services: + aerospike: + image: aerospike/aerospike-server-enterprise:7.0.0.2 + ports: + - "3000:3000" + networks: + - avs-demo + volumes: + - ./config:/opt/aerospike/etc/aerospike + command: + - "--config-file" + - "/opt/aerospike/etc/aerospike/aerospike.conf" + avs: + image: aerospike/aerospike-proximus:0.4.0 + ports: + - "10000:10000" + networks: + - avs-demo + volumes: + - ./config:/etc/aerospike-proximus + +networks: + avs-demo: {} From 4848088f59b2e2dffbe88d1b71e8193bc0d573b9 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Mon, 10 Jun 2024 13:59:37 -0700 Subject: [PATCH 27/35] fix markdown link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f478c12..ba05b4b 100644 --- a/README.md +++ b/README.md @@ -40,4 +40,4 @@ Otherwise, if you are an enterprise customer, please [contact support](https://a ## License -This project is licensed under the MIT License. See the [LICENSE.md](LICENSE.md) file for details. +This project is licensed under the MIT License. See the [LICENSE.md](./LICENSE.md) file for details. From 9a015b1f2bf377d9a7e5f10d27b58a05f7d0e92b Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 11 Jun 2024 13:00:50 -0700 Subject: [PATCH 28/35] add codecov upload --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 35da671..9a82f7a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,9 +25,9 @@ jobs: - name: Run tests run: | make coverage - # - name: Upload coverage to Codecov // Add when public - # uses: codecov/codecov-action@v3 - # with: - # token: ${{secrets.CODECOV_TOKEN}} - # files: testdata/coverage/total.cov - # verbose: false + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{secrets.CODECOV_TOKEN}} + files: ./coverage/total.cov + verbose: false From 0cb992f01d2accc94b711c4a09ac60b6071d2ef7 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 11 Jun 2024 13:02:08 -0700 Subject: [PATCH 29/35] only build pre-release on workflow dispatch --- .github/workflows/create-prerelease.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/create-prerelease.yml b/.github/workflows/create-prerelease.yml index df14589..556ad73 100644 --- a/.github/workflows/create-prerelease.yml +++ b/.github/workflows/create-prerelease.yml @@ -1,9 +1,6 @@ name: Build and Create Pre-Release on: - push: - branches: VEC-168-init - workflow_dispatch: inputs: addCommit: From 419a1b5b97d5bebd3996524b99dcd4a0e27775d3 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 11 Jun 2024 14:14:00 -0700 Subject: [PATCH 30/35] add list index tests --- cmd/listIndex.go | 6 +- e2e_test.go | 158 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/cmd/listIndex.go b/cmd/listIndex.go index c2e7782..2b90ffc 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -45,8 +45,10 @@ var listIndexRequiredFlags = []string{} // listIndexCmd represents the listIndex command func newListIndexCmd() *cobra.Command { - return &cobra.Command{Use: "index", - Short: "A command for listing indexes", + return &cobra.Command{ + Use: "index", + Aliases: []string{"indexes"}, + Short: "A command for listing indexes", Long: fmt.Sprintf(`A command for displaying useful information about AVS indexes. To display additional index information use the --%s flag. For example: diff --git a/e2e_test.go b/e2e_test.go index 50989cf..7d8bf3f 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path" + "regexp" "strings" "testing" "time" @@ -139,6 +140,7 @@ func getBoolPtr(b bool) *bool { type IndexDefinitionBuilder struct { indexName string namespace string + set *string dimension int vectorDistanceMetric protos.VectorDistanceMetric vectorField string @@ -168,6 +170,11 @@ func NewIndexDefinitionBuilder( } } +func (idb *IndexDefinitionBuilder) WithSet(set string) *IndexDefinitionBuilder { + idb.set = &set + return idb +} + func (idb *IndexDefinitionBuilder) WithStorageNamespace(storageNamespace string) *IndexDefinitionBuilder { idb.storageNamespace = &storageNamespace return idb @@ -236,6 +243,10 @@ func (idb *IndexDefinitionBuilder) Build() *protos.IndexDefinition { }, } + if idb.set != nil { + indexDef.SetFilter = idb.set + } + if idb.storageNamespace != nil { indexDef.Storage.Namespace = idb.storageNamespace } @@ -390,6 +401,153 @@ func (suite *CmdTestSuite) TestDropIndexFailsDoesNotExistCmd() { suite.Assert().Contains(lines[0], "server error") } +func removeANSICodes(input string) string { + re := regexp.MustCompile(`\x1b[^m]*m`) + return re.ReplaceAllString(input, "") +} + +func (suite *CmdTestSuite) TestSuccessfulListIndexCmd() { + indexes, err := suite.avsClient.IndexList(context.Background()) + if err != nil { + suite.FailNow(err.Error()) + } + + for _, index := range indexes.GetIndices() { + err := suite.avsClient.IndexDrop(context.Background(), index.Id.Namespace, index.Id.Name) + if err != nil { + suite.FailNow(err.Error()) + } + } + + testCases := []struct { + name string + indexes []*protos.IndexDefinition + cmd string + expectedTable string + }{ + { + "single index", + []*protos.IndexDefinition{ + NewIndexDefinitionBuilder( + "list", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", + ).Build(), + }, + fmt.Sprintf("list index -h %s", suite.avsHostPort.String()), + `╭─────────────────────────────────────────────────────────────────────────╮ +│ Indexes │ +├───┬──────┬───────────┬────────┬────────────┬─────────────────┬──────────┤ +│ │ NAME │ NAMESPACE │ FIELD │ DIMENSIONS │ DISTANCE METRIC │ UNMERGED │ +├───┼──────┼───────────┼────────┼────────────┼─────────────────┼──────────┤ +│ 1 │ list │ test │ vector │ 256 │ COSINE │ 0 │ +╰───┴──────┴───────────┴────────┴────────────┴─────────────────┴──────────╯ +`, + }, + { + "double index with set", + []*protos.IndexDefinition{ + NewIndexDefinitionBuilder( + "list1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", + ).Build(), + NewIndexDefinitionBuilder( + "list2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector", + ).WithSet("barset").Build(), + }, + fmt.Sprintf("list index -h %s", suite.avsHostPort.String()), + `╭───────────────────────────────────────────────────────────────────────────────────╮ +│ Indexes │ +├───┬───────┬───────────┬────────┬────────┬────────────┬─────────────────┬──────────┤ +│ │ NAME │ NAMESPACE │ SET │ FIELD │ DIMENSIONS │ DISTANCE METRIC │ UNMERGED │ +├───┼───────┼───────────┼────────┼────────┼────────────┼─────────────────┼──────────┤ +│ 1 │ list2 │ bar │ barset │ vector │ 256 │ HAMMING │ 0 │ +├───┼───────┼───────────┼────────┼────────┼────────────┼─────────────────┼──────────┤ +│ 2 │ list1 │ test │ │ vector │ 256 │ COSINE │ 0 │ +╰───┴───────┴───────────┴────────┴────────┴────────────┴─────────────────┴──────────╯ +`, + }, + { + "double index with set and verbose", + []*protos.IndexDefinition{ + NewIndexDefinitionBuilder( + "list1", "test", 256, protos.VectorDistanceMetric_COSINE, "vector", + ).Build(), + NewIndexDefinitionBuilder( + "list2", "bar", 256, protos.VectorDistanceMetric_HAMMING, "vector", + ).WithSet("barset").Build(), + }, + fmt.Sprintf("list index -h %s --verbose", suite.avsHostPort.String()), + `╭────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ Indexes │ +├───┬───────┬───────────┬────────┬────────┬────────────┬─────────────────┬──────────┬───────────────────────┬────────────────────────────────┤ +│ │ NAME │ NAMESPACE │ SET │ FIELD │ DIMENSIONS │ DISTANCE METRIC │ UNMERGED │ STORAGE │ INDEX PARAMETERS │ +├───┼───────┼───────────┼────────┼────────┼────────────┼─────────────────┼──────────┼───────────────────────┼────────────────────────────────┤ +│ 1 │ list2 │ bar │ barset │ vector │ 256 │ HAMMING │ 0 │ ╭───────────┬───────╮ │ ╭────────────────────────────╮ │ +│ │ │ │ │ │ │ │ │ │ Namespace │ bar │ │ │ HNSW │ │ +│ │ │ │ │ │ │ │ │ │ Set │ list2 │ │ ├───────────────────┬────────┤ │ +│ │ │ │ │ │ │ │ │ ╰───────────┴───────╯ │ │ Max Edges │ 16 │ │ +│ │ │ │ │ │ │ │ │ │ │ Ef │ 100 │ │ +│ │ │ │ │ │ │ │ │ │ │ Construction Ef │ 100 │ │ +│ │ │ │ │ │ │ │ │ │ │ Batch Max Records │ 100000 │ │ +│ │ │ │ │ │ │ │ │ │ │ Batch Interval │ 30000 │ │ +│ │ │ │ │ │ │ │ │ │ │ Batch Enabled │ true │ │ +│ │ │ │ │ │ │ │ │ │ ╰───────────────────┴────────╯ │ +├───┼───────┼───────────┼────────┼────────┼────────────┼─────────────────┼──────────┼───────────────────────┼────────────────────────────────┤ +│ 2 │ list1 │ test │ │ vector │ 256 │ COSINE │ 0 │ ╭───────────┬───────╮ │ ╭────────────────────────────╮ │ +│ │ │ │ │ │ │ │ │ │ Namespace │ test │ │ │ HNSW │ │ +│ │ │ │ │ │ │ │ │ │ Set │ list1 │ │ ├───────────────────┬────────┤ │ +│ │ │ │ │ │ │ │ │ ╰───────────┴───────╯ │ │ Max Edges │ 16 │ │ +│ │ │ │ │ │ │ │ │ │ │ Ef │ 100 │ │ +│ │ │ │ │ │ │ │ │ │ │ Construction Ef │ 100 │ │ +│ │ │ │ │ │ │ │ │ │ │ Batch Max Records │ 100000 │ │ +│ │ │ │ │ │ │ │ │ │ │ Batch Interval │ 30000 │ │ +│ │ │ │ │ │ │ │ │ │ │ Batch Enabled │ true │ │ +│ │ │ │ │ │ │ │ │ │ ╰───────────────────┴────────╯ │ +╰───┴───────┴───────────┴────────┴────────┴────────────┴─────────────────┴──────────┴───────────────────────┴────────────────────────────────╯ +`, + }, + } + + for _, tc := range testCases { + suite.Run(tc.name, func() { + for _, index := range tc.indexes { + setFilter := []string{} + if index.SetFilter != nil { + setFilter = append(setFilter, *index.SetFilter) + } + + err := suite.avsClient.IndexCreate( + context.Background(), + index.Id.Namespace, + setFilter, + index.Id.Name, + index.GetField(), + index.GetDimensions(), + index.GetVectorDistanceMetric(), + index.GetHnswParams(), + index.GetLabels(), + index.GetStorage(), + ) + if err != nil { + suite.FailNowf("unable to create index", "%v", err) + } + + defer suite.avsClient.IndexDrop( + context.Background(), + index.Id.Namespace, + index.Id.Name, + ) + } + + lines, err := suite.runCmd(strings.Split(tc.cmd, " ")...) + suite.Assert().NoError(err, "error: %s, stdout/err: %s", err, lines) + + actualTable := removeANSICodes(strings.Join(lines, "\n")) + + suite.Assert().Equal(tc.expectedTable, actualTable) + + }) + } +} + func (suite *CmdTestSuite) TestFailInvalidArg() { testCases := []struct { name string From 089105e7ec2b42cb6f23d3153b1584e6baedb2f2 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Thu, 13 Jun 2024 13:25:13 -0700 Subject: [PATCH 31/35] fix lint --- cmd/root.go | 6 +++++- cmd/utils_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 cmd/utils_test.go diff --git a/cmd/root.go b/cmd/root.go index 017228e..8305e98 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -86,7 +86,11 @@ func Execute() { } func init() { - rootCmd.PersistentFlags().Var(&rootFlags.logLevel, logLevelFlagName, common.DefaultWrapHelpString(fmt.Sprintf("Log level for additional details and debugging. Valid values: %s", strings.Join(flags.LogLevelEnum(), ", ")))) + rootCmd.PersistentFlags().Var( + &rootFlags.logLevel, + logLevelFlagName, + common.DefaultWrapHelpString(fmt.Sprintf("Log level for additional details and debugging. Valid values: %s", strings.Join(flags.LogLevelEnum(), ", "))) //nolint:lll // For readability + ) common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") // TODO: Handle version viper.SetEnvPrefix("ASVEC") diff --git a/cmd/utils_test.go b/cmd/utils_test.go new file mode 100644 index 0000000..53c8a84 --- /dev/null +++ b/cmd/utils_test.go @@ -0,0 +1,49 @@ +//go:build unit + +package cmd + +import ( + "asvec/cmd/flags" + "testing" + + avs "github.com/aerospike/aerospike-proximus-client-go" + "github.com/stretchr/testify/assert" +) + +func TestParseBothHostSeedsFlag(t *testing.T) { + testCases := []struct { + seeds *flags.SeedsSliceFlag + host *flags.HostPortFlag + expectedSlice avs.HostPortSlice + expectedIsLoadBalancer bool + }{ + { + &flags.SeedsSliceFlag{ + Seeds: avs.HostPortSlice{ + avs.NewHostPort("1.1.1.1", 5000, false), + }, + }, + flags.NewDefaultHostPortFlag(), + avs.HostPortSlice{ + avs.NewHostPort("1.1.1.1", 5000, false), + }, + false, + }, + { + &flags.SeedsSliceFlag{ + Seeds: avs.HostPortSlice{}, + }, + flags.NewDefaultHostPortFlag(), + avs.HostPortSlice{ + &flags.NewDefaultHostPortFlag().HostPort, + }, + true, + }, + } + + for _, tc := range testCases { + actualSlice, actualBool := parseBothHostSeedsFlag(tc.seeds, tc.host) + assert.Equal(t, tc.expectedSlice, actualSlice) + assert.Equal(t, tc.expectedIsLoadBalancer, actualBool) + } +} From 9110d0841ef017f6cd7cb713f6d6c6a441ed03d4 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Thu, 13 Jun 2024 13:25:39 -0700 Subject: [PATCH 32/35] fix --- cmd/root.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 8305e98..28bb576 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -87,9 +87,9 @@ func Execute() { func init() { rootCmd.PersistentFlags().Var( - &rootFlags.logLevel, - logLevelFlagName, - common.DefaultWrapHelpString(fmt.Sprintf("Log level for additional details and debugging. Valid values: %s", strings.Join(flags.LogLevelEnum(), ", "))) //nolint:lll // For readability + &rootFlags.logLevel, + logLevelFlagName, + common.DefaultWrapHelpString(fmt.Sprintf("Log level for additional details and debugging. Valid values: %s", strings.Join(flags.LogLevelEnum(), ", "))), //nolint:lll // For readability ) common.SetupRoot(rootCmd, "aerospike-vector-search", "0.0.0") // TODO: Handle version viper.SetEnvPrefix("ASVEC") From 6225d45843e4ab5794e530c38977dd29b0c532db Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Thu, 13 Jun 2024 14:26:09 -0700 Subject: [PATCH 33/35] fix tests --- cmd/createIndex.go | 2 +- cmd/dropIndex.go | 2 +- cmd/listIndex.go | 2 +- cmd/utils.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/createIndex.go b/cmd/createIndex.go index 20ed67f..150086a 100644 --- a/cmd/createIndex.go +++ b/cmd/createIndex.go @@ -110,7 +110,7 @@ func newCreateIndexCmd() *cobra.Command { return nil }, RunE: func(_ *cobra.Command, _ []string) error { - hosts, isLoadBalancer := parseBothHostSeedsFlag(*createIndexFlags.seeds, *createIndexFlags.host) + hosts, isLoadBalancer := parseBothHostSeedsFlag(createIndexFlags.seeds, createIndexFlags.host) logger.Debug("parsed flags", slog.String(flagNameHost, createIndexFlags.host.String()), diff --git a/cmd/dropIndex.go b/cmd/dropIndex.go index 39907df..65b699c 100644 --- a/cmd/dropIndex.go +++ b/cmd/dropIndex.go @@ -79,7 +79,7 @@ func newDropIndexCommand() *cobra.Command { slog.Duration(flagNameTimeout, dropIndexFlags.timeout), ) - hosts, isLoadBalancer := parseBothHostSeedsFlag(*dropIndexFlags.seeds, *dropIndexFlags.host) + hosts, isLoadBalancer := parseBothHostSeedsFlag(dropIndexFlags.seeds, dropIndexFlags.host) ctx, cancel := context.WithTimeout(context.Background(), dropIndexFlags.timeout) defer cancel() diff --git a/cmd/listIndex.go b/cmd/listIndex.go index 2b90ffc..dcd8ccc 100644 --- a/cmd/listIndex.go +++ b/cmd/listIndex.go @@ -71,7 +71,7 @@ func newListIndexCmd() *cobra.Command { slog.Duration(flagNameTimeout, listIndexFlags.timeout), ) - hosts, isLoadBalancer := parseBothHostSeedsFlag(*listIndexFlags.seeds, *listIndexFlags.host) + hosts, isLoadBalancer := parseBothHostSeedsFlag(listIndexFlags.seeds, listIndexFlags.host) ctx, cancel := context.WithTimeout(context.Background(), listIndexFlags.timeout) defer cancel() diff --git a/cmd/utils.go b/cmd/utils.go index 6a22f95..7e973d6 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -6,7 +6,7 @@ import ( avs "github.com/aerospike/aerospike-proximus-client-go" ) -func parseBothHostSeedsFlag(seeds flags.SeedsSliceFlag, host flags.HostPortFlag) (avs.HostPortSlice, bool) { +func parseBothHostSeedsFlag(seeds *flags.SeedsSliceFlag, host *flags.HostPortFlag) (avs.HostPortSlice, bool) { isLoadBalancer := false hosts := avs.HostPortSlice{} From 67e930aa5ffe8a405b0aca4f31555a8b583a9a08 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 2 Jul 2024 14:36:51 -0700 Subject: [PATCH 34/35] review changes --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++ Makefile | 4 +- README.md | 2 +- cmd/flags/hostPort.go | 2 +- e2e_test.go | 2 - 5 files changed, 205 insertions(+), 6 deletions(-) diff --git a/LICENSE b/LICENSE index e69de29..f49a4e1 100644 --- a/LICENSE +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/Makefile b/Makefile index e93b729..737fb92 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ Package: asvec Section: aerospike Version: ${ver} Architecture: amd64 -Description: Tool for deploying non-prod Aerospike server clusters on docker, GCP or in AWS +Description: Tool for managing Aerospike Vector Search clusters. EOF endef export amddebscript = $(value _amddebscript) @@ -182,7 +182,7 @@ Package: asvec Section: aerospike Version: ${ver} Architecture: arm64 -Description: Tool for deploying non-prod Aerospike server clusters on docker, GCP or in AWS +Description: Tool for managing Aerospike Vector Search clusters. EOF endef export armdebscript = $(value _armdebscript) diff --git a/README.md b/README.md index ba05b4b..e1e730a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Overview The Aerospike Vector Search (AVS) CLI Tool is designed to simplify the -management of AVS deployments. It offers a features aimed at enhancing +management of AVS deployments. It offers features aimed at enhancing efficiency and productivity for users getting started with vector search. ## Getting Started diff --git a/cmd/flags/hostPort.go b/cmd/flags/hostPort.go index 638dadf..36d7578 100644 --- a/cmd/flags/hostPort.go +++ b/cmd/flags/hostPort.go @@ -73,7 +73,7 @@ func parseHostPort(v string) (*avs.HostPort, error) { return host, fmt.Errorf("does not match any expected formats") } -// A cobra PFlag to parse and display help info for the host[:tls-name][:port] +// A cobra PFlag to parse and display help info for the host[:port] // input option. It implements the pflag Value and SliceValue interfaces to // enable automatic parsing by cobra. type HostPortFlag struct { diff --git a/e2e_test.go b/e2e_test.go index 7d8bf3f..cf8a23b 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -326,8 +326,6 @@ func (suite *CmdTestSuite) TestSuccessfulCreateIndexCmd() { actual, err := suite.avsClient.IndexGet(context.Background(), tc.indexNamespace, tc.indexName) - time.Sleep(time.Second) - if err != nil { suite.FailNowf("unable to get index", "%v", err) } From f7d1f224453a85fc104083008e302da379a2a488 Mon Sep 17 00:00:00 2001 From: Jesse Schmidt Date: Tue, 2 Jul 2024 14:40:54 -0700 Subject: [PATCH 35/35] fix license link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e1e730a..082b2e6 100644 --- a/README.md +++ b/README.md @@ -40,4 +40,4 @@ Otherwise, if you are an enterprise customer, please [contact support](https://a ## License -This project is licensed under the MIT License. See the [LICENSE.md](./LICENSE.md) file for details. +This project is licensed under the MIT License. See the [LICENSE.md](./LICENSE) file for details.