From 886f19d7a7ef0da3f9aed76869247c249974a54b Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 25 Sep 2023 16:41:52 -0400 Subject: [PATCH 1/5] migrate zk binary to cobra with a restructure of files Signed-off-by: Andrew Mason --- go/cmd/zk/command/add_auth.go | 36 + go/cmd/zk/command/cat.go | 103 ++ go/cmd/zk/command/chmod.go | 91 ++ go/cmd/zk/command/cp.go | 43 + go/cmd/zk/command/edit.go | 101 ++ go/cmd/zk/command/ls.go | 153 +++ go/cmd/zk/command/rm.go | 97 ++ go/cmd/zk/command/root.go | 66 ++ go/cmd/zk/command/stat.go | 88 ++ go/cmd/zk/command/touch.go | 93 ++ go/cmd/zk/command/unzip.go | 81 ++ go/cmd/zk/command/wait.go | 78 ++ go/cmd/zk/command/watch.go | 86 ++ go/cmd/zk/command/zip.go | 116 +++ go/cmd/zk/docgen/main.go | 37 + go/cmd/zk/internal/zkfilepath/zkfilepath.go | 75 ++ go/cmd/zk/internal/zkfs/zkfs.go | 174 ++++ go/cmd/zk/zkcmd.go | 982 +------------------- go/flags/endtoend/zk.txt | 37 +- 19 files changed, 1555 insertions(+), 982 deletions(-) create mode 100644 go/cmd/zk/command/add_auth.go create mode 100644 go/cmd/zk/command/cat.go create mode 100644 go/cmd/zk/command/chmod.go create mode 100644 go/cmd/zk/command/cp.go create mode 100644 go/cmd/zk/command/edit.go create mode 100644 go/cmd/zk/command/ls.go create mode 100644 go/cmd/zk/command/rm.go create mode 100644 go/cmd/zk/command/root.go create mode 100644 go/cmd/zk/command/stat.go create mode 100644 go/cmd/zk/command/touch.go create mode 100644 go/cmd/zk/command/unzip.go create mode 100644 go/cmd/zk/command/wait.go create mode 100644 go/cmd/zk/command/watch.go create mode 100644 go/cmd/zk/command/zip.go create mode 100644 go/cmd/zk/docgen/main.go create mode 100644 go/cmd/zk/internal/zkfilepath/zkfilepath.go create mode 100644 go/cmd/zk/internal/zkfs/zkfs.go diff --git a/go/cmd/zk/command/add_auth.go b/go/cmd/zk/command/add_auth.go new file mode 100644 index 00000000000..566c463f4a8 --- /dev/null +++ b/go/cmd/zk/command/add_auth.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "github.com/spf13/cobra" +) + +var AddAuth = &cobra.Command{ + Use: "addAuth ", + Args: cobra.ExactArgs(2), + RunE: commandAddAuth, +} + +func commandAddAuth(cmd *cobra.Command, args []string) error { + scheme, auth := cmd.Flags().Arg(0), cmd.Flags().Arg(1) + return fs.Conn.AddAuth(cmd.Context(), scheme, []byte(auth)) +} + +func init() { + Root.AddCommand(AddAuth) +} diff --git a/go/cmd/zk/command/cat.go b/go/cmd/zk/command/cat.go new file mode 100644 index 00000000000..1d5460f7006 --- /dev/null +++ b/go/cmd/zk/command/cat.go @@ -0,0 +1,103 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + "golang.org/x/term" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var ( + catArgs = struct { + LongListing bool + Force bool + DecodeProto bool + }{} + + Cat = &cobra.Command{ + Use: "cat [ ...]", + Example: `zk cat /zk/path + +# List filename before file data +zk cat -l /zk/path1 /zk/path2`, + Args: cobra.MinimumNArgs(1), + RunE: commandCat, + } +) + +func commandCat(cmd *cobra.Command, args []string) error { + resolved, err := zk2topo.ResolveWildcards(cmd.Context(), fs.Conn, cmd.Flags().Args()) + if err != nil { + return fmt.Errorf("cat: invalid wildcards: %w", err) + } + if len(resolved) == 0 { + // the wildcards didn't result in anything, we're done + return nil + } + + hasError := false + for _, arg := range resolved { + zkPath := zkfilepath.Clean(arg) + data, _, err := fs.Conn.Get(cmd.Context(), zkPath) + if err != nil { + hasError = true + if !catArgs.Force || err != zk.ErrNoNode { + log.Warningf("cat: cannot access %v: %v", zkPath, err) + } + continue + } + + if catArgs.LongListing { + fmt.Printf("%v:\n", zkPath) + } + decoded := "" + if catArgs.DecodeProto { + decoded, err = topo.DecodeContent(zkPath, data, false) + if err != nil { + log.Warningf("cat: cannot proto decode %v: %v", zkPath, err) + decoded = string(data) + } + } else { + decoded = string(data) + } + fmt.Print(decoded) + if len(decoded) > 0 && decoded[len(decoded)-1] != '\n' && (term.IsTerminal(int(os.Stdout.Fd())) || catArgs.LongListing) { + fmt.Print("\n") + } + } + if hasError { + return fmt.Errorf("cat: some paths had errors") + } + return nil +} + +func init() { + Cat.Flags().BoolVarP(&catArgs.LongListing, "longListing", "l", false, "long listing") + Cat.Flags().BoolVarP(&catArgs.Force, "force", "f", false, "no warning on nonexistent node") + Cat.Flags().BoolVarP(&catArgs.DecodeProto, "decodeProto", "p", false, "decode proto files and display them as text") + + Root.AddCommand(Cat) +} diff --git a/go/cmd/zk/command/chmod.go b/go/cmd/zk/command/chmod.go new file mode 100644 index 00000000000..39125d618c4 --- /dev/null +++ b/go/cmd/zk/command/chmod.go @@ -0,0 +1,91 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/cmd/zk/internal/zkfs" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var Chmod = &cobra.Command{ + Use: "chmod ", + Example: `zk chmod n-mode /zk/path +zk chmod n+mode /zk/path`, + Args: cobra.MinimumNArgs(2), + RunE: commandChmod, +} + +func commandChmod(cmd *cobra.Command, args []string) error { + mode := cmd.Flags().Arg(0) + if mode[0] != 'n' { + return fmt.Errorf("chmod: invalid mode") + } + + addPerms := false + if mode[1] == '+' { + addPerms = true + } else if mode[1] != '-' { + return fmt.Errorf("chmod: invalid mode") + } + + permMask := zkfs.ParsePermMode(mode[2:]) + + resolved, err := zk2topo.ResolveWildcards(cmd.Context(), fs.Conn, cmd.Flags().Args()[1:]) + if err != nil { + return fmt.Errorf("chmod: invalid wildcards: %w", err) + } + if len(resolved) == 0 { + // the wildcards didn't result in anything, we're done + return nil + } + + hasError := false + for _, arg := range resolved { + zkPath := zkfilepath.Clean(arg) + aclv, _, err := fs.Conn.GetACL(cmd.Context(), zkPath) + if err != nil { + hasError = true + log.Warningf("chmod: cannot set access %v: %v", zkPath, err) + continue + } + if addPerms { + aclv[0].Perms |= permMask + } else { + aclv[0].Perms &= ^permMask + } + err = fs.Conn.SetACL(cmd.Context(), zkPath, aclv, -1) + if err != nil { + hasError = true + log.Warningf("chmod: cannot set access %v: %v", zkPath, err) + continue + } + } + if hasError { + return fmt.Errorf("chmod: some paths had errors") + } + return nil +} + +func init() { + Root.AddCommand(Chmod) +} diff --git a/go/cmd/zk/command/cp.go b/go/cmd/zk/command/cp.go new file mode 100644 index 00000000000..e89486413ea --- /dev/null +++ b/go/cmd/zk/command/cp.go @@ -0,0 +1,43 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import "github.com/spf13/cobra" + +var Cp = &cobra.Command{ + Use: "cp ", + Example: `zk cp /zk/path . +zk cp ./config /zk/path/config + +# Trailing slash indicates directory +zk cp ./config /zk/path/`, + Args: cobra.MinimumNArgs(2), + RunE: commandCp, +} + +func commandCp(cmd *cobra.Command, args []string) error { + switch cmd.Flags().NArg() { + case 2: + return fs.CopyContext(cmd.Context(), cmd.Flags().Arg(0), cmd.Flags().Arg(1)) + default: + return fs.MultiCopyContext(cmd.Context(), cmd.Flags().Args()) + } +} + +func init() { + Root.AddCommand(Cp) +} diff --git a/go/cmd/zk/command/edit.go b/go/cmd/zk/command/edit.go new file mode 100644 index 00000000000..ec4b74c4b62 --- /dev/null +++ b/go/cmd/zk/command/edit.go @@ -0,0 +1,101 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path" + "time" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/vt/log" +) + +var ( + editArgs = struct { + Force bool + }{} + + Edit = &cobra.Command{ + Use: "edit ", + Short: "Create a local copy, edit, and write changes back to cell.", + Args: cobra.ExactArgs(1), + RunE: commandEdit, + } +) + +func commandEdit(cmd *cobra.Command, args []string) error { + arg := cmd.Flags().Arg(0) + zkPath := zkfilepath.Clean(arg) + data, stat, err := fs.Conn.Get(cmd.Context(), zkPath) + if err != nil { + if !editArgs.Force || err != zk.ErrNoNode { + log.Warningf("edit: cannot access %v: %v", zkPath, err) + } + return fmt.Errorf("edit: cannot access %v: %v", zkPath, err) + } + + name := path.Base(zkPath) + tmpPath := fmt.Sprintf("/tmp/zk-edit-%v-%v", name, time.Now().UnixNano()) + f, err := os.Create(tmpPath) + if err == nil { + _, err = f.Write(data) + f.Close() + } + if err != nil { + return fmt.Errorf("edit: cannot write file %v", err) + } + + editor := exec.Command(os.Getenv("EDITOR"), tmpPath) + editor.Stdin = os.Stdin + editor.Stdout = os.Stdout + editor.Stderr = os.Stderr + err = editor.Run() + if err != nil { + os.Remove(tmpPath) + return fmt.Errorf("edit: cannot start $EDITOR: %v", err) + } + + fileData, err := os.ReadFile(tmpPath) + if err != nil { + os.Remove(tmpPath) + return fmt.Errorf("edit: cannot read file %v", err) + } + + if !bytes.Equal(fileData, data) { + // data changed - update if we can + _, err = fs.Conn.Set(cmd.Context(), zkPath, fileData, stat.Version) + if err != nil { + os.Remove(tmpPath) + return fmt.Errorf("edit: cannot write zk file %v", err) + } + } + os.Remove(tmpPath) + return nil +} + +func init() { + Edit.Flags().BoolVarP(&editArgs.Force, "force", "f", false, "no warning on nonexistent node") + + Root.AddCommand(Edit) +} diff --git a/go/cmd/zk/command/ls.go b/go/cmd/zk/command/ls.go new file mode 100644 index 00000000000..83c1d31363b --- /dev/null +++ b/go/cmd/zk/command/ls.go @@ -0,0 +1,153 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + "path" + "sort" + "sync" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var ( + lsArgs = struct { + LongListing bool + DirectoryListing bool + Force bool + RecursiveListing bool + }{} + + Ls = &cobra.Command{ + Use: "ls ", + Example: `zk ls /zk +zk ls -l /zk + +# List directory node itself) +zk ls -ld /zk + +# Recursive (expensive) +zk ls -R /zk`, + Args: cobra.MinimumNArgs(1), + RunE: commandLs, + } +) + +func commandLs(cmd *cobra.Command, args []string) error { + resolved, err := zk2topo.ResolveWildcards(cmd.Context(), fs.Conn, cmd.Flags().Args()) + if err != nil { + return fmt.Errorf("ls: invalid wildcards: %v", err) + } + if len(resolved) == 0 { + // the wildcards didn't result in anything, we're + // done. + return nil + } + + hasError := false + needsHeader := len(resolved) > 1 && !lsArgs.DirectoryListing + for _, arg := range resolved { + zkPath := zkfilepath.Clean(arg) + var children []string + var err error + isDir := true + if lsArgs.DirectoryListing { + children = []string{""} + isDir = false + } else if lsArgs.RecursiveListing { + children, err = zk2topo.ChildrenRecursive(cmd.Context(), fs.Conn, zkPath) + } else { + children, _, err = fs.Conn.Children(cmd.Context(), zkPath) + // Assume this is a file node if it has no children. + if len(children) == 0 { + children = []string{""} + isDir = false + } + } + if err != nil { + hasError = true + if !lsArgs.Force || err != zk.ErrNoNode { + log.Warningf("ls: cannot access %v: %v", zkPath, err) + } + } + + // Show the full path when it helps. + showFullPath := false + if lsArgs.RecursiveListing { + showFullPath = true + } else if lsArgs.LongListing && (lsArgs.DirectoryListing || !isDir) { + showFullPath = true + } + if needsHeader { + fmt.Printf("%v:\n", zkPath) + } + if len(children) > 0 { + if lsArgs.LongListing && isDir { + fmt.Printf("total: %v\n", len(children)) + } + sort.Strings(children) + stats := make([]*zk.Stat, len(children)) + wg := sync.WaitGroup{} + f := func(i int) { + localPath := path.Join(zkPath, children[i]) + _, stat, err := fs.Conn.Exists(cmd.Context(), localPath) + if err != nil { + if !lsArgs.Force || err != zk.ErrNoNode { + log.Warningf("ls: cannot access: %v: %v", localPath, err) + } + } else { + stats[i] = stat + } + wg.Done() + } + for i := range children { + wg.Add(1) + go f(i) + } + wg.Wait() + + for i, child := range children { + localPath := path.Join(zkPath, child) + if stat := stats[i]; stat != nil { + fmt.Println(zkfilepath.Format(stat, localPath, showFullPath, lsArgs.LongListing)) + } + } + } + if needsHeader { + fmt.Println() + } + } + if hasError { + return fmt.Errorf("ls: some paths had errors") + } + return nil +} + +func init() { + Ls.Flags().BoolVarP(&lsArgs.LongListing, "longlisting", "l", false, "long listing") + Ls.Flags().BoolVarP(&lsArgs.DirectoryListing, "directorylisting", "d", false, "list directory instead of contents") + Ls.Flags().BoolVarP(&lsArgs.Force, "force", "f", false, "no warning on nonexistent node") + Ls.Flags().BoolVarP(&lsArgs.RecursiveListing, "recursivelisting", "R", false, "recursive listing") + + Root.AddCommand(Ls) +} diff --git a/go/cmd/zk/command/rm.go b/go/cmd/zk/command/rm.go new file mode 100644 index 00000000000..5e5b5f4c494 --- /dev/null +++ b/go/cmd/zk/command/rm.go @@ -0,0 +1,97 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var ( + rmArgs = struct { + Force bool + RecursiveDelete bool + }{} + + Rm = &cobra.Command{ + Use: "rm ", + Example: `zk rm /zk/path + +# Recursive. +zk rm -R /zk/path + +# No error on nonexistent node. +zk rm -f /zk/path`, + Args: cobra.MinimumNArgs(1), + RunE: commandRm, + } +) + +func commandRm(cmd *cobra.Command, args []string) error { + if rmArgs.RecursiveDelete { + for _, arg := range cmd.Flags().Args() { + zkPath := zkfilepath.Clean(arg) + if strings.Count(zkPath, "/") < 2 { + return fmt.Errorf("rm: overly general path: %v", zkPath) + } + } + } + + resolved, err := zk2topo.ResolveWildcards(cmd.Context(), fs.Conn, cmd.Flags().Args()) + if err != nil { + return fmt.Errorf("rm: invalid wildcards: %v", err) + } + if len(resolved) == 0 { + // the wildcards didn't result in anything, we're done + return nil + } + + hasError := false + for _, arg := range resolved { + zkPath := zkfilepath.Clean(arg) + var err error + if rmArgs.RecursiveDelete { + err = zk2topo.DeleteRecursive(cmd.Context(), fs.Conn, zkPath, -1) + } else { + err = fs.Conn.Delete(cmd.Context(), zkPath, -1) + } + if err != nil && (!rmArgs.Force || err != zk.ErrNoNode) { + hasError = true + log.Warningf("rm: cannot delete %v: %v", zkPath, err) + } + } + if hasError { + // to be consistent with the command line 'rm -f', return + // 0 if using 'zk rm -f' and the file doesn't exist. + return fmt.Errorf("rm: some paths had errors") + } + return nil +} + +func init() { + Rm.Flags().BoolVarP(&rmArgs.Force, "force", "f", false, "no warning on nonexistent node") + Rm.Flags().BoolVarP(&rmArgs.RecursiveDelete, "recursivedelete", "r", false, "recursive delete") + + Root.AddCommand(Rm) +} diff --git a/go/cmd/zk/command/root.go b/go/cmd/zk/command/root.go new file mode 100644 index 00000000000..30c2a93fd0a --- /dev/null +++ b/go/cmd/zk/command/root.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/cmd/zk/internal/zkfs" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var ( + fs *zkfs.FS + server string + + Root = &cobra.Command{ + Use: "zk", + Short: "zk is a tool for wrangling the zookeeper.", + Long: `zk is a tool for wrangling the zookeeper. + +It tries to mimic unix file system commands wherever possible, but +there are some slight differences in flag handling. + +The zk tool looks for the address of the cluster in /etc/zookeeper/zk_client.conf, +or the file specified in the ZK_CLIENT_CONFIG environment variable. + +The local cell may be overridden with the ZK_CLIENT_LOCAL_CELL environment +variable.`, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + logutil.PurgeLogs() + + // Connect to the server. + fs = &zkfs.FS{ + Conn: zk2topo.Connect(server), + } + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + logutil.Flush() + }, + } +) + +func init() { + Root.Flags().StringVar(&server, "server", server, "server(s) to connect to") + + log.RegisterFlags(Root.Flags()) + logutil.RegisterFlags(Root.Flags()) + acl.RegisterFlags(Root.Flags()) +} diff --git a/go/cmd/zk/command/stat.go b/go/cmd/zk/command/stat.go new file mode 100644 index 00000000000..713a68a3d4e --- /dev/null +++ b/go/cmd/zk/command/stat.go @@ -0,0 +1,88 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/cmd/zk/internal/zkfs" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var ( + statArgs = struct { + Force bool + }{} + Stat = &cobra.Command{ + Use: "stat ", + Args: cobra.MinimumNArgs(1), + RunE: commandStat, + } +) + +func commandStat(cmd *cobra.Command, args []string) error { + resolved, err := zk2topo.ResolveWildcards(cmd.Context(), fs.Conn, cmd.Flags().Args()) + if err != nil { + return fmt.Errorf("stat: invalid wildcards: %v", err) + } + if len(resolved) == 0 { + // the wildcards didn't result in anything, we're done + return nil + } + + hasError := false + for _, arg := range resolved { + zkPath := zkfilepath.Clean(arg) + acls, stat, err := fs.Conn.GetACL(cmd.Context(), zkPath) + if stat == nil { + err = fmt.Errorf("no such node") + } + if err != nil { + hasError = true + if !statArgs.Force || err != zk.ErrNoNode { + log.Warningf("stat: cannot access %v: %v", zkPath, err) + } + continue + } + fmt.Printf("Path: %s\n", zkPath) + fmt.Printf("Created: %s\n", zk2topo.Time(stat.Ctime).Format(zkfilepath.TimeFmtMicro)) + fmt.Printf("Modified: %s\n", zk2topo.Time(stat.Mtime).Format(zkfilepath.TimeFmtMicro)) + fmt.Printf("Size: %v\n", stat.DataLength) + fmt.Printf("Children: %v\n", stat.NumChildren) + fmt.Printf("Version: %v\n", stat.Version) + fmt.Printf("Ephemeral: %v\n", stat.EphemeralOwner) + fmt.Printf("ACL:\n") + for _, acl := range acls { + fmt.Printf(" %v:%v %v\n", acl.Scheme, acl.ID, zkfs.FormatACL(acl)) + } + } + if hasError { + return fmt.Errorf("stat: some paths had errors") + } + return nil +} + +func init() { + Stat.Flags().BoolVarP(&statArgs.Force, "force", "f", false, "no warning on nonexistent node") + + Root.AddCommand(Stat) +} diff --git a/go/cmd/zk/command/touch.go b/go/cmd/zk/command/touch.go new file mode 100644 index 00000000000..76c390cf169 --- /dev/null +++ b/go/cmd/zk/command/touch.go @@ -0,0 +1,93 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var ( + touchArgs = struct { + CreateParents bool + TouchOnly bool + }{} + + Touch = &cobra.Command{ + Use: "touch ", + Short: "Change node access time.", + Long: `Change node access time. + +NOTE: There is no mkdir - just touch a node. +The disntinction between file and directory is not relevant in zookeeper.`, + Example: `zk touch /zk/path + +# Don't create, just touch timestamp. +zk touch -c /zk/path + +# Create all parts necessary (think mkdir -p). +zk touch -p /zk/path`, + Args: cobra.ExactArgs(1), + RunE: commandTouch, + } +) + +func commandTouch(cmd *cobra.Command, args []string) error { + zkPath := zkfilepath.Clean(cmd.Flags().Arg(0)) + var ( + version int32 = -1 + create = false + ) + + data, stat, err := fs.Conn.Get(cmd.Context(), zkPath) + switch { + case err == nil: + version = stat.Version + case err == zk.ErrNoNode: + create = true + default: + return fmt.Errorf("touch: cannot access %v: %v", zkPath, err) + } + + switch { + case !create: + _, err = fs.Conn.Set(cmd.Context(), zkPath, data, version) + case touchArgs.TouchOnly: + return fmt.Errorf("touch: no such path %v", zkPath) + case touchArgs.CreateParents: + _, err = zk2topo.CreateRecursive(cmd.Context(), fs.Conn, zkPath, data, 0, zk.WorldACL(zk.PermAll), 10) + default: + _, err = fs.Conn.Create(cmd.Context(), zkPath, data, 0, zk.WorldACL(zk.PermAll)) + } + + if err != nil { + return fmt.Errorf("touch: cannot modify %v: %v", zkPath, err) + } + return nil +} + +func init() { + Touch.Flags().BoolVarP(&touchArgs.CreateParents, "createparent", "p", false, "create parents") + Touch.Flags().BoolVarP(&touchArgs.TouchOnly, "touchonly", "c", false, "touch only - don't create") + + Root.AddCommand(Touch) +} diff --git a/go/cmd/zk/command/unzip.go b/go/cmd/zk/command/unzip.go new file mode 100644 index 00000000000..f4c800e0533 --- /dev/null +++ b/go/cmd/zk/command/unzip.go @@ -0,0 +1,81 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "archive/zip" + "fmt" + "io" + "path" + "strings" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var Unzip = &cobra.Command{ + Use: "unzip ", + Example: `zk unzip zktree.zip / +zk unzip zktree.zip /zk/prefix`, + Args: cobra.ExactArgs(1), + RunE: commandUnzip, +} + +func commandUnzip(cmd *cobra.Command, args []string) error { + srcPath, dstPath := cmd.Flags().Arg(0), cmd.Flags().Arg(1) + + if !strings.HasSuffix(srcPath, ".zip") { + return fmt.Errorf("zip: need to specify src .zip path: %v", srcPath) + } + + zipReader, err := zip.OpenReader(srcPath) + if err != nil { + return fmt.Errorf("zip: error %v", err) + } + defer zipReader.Close() + + for _, zf := range zipReader.File { + rc, err := zf.Open() + if err != nil { + return fmt.Errorf("unzip: error %v", err) + } + data, err := io.ReadAll(rc) + if err != nil { + return fmt.Errorf("unzip: failed reading archive: %v", err) + } + zkPath := zf.Name + if dstPath != "/" { + zkPath = path.Join(dstPath, zkPath) + } + _, err = zk2topo.CreateRecursive(cmd.Context(), fs.Conn, zkPath, data, 0, zk.WorldACL(zk.PermAll), 10) + if err != nil && err != zk.ErrNodeExists { + return fmt.Errorf("unzip: zk create failed: %v", err) + } + _, err = fs.Conn.Set(cmd.Context(), zkPath, data, -1) + if err != nil { + return fmt.Errorf("unzip: zk set failed: %v", err) + } + rc.Close() + } + return nil +} + +func init() { + Root.AddCommand(Unzip) +} diff --git a/go/cmd/zk/command/wait.go b/go/cmd/zk/command/wait.go new file mode 100644 index 00000000000..864f6e83626 --- /dev/null +++ b/go/cmd/zk/command/wait.go @@ -0,0 +1,78 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" +) + +var ( + waitArgs = struct { + ExitIfExists bool + }{} + + Wait = &cobra.Command{ + Use: "wait ", + Short: "Sets a watch on the node and then waits for an event to fire.", + Example: ` # Wait for node change or creation. +zk wait /zk/path + +# Trailing slash waits on children. +zk wait /zk/path/children/`, + Args: cobra.ExactArgs(1), + RunE: commandWait, + } +) + +func commandWait(cmd *cobra.Command, args []string) error { + zkPath := cmd.Flags().Arg(0) + isDir := zkPath[len(zkPath)-1] == '/' + zkPath = zkfilepath.Clean(zkPath) + + var wait <-chan zk.Event + var err error + if isDir { + _, _, wait, err = fs.Conn.ChildrenW(cmd.Context(), zkPath) + } else { + _, _, wait, err = fs.Conn.GetW(cmd.Context(), zkPath) + } + if err != nil { + if err == zk.ErrNoNode { + _, _, wait, _ = fs.Conn.ExistsW(cmd.Context(), zkPath) + } else { + return fmt.Errorf("wait: error %v: %v", zkPath, err) + } + } else { + if waitArgs.ExitIfExists { + return fmt.Errorf("already exists: %v", zkPath) + } + } + event := <-wait + fmt.Printf("event: %v\n", event) + return nil +} + +func init() { + Wait.Flags().BoolVarP(&waitArgs.ExitIfExists, "exit", "e", false, "exit if the path already exists") + + Root.AddCommand(Wait) +} diff --git a/go/cmd/zk/command/watch.go b/go/cmd/zk/command/watch.go new file mode 100644 index 00000000000..eb28cc29ca2 --- /dev/null +++ b/go/cmd/zk/command/watch.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/vt/log" +) + +var Watch = &cobra.Command{ + Use: "watch ", + Short: "Watches for changes to nodes and prints events as they occur.", + Example: `watch /zk/path`, + Args: cobra.MinimumNArgs(1), + RunE: commandWatch, +} + +func commandWatch(cmd *cobra.Command, args []string) error { + eventChan := make(chan zk.Event, 16) + for _, arg := range cmd.Flags().Args() { + zkPath := zkfilepath.Clean(arg) + _, _, watch, err := fs.Conn.GetW(cmd.Context(), zkPath) + if err != nil { + return fmt.Errorf("watch error: %v", err) + } + go func() { + eventChan <- <-watch + }() + } + + for { + select { + case <-cmd.Context().Done(): + return nil + case event := <-eventChan: + log.Infof("watch: event %v: %v", event.Path, event) + if event.Type == zk.EventNodeDataChanged { + data, stat, watch, err := fs.Conn.GetW(cmd.Context(), event.Path) + if err != nil { + return fmt.Errorf("ERROR: failed to watch %v", err) + } + log.Infof("watch: %v %v\n", event.Path, stat) + println(data) + go func() { + eventChan <- <-watch + }() + } else if event.State == zk.StateDisconnected { + return nil + } else if event.Type == zk.EventNodeDeleted { + log.Infof("watch: %v deleted\n", event.Path) + } else { + // Most likely a session event - try t + _, _, watch, err := fs.Conn.GetW(cmd.Context(), event.Path) + if err != nil { + return fmt.Errorf("ERROR: failed to watch %v", err) + } + go func() { + eventChan <- <-watch + }() + } + } + } +} + +func init() { + Root.AddCommand(Watch) +} diff --git a/go/cmd/zk/command/zip.go b/go/cmd/zk/command/zip.go new file mode 100644 index 00000000000..b765f5bb00e --- /dev/null +++ b/go/cmd/zk/command/zip.go @@ -0,0 +1,116 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "archive/zip" + "fmt" + "os" + "path" + "strings" + "sync" + + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/cmd/zk/internal/zkfs" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +var Zip = &cobra.Command{ + Use: "zip [ ...] ", + Short: "Store a zk tree in a zip archive.", + Long: `Store a zk tree in a zip archive. + +Note this won't be immediately useful to the local filesystem since znodes can have data and children; +that is, even "directories" can contain data.`, + Args: cobra.MinimumNArgs(2), + RunE: commandZip, +} + +func commandZip(cmd *cobra.Command, args []string) error { + posargs := cmd.Flags().Args() + dstPath := posargs[len(posargs)-1] + paths := posargs[:len(posargs)-1] + if !strings.HasSuffix(dstPath, ".zip") { + return fmt.Errorf("zip: need to specify destination .zip path: %v", dstPath) + } + zipFile, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("zip: error %v", err) + } + + wg := sync.WaitGroup{} + items := make(chan *zkfs.Item, 64) + for _, arg := range paths { + zkPath := zkfilepath.Clean(arg) + children, err := zk2topo.ChildrenRecursive(cmd.Context(), fs.Conn, zkPath) + if err != nil { + return fmt.Errorf("zip: error %v", err) + } + for _, child := range children { + toAdd := path.Join(zkPath, child) + wg.Add(1) + go func() { + data, stat, err := fs.Conn.Get(cmd.Context(), toAdd) + items <- &zkfs.Item{ + Path: toAdd, + Data: data, + Stat: stat, + Err: err, + } + wg.Done() + }() + } + } + go func() { + wg.Wait() + close(items) + }() + + zipWriter := zip.NewWriter(zipFile) + for item := range items { + path, data, stat, err := item.Path, item.Data, item.Stat, item.Err + if err != nil { + return fmt.Errorf("zip: get failed: %v", err) + } + // Skip ephemerals - not sure why you would archive them. + if stat.EphemeralOwner > 0 { + continue + } + fi := &zip.FileHeader{Name: path, Method: zip.Deflate} + fi.Modified = zk2topo.Time(stat.Mtime) + f, err := zipWriter.CreateHeader(fi) + if err != nil { + return fmt.Errorf("zip: create failed: %v", err) + } + _, err = f.Write(data) + if err != nil { + return fmt.Errorf("zip: create failed: %v", err) + } + } + err = zipWriter.Close() + if err != nil { + return fmt.Errorf("zip: close failed: %v", err) + } + zipFile.Close() + return nil +} + +func init() { + Root.AddCommand(Zip) +} diff --git a/go/cmd/zk/docgen/main.go b/go/cmd/zk/docgen/main.go new file mode 100644 index 00000000000..b8a7bde3d14 --- /dev/null +++ b/go/cmd/zk/docgen/main.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/internal/docgen" + "vitess.io/vitess/go/cmd/zk/command" +) + +func main() { + var dir string + cmd := cobra.Command{ + Use: "docgen [-d ]", + RunE: func(cmd *cobra.Command, args []string) error { + return docgen.GenerateMarkdownTree(command.Root, dir) + }, + } + + cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation") + _ = cmd.Execute() +} diff --git a/go/cmd/zk/internal/zkfilepath/zkfilepath.go b/go/cmd/zk/internal/zkfilepath/zkfilepath.go new file mode 100644 index 00000000000..7febc7a9677 --- /dev/null +++ b/go/cmd/zk/internal/zkfilepath/zkfilepath.go @@ -0,0 +1,75 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package zkfilepath provides filepath utilities specialized to zookeeper. +package zkfilepath + +import ( + "fmt" + "path" + "strings" + + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +const ( + TimeFmt = "2006-01-02 15:04:05" + TimeFmtMicro = "2006-01-02 15:04:05.000000" +) + +// Clean returns the shortest path name of a zookeeper path after trimming +// trailing slashes. +func Clean(zkPath string) string { + if zkPath != "/" { + zkPath = strings.TrimSuffix(zkPath, "/") + } + + return path.Clean(zkPath) +} + +// Format returns a path formatted to a canonical string. +func Format(stat *zk.Stat, zkPath string, showFullPath bool, longListing bool) string { + var name, perms string + + if !showFullPath { + name = path.Base(zkPath) + } else { + name = zkPath + } + + if longListing { + if stat.NumChildren > 0 { + // FIXME(msolomon) do permissions check? + perms = "drwxrwxrwx" + if stat.DataLength > 0 { + // give a visual indication that this node has data as well as children + perms = "nrw-rw-rw-" + } + } else if stat.EphemeralOwner != 0 { + perms = "erw-rw-rw-" + } else { + perms = "-rw-rw-rw-" + } + // always print the Local version of the time. zookeeper's + // go / C library would return a local time anyway, but + // might as well be sure. + return fmt.Sprintf("%v %v %v % 8v % 20v %v\n", perms, "zk", "zk", stat.DataLength, zk2topo.Time(stat.Mtime).Local().Format(TimeFmt), name) + } else { + return fmt.Sprintf("%v\n", name) + } +} diff --git a/go/cmd/zk/internal/zkfs/zkfs.go b/go/cmd/zk/internal/zkfs/zkfs.go new file mode 100644 index 00000000000..9bab19ec1e4 --- /dev/null +++ b/go/cmd/zk/internal/zkfs/zkfs.go @@ -0,0 +1,174 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package zkfs provides utilities for working with zookeepr in a filesystem-like manner. +package zkfs + +import ( + "context" + "fmt" + "io" + "os" + "path" + "strings" + "syscall" + + "github.com/z-division/go-zookeeper/zk" + + "vitess.io/vitess/go/cmd/zk/internal/zkfilepath" + "vitess.io/vitess/go/vt/topo/zk2topo" +) + +// FS wraps a zk2topo connection to provide FS utility methods. +type FS struct { + Conn *zk2topo.ZkConn +} + +// CopyContext copies the contents of src to dst. +func (fs *FS) CopyContext(ctx context.Context, src, dst string) error { + dstIsDir := dst[len(dst)-1] == '/' + src = zkfilepath.Clean(src) + dst = zkfilepath.Clean(dst) + + if !IsFile(src) && !IsFile(dst) { + return fmt.Errorf("cp: neither src nor dst is a /zk file") + } + + data, err := fs.ReadContext(ctx, src) + if err != nil { + return fmt.Errorf("cp: cannot read %v: %v", src, err) + } + + // If we are copying to a local directory - say '.', make the filename + // the same as the source. + if !IsFile(dst) { + fileInfo, err := os.Stat(dst) + if err != nil { + if err.(*os.PathError).Err != syscall.ENOENT { + return fmt.Errorf("cp: cannot stat %v: %v", dst, err) + } + } else if fileInfo.IsDir() { + dst = path.Join(dst, path.Base(src)) + } + } else if dstIsDir { + // If we are copying into zk, interpret trailing slash as treating the + // dst as a directory. + dst = path.Join(dst, path.Base(src)) + } + if err := fs.WriteContext(ctx, dst, data); err != nil { + return fmt.Errorf("cp: cannot write %v: %v", dst, err) + } + return nil +} + +// MultiCopyContext copies the contents of multiple sources to a single dst directory. +func (fs *FS) MultiCopyContext(ctx context.Context, args []string) error { + dstPath := args[len(args)-1] + if dstPath[len(dstPath)-1] != '/' { + // In multifile context, dstPath must be a directory. + dstPath += "/" + } + + for _, srcPath := range args[:len(args)-1] { + if err := fs.CopyContext(ctx, srcPath, dstPath); err != nil { + return err + } + } + return nil +} + +// ReadContext reads the data stored at path. +func (fs *FS) ReadContext(ctx context.Context, path string) (data []byte, err error) { + if !IsFile(path) { + data, _, err = fs.Conn.Get(ctx, path) + return data, err + } + + file, err := os.Open(path) + if err != nil { + return nil, err + } + + data, err = io.ReadAll(file) + return data, err +} + +// WriteContext writes the given data to path. +func (fs *FS) WriteContext(ctx context.Context, path string, data []byte) (err error) { + if IsFile(path) { + _, err = fs.Conn.Set(ctx, path, data, -1) + if err == zk.ErrNoNode { + _, err = zk2topo.CreateRecursive(ctx, fs.Conn, path, data, 0, zk.WorldACL(zk.PermAll), 10) + } + return err + } + return os.WriteFile(path, []byte(data), 0666) +} + +var ( + charPermMap map[string]int32 + permCharMap map[int32]string +) + +func init() { + charPermMap = map[string]int32{ + "r": zk.PermRead, + "w": zk.PermWrite, + "d": zk.PermDelete, + "c": zk.PermCreate, + "a": zk.PermAdmin, + } + permCharMap = make(map[int32]string) + for c, p := range charPermMap { + permCharMap[p] = c + } +} + +// FormatACL returns a string representation of a zookeeper ACL permission. +func FormatACL(acl zk.ACL) string { + s := "" + + for _, perm := range []int32{zk.PermRead, zk.PermWrite, zk.PermDelete, zk.PermCreate, zk.PermAdmin} { + if acl.Perms&perm != 0 { + s += permCharMap[perm] + } else { + s += "-" + } + } + return s +} + +// IsFile returns true if the path is a zk type of file. +func IsFile(path string) bool { + return strings.HasPrefix(path, "/zk") +} + +// ParsePermMode parses the mode string as a perm mask. +func ParsePermMode(mode string) (mask int32) { + for _, c := range mode[2:] { + mask |= charPermMap[string(c)] + } + + return mask +} + +// Item represents an item in a zookeeper filesystem. +type Item struct { + Path string + Data []byte + Stat *zk.Stat + Err error +} diff --git a/go/cmd/zk/zkcmd.go b/go/cmd/zk/zkcmd.go index 8d456f6b081..f03ac41c6ef 100644 --- a/go/cmd/zk/zkcmd.go +++ b/go/cmd/zk/zkcmd.go @@ -17,156 +17,17 @@ limitations under the License. package main import ( - "archive/zip" - "bytes" "context" - "fmt" - "io" "os" - "os/exec" "os/signal" - "path" - "sort" - "strings" - "sync" - "syscall" - "time" - "github.com/spf13/pflag" - "github.com/z-division/go-zookeeper/zk" - "golang.org/x/term" - - "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/cmd/zk/command" "vitess.io/vitess/go/exit" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/topo" - "vitess.io/vitess/go/vt/topo/zk2topo" -) - -var doc = ` -zk is a tool for wrangling the zookeeper - -It tries to mimic unix file system commands wherever possible, but -there are some slight differences in flag handling. - -zk -h - provide help on overriding cell selection - -zk addAuth digest user:pass - -zk cat /zk/path -zk cat -l /zk/path1 /zk/path2 (list filename before file data) - -zk chmod n-mode /zk/path -zk chmod n+mode /zk/path - -zk cp /zk/path . -zk cp ./config /zk/path/config -zk cp ./config /zk/path/ (trailing slash indicates directory) - -zk edit /zk/path (create a local copy, edit and write changes back to cell) - -zk ls /zk -zk ls -l /zk -zk ls -ld /zk (list directory node itself) -zk ls -R /zk (recursive, expensive) - -zk stat /zk/path - -zk touch /zk/path -zk touch -c /zk/path (don't create, just touch timestamp) -zk touch -p /zk/path (create all parts necessary, think mkdir -p) -NOTE: there is no mkdir - just touch a node. The distinction -between file and directory is just not relevant in zookeeper. - -zk rm /zk/path -zk rm -r /zk/path (recursive) -zk rm -f /zk/path (no error on nonexistent node) - -zk wait /zk/path (wait for node change or creation) -zk wait /zk/path/children/ (trailing slash waits on children) - -zk watch /zk/path (print changes) - -zk unzip zktree.zip / -zk unzip zktree.zip /zk/prefix - -zk zip /zk/root zktree.zip -NOTE: zip file can't be dumped to the file system since znodes -can have data and children. - -The zk tool looks for the address of the cluster in /etc/zookeeper/zk_client.conf, -or the file specified in the ZK_CLIENT_CONFIG environment variable. - -The local cell may be overridden with the ZK_CLIENT_LOCAL_CELL environment -variable. -` - -const ( - timeFmt = "2006-01-02 15:04:05" - timeFmtMicro = "2006-01-02 15:04:05.000000" ) -type cmdFunc func(ctx context.Context, subFlags *pflag.FlagSet, args []string) error - -var cmdMap map[string]cmdFunc -var zconn *zk2topo.ZkConn -var server string - -func init() { - cmdMap = map[string]cmdFunc{ - "addAuth": cmdAddAuth, - "cat": cmdCat, - "chmod": cmdChmod, - "cp": cmdCp, - "edit": cmdEdit, - "ls": cmdLs, - "rm": cmdRm, - "stat": cmdStat, - "touch": cmdTouch, - "unzip": cmdUnzip, - "wait": cmdWait, - "watch": cmdWatch, - "zip": cmdZip, - } -} - func main() { defer exit.Recover() - defer logutil.Flush() - pflag.StringVar(&server, "server", server, "server(s) to connect to") - // handling case of --help & -h - var help bool - pflag.BoolVarP(&help, "help", "h", false, "display usage and exit") - log.RegisterFlags(pflag.CommandLine) - logutil.RegisterFlags(pflag.CommandLine) - acl.RegisterFlags(pflag.CommandLine) - pflag.CommandLine.Usage = func() { - fmt.Fprint(os.Stderr, doc) - pflag.Usage() - } - - pflag.Parse() - logutil.PurgeLogs() - - if help || pflag.Arg(0) == "help" { - pflag.Usage() - os.Exit(0) - } - - // if no zk command is provided after --server then we need to print doc & usage both - args := pflag.Args() - if len(args) == 0 { - pflag.CommandLine.Usage() - exit.Return(1) - } - cmdName := args[0] - args = args[1:] - cmd, ok := cmdMap[cmdName] - if !ok { - log.Exitf("Unknown command %v", cmdName) - } - subFlags := pflag.NewFlagSet(cmdName, pflag.ContinueOnError) // Create a context for the command, cancel it if we get a signal. ctx, cancel := context.WithCancel(context.Background()) @@ -177,848 +38,9 @@ func main() { cancel() }() - // Connect to the server. - zconn = zk2topo.Connect(server) - // Run the command. - if err := cmd(ctx, subFlags, args); err != nil { + if err := command.Root.ExecuteContext(ctx); err != nil { log.Error(err) exit.Return(1) } } - -func fixZkPath(zkPath string) string { - if zkPath != "/" { - zkPath = strings.TrimSuffix(zkPath, "/") - } - return path.Clean(zkPath) -} - -func isZkFile(path string) bool { - return strings.HasPrefix(path, "/zk") -} - -func cmdWait(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - var exitIfExists bool - subFlags.BoolVarP(&exitIfExists, "exit", "e", false, "exit if the path already exists") - - if err := subFlags.Parse(args); err != nil { - return err - } - - if subFlags.NArg() != 1 { - return fmt.Errorf("wait: can only wait for one path") - } - zkPath := subFlags.Arg(0) - isDir := zkPath[len(zkPath)-1] == '/' - zkPath = fixZkPath(zkPath) - - var wait <-chan zk.Event - var err error - if isDir { - _, _, wait, err = zconn.ChildrenW(ctx, zkPath) - } else { - _, _, wait, err = zconn.GetW(ctx, zkPath) - } - if err != nil { - if err == zk.ErrNoNode { - _, _, wait, _ = zconn.ExistsW(ctx, zkPath) - } else { - return fmt.Errorf("wait: error %v: %v", zkPath, err) - } - } else { - if exitIfExists { - return fmt.Errorf("already exists: %v", zkPath) - } - } - event := <-wait - fmt.Printf("event: %v\n", event) - return nil -} - -// Watch for changes to the node. -func cmdWatch(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - if err := subFlags.Parse(args); err != nil { - return err - } - - eventChan := make(chan zk.Event, 16) - for _, arg := range subFlags.Args() { - zkPath := fixZkPath(arg) - _, _, watch, err := zconn.GetW(ctx, zkPath) - if err != nil { - return fmt.Errorf("watch error: %v", err) - } - go func() { - eventChan <- <-watch - }() - } - - for { - select { - case <-ctx.Done(): - return nil - case event := <-eventChan: - log.Infof("watch: event %v: %v", event.Path, event) - if event.Type == zk.EventNodeDataChanged { - data, stat, watch, err := zconn.GetW(ctx, event.Path) - if err != nil { - return fmt.Errorf("ERROR: failed to watch %v", err) - } - log.Infof("watch: %v %v\n", event.Path, stat) - println(data) - go func() { - eventChan <- <-watch - }() - } else if event.State == zk.StateDisconnected { - return nil - } else if event.Type == zk.EventNodeDeleted { - log.Infof("watch: %v deleted\n", event.Path) - } else { - // Most likely a session event - try t - _, _, watch, err := zconn.GetW(ctx, event.Path) - if err != nil { - return fmt.Errorf("ERROR: failed to watch %v", err) - } - go func() { - eventChan <- <-watch - }() - } - } - } -} - -func cmdLs(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - var ( - longListing bool - directoryListing bool - force bool - recursiveListing bool - ) - subFlags.BoolVarP(&longListing, "longlisting", "l", false, "long listing") - subFlags.BoolVarP(&directoryListing, "directorylisting", "d", false, "list directory instead of contents") - subFlags.BoolVarP(&force, "force", "f", false, "no warning on nonexistent node") - subFlags.BoolVarP(&recursiveListing, "recursivelisting", "R", false, "recursive listing") - - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() == 0 { - return fmt.Errorf("ls: no path specified") - } - resolved, err := zk2topo.ResolveWildcards(ctx, zconn, subFlags.Args()) - if err != nil { - return fmt.Errorf("ls: invalid wildcards: %v", err) - } - if len(resolved) == 0 { - // the wildcards didn't result in anything, we're - // done. - return nil - } - - hasError := false - needsHeader := len(resolved) > 1 && !directoryListing - for _, arg := range resolved { - zkPath := fixZkPath(arg) - var children []string - var err error - isDir := true - if directoryListing { - children = []string{""} - isDir = false - } else if recursiveListing { - children, err = zk2topo.ChildrenRecursive(ctx, zconn, zkPath) - } else { - children, _, err = zconn.Children(ctx, zkPath) - // Assume this is a file node if it has no children. - if len(children) == 0 { - children = []string{""} - isDir = false - } - } - if err != nil { - hasError = true - if !force || err != zk.ErrNoNode { - log.Warningf("ls: cannot access %v: %v", zkPath, err) - } - } - - // Show the full path when it helps. - showFullPath := false - if recursiveListing { - showFullPath = true - } else if longListing && (directoryListing || !isDir) { - showFullPath = true - } - if needsHeader { - fmt.Printf("%v:\n", zkPath) - } - if len(children) > 0 { - if longListing && isDir { - fmt.Printf("total: %v\n", len(children)) - } - sort.Strings(children) - stats := make([]*zk.Stat, len(children)) - wg := sync.WaitGroup{} - f := func(i int) { - localPath := path.Join(zkPath, children[i]) - _, stat, err := zconn.Exists(ctx, localPath) - if err != nil { - if !force || err != zk.ErrNoNode { - log.Warningf("ls: cannot access: %v: %v", localPath, err) - } - } else { - stats[i] = stat - } - wg.Done() - } - for i := range children { - wg.Add(1) - go f(i) - } - wg.Wait() - - for i, child := range children { - localPath := path.Join(zkPath, child) - if stat := stats[i]; stat != nil { - fmtPath(stat, localPath, showFullPath, longListing) - } - } - } - if needsHeader { - fmt.Println() - } - } - if hasError { - return fmt.Errorf("ls: some paths had errors") - } - return nil -} - -func fmtPath(stat *zk.Stat, zkPath string, showFullPath bool, longListing bool) { - var name, perms string - - if !showFullPath { - name = path.Base(zkPath) - } else { - name = zkPath - } - - if longListing { - if stat.NumChildren > 0 { - // FIXME(msolomon) do permissions check? - perms = "drwxrwxrwx" - if stat.DataLength > 0 { - // give a visual indication that this node has data as well as children - perms = "nrw-rw-rw-" - } - } else if stat.EphemeralOwner != 0 { - perms = "erw-rw-rw-" - } else { - perms = "-rw-rw-rw-" - } - // always print the Local version of the time. zookeeper's - // go / C library would return a local time anyway, but - // might as well be sure. - fmt.Printf("%v %v %v % 8v % 20v %v\n", perms, "zk", "zk", stat.DataLength, zk2topo.Time(stat.Mtime).Local().Format(timeFmt), name) - } else { - fmt.Printf("%v\n", name) - } -} - -func cmdTouch(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - var ( - createParents bool - touchOnly bool - ) - - subFlags.BoolVarP(&createParents, "createparent", "p", false, "create parents") - subFlags.BoolVarP(&touchOnly, "touchonly", "c", false, "touch only - don't create") - - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() != 1 { - return fmt.Errorf("touch: need to specify exactly one path") - } - - zkPath := fixZkPath(subFlags.Arg(0)) - - var ( - version int32 = -1 - create = false - ) - - data, stat, err := zconn.Get(ctx, zkPath) - switch { - case err == nil: - version = stat.Version - case err == zk.ErrNoNode: - create = true - default: - return fmt.Errorf("touch: cannot access %v: %v", zkPath, err) - } - - switch { - case !create: - _, err = zconn.Set(ctx, zkPath, data, version) - case touchOnly: - return fmt.Errorf("touch: no such path %v", zkPath) - case createParents: - _, err = zk2topo.CreateRecursive(ctx, zconn, zkPath, data, 0, zk.WorldACL(zk.PermAll), 10) - default: - _, err = zconn.Create(ctx, zkPath, data, 0, zk.WorldACL(zk.PermAll)) - } - - if err != nil { - return fmt.Errorf("touch: cannot modify %v: %v", zkPath, err) - } - return nil -} - -func cmdRm(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - var ( - force bool - recursiveDelete bool - ) - subFlags.BoolVarP(&force, "force", "f", false, "no warning on nonexistent node") - subFlags.BoolVarP(&recursiveDelete, "recursivedelete", "r", false, "recursive delete") - - if err := subFlags.Parse(args); err != nil { - return err - } - - if subFlags.NArg() == 0 { - return fmt.Errorf("rm: no path specified") - } - - if recursiveDelete { - for _, arg := range subFlags.Args() { - zkPath := fixZkPath(arg) - if strings.Count(zkPath, "/") < 2 { - return fmt.Errorf("rm: overly general path: %v", zkPath) - } - } - } - - resolved, err := zk2topo.ResolveWildcards(ctx, zconn, subFlags.Args()) - if err != nil { - return fmt.Errorf("rm: invalid wildcards: %v", err) - } - if len(resolved) == 0 { - // the wildcards didn't result in anything, we're done - return nil - } - - hasError := false - for _, arg := range resolved { - zkPath := fixZkPath(arg) - var err error - if recursiveDelete { - err = zk2topo.DeleteRecursive(ctx, zconn, zkPath, -1) - } else { - err = zconn.Delete(ctx, zkPath, -1) - } - if err != nil && (!force || err != zk.ErrNoNode) { - hasError = true - log.Warningf("rm: cannot delete %v: %v", zkPath, err) - } - } - if hasError { - // to be consistent with the command line 'rm -f', return - // 0 if using 'zk rm -f' and the file doesn't exist. - return fmt.Errorf("rm: some paths had errors") - } - return nil -} - -func cmdAddAuth(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() < 2 { - return fmt.Errorf("addAuth: expected args ") - } - scheme, auth := subFlags.Arg(0), subFlags.Arg(1) - return zconn.AddAuth(ctx, scheme, []byte(auth)) -} - -func cmdCat(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - var ( - longListing bool - force bool - decodeProto bool - ) - subFlags.BoolVarP(&longListing, "longListing", "l", false, "long listing") - subFlags.BoolVarP(&force, "force", "f", false, "no warning on nonexistent node") - subFlags.BoolVarP(&decodeProto, "decodeProto", "p", false, "decode proto files and display them as text") - - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() == 0 { - return fmt.Errorf("cat: no path specified") - } - resolved, err := zk2topo.ResolveWildcards(ctx, zconn, subFlags.Args()) - if err != nil { - return fmt.Errorf("cat: invalid wildcards: %v", err) - } - if len(resolved) == 0 { - // the wildcards didn't result in anything, we're done - return nil - } - - hasError := false - for _, arg := range resolved { - zkPath := fixZkPath(arg) - data, _, err := zconn.Get(ctx, zkPath) - if err != nil { - hasError = true - if !force || err != zk.ErrNoNode { - log.Warningf("cat: cannot access %v: %v", zkPath, err) - } - continue - } - - if longListing { - fmt.Printf("%v:\n", zkPath) - } - decoded := "" - if decodeProto { - decoded, err = topo.DecodeContent(zkPath, data, false) - if err != nil { - log.Warningf("cat: cannot proto decode %v: %v", zkPath, err) - decoded = string(data) - } - } else { - decoded = string(data) - } - fmt.Print(decoded) - if len(decoded) > 0 && decoded[len(decoded)-1] != '\n' && (term.IsTerminal(int(os.Stdout.Fd())) || longListing) { - fmt.Print("\n") - } - } - if hasError { - return fmt.Errorf("cat: some paths had errors") - } - return nil -} - -func cmdEdit(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - var force bool - subFlags.BoolVarP(&force, "force", "f", false, "no warning on nonexistent node") - - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() == 0 { - return fmt.Errorf("edit: no path specified") - } - arg := subFlags.Arg(0) - zkPath := fixZkPath(arg) - data, stat, err := zconn.Get(ctx, zkPath) - if err != nil { - if !force || err != zk.ErrNoNode { - log.Warningf("edit: cannot access %v: %v", zkPath, err) - } - return fmt.Errorf("edit: cannot access %v: %v", zkPath, err) - } - - name := path.Base(zkPath) - tmpPath := fmt.Sprintf("/tmp/zk-edit-%v-%v", name, time.Now().UnixNano()) - f, err := os.Create(tmpPath) - if err == nil { - _, err = f.Write(data) - f.Close() - } - if err != nil { - return fmt.Errorf("edit: cannot write file %v", err) - } - - cmd := exec.Command(os.Getenv("EDITOR"), tmpPath) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - os.Remove(tmpPath) - return fmt.Errorf("edit: cannot start $EDITOR: %v", err) - } - - fileData, err := os.ReadFile(tmpPath) - if err != nil { - os.Remove(tmpPath) - return fmt.Errorf("edit: cannot read file %v", err) - } - - if !bytes.Equal(fileData, data) { - // data changed - update if we can - _, err = zconn.Set(ctx, zkPath, fileData, stat.Version) - if err != nil { - os.Remove(tmpPath) - return fmt.Errorf("edit: cannot write zk file %v", err) - } - } - os.Remove(tmpPath) - return nil -} - -func cmdStat(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - var force bool - subFlags.BoolVarP(&force, "force", "f", false, "no warning on nonexistent node") - - if err := subFlags.Parse(args); err != nil { - return err - } - - if subFlags.NArg() == 0 { - return fmt.Errorf("stat: no path specified") - } - - resolved, err := zk2topo.ResolveWildcards(ctx, zconn, subFlags.Args()) - if err != nil { - return fmt.Errorf("stat: invalid wildcards: %v", err) - } - if len(resolved) == 0 { - // the wildcards didn't result in anything, we're done - return nil - } - - hasError := false - for _, arg := range resolved { - zkPath := fixZkPath(arg) - acls, stat, err := zconn.GetACL(ctx, zkPath) - if stat == nil { - err = fmt.Errorf("no such node") - } - if err != nil { - hasError = true - if !force || err != zk.ErrNoNode { - log.Warningf("stat: cannot access %v: %v", zkPath, err) - } - continue - } - fmt.Printf("Path: %s\n", zkPath) - fmt.Printf("Created: %s\n", zk2topo.Time(stat.Ctime).Format(timeFmtMicro)) - fmt.Printf("Modified: %s\n", zk2topo.Time(stat.Mtime).Format(timeFmtMicro)) - fmt.Printf("Size: %v\n", stat.DataLength) - fmt.Printf("Children: %v\n", stat.NumChildren) - fmt.Printf("Version: %v\n", stat.Version) - fmt.Printf("Ephemeral: %v\n", stat.EphemeralOwner) - fmt.Printf("ACL:\n") - for _, acl := range acls { - fmt.Printf(" %v:%v %v\n", acl.Scheme, acl.ID, fmtACL(acl)) - } - } - if hasError { - return fmt.Errorf("stat: some paths had errors") - } - return nil -} - -var charPermMap map[string]int32 -var permCharMap map[int32]string - -func init() { - charPermMap = map[string]int32{ - "r": zk.PermRead, - "w": zk.PermWrite, - "d": zk.PermDelete, - "c": zk.PermCreate, - "a": zk.PermAdmin, - } - permCharMap = make(map[int32]string) - for c, p := range charPermMap { - permCharMap[p] = c - } -} - -func fmtACL(acl zk.ACL) string { - s := "" - - for _, perm := range []int32{zk.PermRead, zk.PermWrite, zk.PermDelete, zk.PermCreate, zk.PermAdmin} { - if acl.Perms&perm != 0 { - s += permCharMap[perm] - } else { - s += "-" - } - } - return s -} - -func cmdChmod(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() < 2 { - return fmt.Errorf("chmod: no permission specified") - } - mode := subFlags.Arg(0) - if mode[0] != 'n' { - return fmt.Errorf("chmod: invalid mode") - } - - addPerms := false - if mode[1] == '+' { - addPerms = true - } else if mode[1] != '-' { - return fmt.Errorf("chmod: invalid mode") - } - - var permMask int32 - for _, c := range mode[2:] { - permMask |= charPermMap[string(c)] - } - - resolved, err := zk2topo.ResolveWildcards(ctx, zconn, subFlags.Args()[1:]) - if err != nil { - return fmt.Errorf("chmod: invalid wildcards: %v", err) - } - if len(resolved) == 0 { - // the wildcards didn't result in anything, we're done - return nil - } - - hasError := false - for _, arg := range resolved { - zkPath := fixZkPath(arg) - aclv, _, err := zconn.GetACL(ctx, zkPath) - if err != nil { - hasError = true - log.Warningf("chmod: cannot set access %v: %v", zkPath, err) - continue - } - if addPerms { - aclv[0].Perms |= permMask - } else { - aclv[0].Perms &= ^permMask - } - err = zconn.SetACL(ctx, zkPath, aclv, -1) - if err != nil { - hasError = true - log.Warningf("chmod: cannot set access %v: %v", zkPath, err) - continue - } - } - if hasError { - return fmt.Errorf("chmod: some paths had errors") - } - return nil -} - -func cmdCp(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - if err := subFlags.Parse(args); err != nil { - return err - } - switch { - case subFlags.NArg() < 2: - return fmt.Errorf("cp: need to specify source and destination paths") - case subFlags.NArg() == 2: - return fileCp(ctx, args[0], args[1]) - default: - return multiFileCp(ctx, args) - } -} - -func getPathData(ctx context.Context, filePath string) ([]byte, error) { - if isZkFile(filePath) { - data, _, err := zconn.Get(ctx, filePath) - return data, err - } - var err error - file, err := os.Open(filePath) - if err == nil { - data, err := io.ReadAll(file) - if err == nil { - return data, err - } - } - return nil, err -} - -func setPathData(ctx context.Context, filePath string, data []byte) error { - if isZkFile(filePath) { - _, err := zconn.Set(ctx, filePath, data, -1) - if err == zk.ErrNoNode { - _, err = zk2topo.CreateRecursive(ctx, zconn, filePath, data, 0, zk.WorldACL(zk.PermAll), 10) - } - return err - } - return os.WriteFile(filePath, []byte(data), 0666) -} - -func fileCp(ctx context.Context, srcPath, dstPath string) error { - dstIsDir := dstPath[len(dstPath)-1] == '/' - srcPath = fixZkPath(srcPath) - dstPath = fixZkPath(dstPath) - - if !isZkFile(srcPath) && !isZkFile(dstPath) { - return fmt.Errorf("cp: neither src nor dst is a /zk file: exitting") - } - - data, err := getPathData(ctx, srcPath) - if err != nil { - return fmt.Errorf("cp: cannot read %v: %v", srcPath, err) - } - - // If we are copying to a local directory - say '.', make the filename - // the same as the source. - if !isZkFile(dstPath) { - fileInfo, err := os.Stat(dstPath) - if err != nil { - if err.(*os.PathError).Err != syscall.ENOENT { - return fmt.Errorf("cp: cannot stat %v: %v", dstPath, err) - } - } else if fileInfo.IsDir() { - dstPath = path.Join(dstPath, path.Base(srcPath)) - } - } else if dstIsDir { - // If we are copying into zk, interpret trailing slash as treating the - // dstPath as a directory. - dstPath = path.Join(dstPath, path.Base(srcPath)) - } - if err := setPathData(ctx, dstPath, data); err != nil { - return fmt.Errorf("cp: cannot write %v: %v", dstPath, err) - } - return nil -} - -func multiFileCp(ctx context.Context, args []string) error { - dstPath := args[len(args)-1] - if dstPath[len(dstPath)-1] != '/' { - // In multifile context, dstPath must be a directory. - dstPath += "/" - } - - for _, srcPath := range args[:len(args)-1] { - if err := fileCp(ctx, srcPath, dstPath); err != nil { - return err - } - } - return nil -} - -type zkItem struct { - path string - data []byte - stat *zk.Stat - err error -} - -// Store a zk tree in a zip archive. This won't be immediately useful to -// zip tools since even "directories" can contain data. -func cmdZip(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() < 2 { - return fmt.Errorf("zip: need to specify source and destination paths") - } - - dstPath := subFlags.Arg(subFlags.NArg() - 1) - paths := subFlags.Args()[:len(args)-1] - if !strings.HasSuffix(dstPath, ".zip") { - return fmt.Errorf("zip: need to specify destination .zip path: %v", dstPath) - } - zipFile, err := os.Create(dstPath) - if err != nil { - return fmt.Errorf("zip: error %v", err) - } - - wg := sync.WaitGroup{} - items := make(chan *zkItem, 64) - for _, arg := range paths { - zkPath := fixZkPath(arg) - children, err := zk2topo.ChildrenRecursive(ctx, zconn, zkPath) - if err != nil { - return fmt.Errorf("zip: error %v", err) - } - for _, child := range children { - toAdd := path.Join(zkPath, child) - wg.Add(1) - go func() { - data, stat, err := zconn.Get(ctx, toAdd) - items <- &zkItem{toAdd, data, stat, err} - wg.Done() - }() - } - } - go func() { - wg.Wait() - close(items) - }() - - zipWriter := zip.NewWriter(zipFile) - for item := range items { - path, data, stat, err := item.path, item.data, item.stat, item.err - if err != nil { - return fmt.Errorf("zip: get failed: %v", err) - } - // Skip ephemerals - not sure why you would archive them. - if stat.EphemeralOwner > 0 { - continue - } - fi := &zip.FileHeader{Name: path, Method: zip.Deflate} - fi.Modified = zk2topo.Time(stat.Mtime) - f, err := zipWriter.CreateHeader(fi) - if err != nil { - return fmt.Errorf("zip: create failed: %v", err) - } - _, err = f.Write(data) - if err != nil { - return fmt.Errorf("zip: create failed: %v", err) - } - } - err = zipWriter.Close() - if err != nil { - return fmt.Errorf("zip: close failed: %v", err) - } - zipFile.Close() - return nil -} - -func cmdUnzip(ctx context.Context, subFlags *pflag.FlagSet, args []string) error { - if err := subFlags.Parse(args); err != nil { - return err - } - if subFlags.NArg() != 2 { - return fmt.Errorf("zip: need to specify source and destination paths") - } - - srcPath, dstPath := subFlags.Arg(0), subFlags.Arg(1) - - if !strings.HasSuffix(srcPath, ".zip") { - return fmt.Errorf("zip: need to specify src .zip path: %v", srcPath) - } - - zipReader, err := zip.OpenReader(srcPath) - if err != nil { - return fmt.Errorf("zip: error %v", err) - } - defer zipReader.Close() - - for _, zf := range zipReader.File { - rc, err := zf.Open() - if err != nil { - return fmt.Errorf("unzip: error %v", err) - } - data, err := io.ReadAll(rc) - if err != nil { - return fmt.Errorf("unzip: failed reading archive: %v", err) - } - zkPath := zf.Name - if dstPath != "/" { - zkPath = path.Join(dstPath, zkPath) - } - _, err = zk2topo.CreateRecursive(ctx, zconn, zkPath, data, 0, zk.WorldACL(zk.PermAll), 10) - if err != nil && err != zk.ErrNodeExists { - return fmt.Errorf("unzip: zk create failed: %v", err) - } - _, err = zconn.Set(ctx, zkPath, data, -1) - if err != nil { - return fmt.Errorf("unzip: zk set failed: %v", err) - } - rc.Close() - } - return nil -} diff --git a/go/flags/endtoend/zk.txt b/go/flags/endtoend/zk.txt index 443bf0b9ca2..fcae6a479c0 100644 --- a/go/flags/endtoend/zk.txt +++ b/go/flags/endtoend/zk.txt @@ -1,8 +1,41 @@ -Usage of zk: - -h, --help display usage and exit +zk is a tool for wrangling the zookeeper. + +It tries to mimic unix file system commands wherever possible, but +there are some slight differences in flag handling. + +The zk tool looks for the address of the cluster in /etc/zookeeper/zk_client.conf, +or the file specified in the ZK_CLIENT_CONFIG environment variable. + +The local cell may be overridden with the ZK_CLIENT_LOCAL_CELL environment +variable. + +Usage: + zk [command] + +Available Commands: + addAuth + cat + chmod + completion Generate the autocompletion script for the specified shell + cp + edit Create a local copy, edit, and write changes back to cell. + help Help about any command + ls + rm + stat + touch Change node access time. + unzip + wait Sets a watch on the node and then waits for an event to fire. + watch Watches for changes to nodes and prints events as they occur. + zip Store a zk tree in a zip archive. + +Flags: + -h, --help help for zk --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) --server string server(s) to connect to + +Use "zk [command] --help" for more information about a command. From 26dff0aa0f8812cfa3d246a1c0cf22d5de9c4aa0 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 25 Sep 2023 17:27:57 -0400 Subject: [PATCH 2/5] migrate zkctl binary to cobra Signed-off-by: Andrew Mason --- go/cmd/zkctl/command/init.go | 32 ++++++++++++++++ go/cmd/zkctl/command/root.go | 63 ++++++++++++++++++++++++++++++++ go/cmd/zkctl/command/shutdown.go | 32 ++++++++++++++++ go/cmd/zkctl/command/start.go | 32 ++++++++++++++++ go/cmd/zkctl/command/teardown.go | 32 ++++++++++++++++ go/cmd/zkctl/docgen/main.go | 37 +++++++++++++++++++ go/cmd/zkctl/zkctl.go | 62 ++----------------------------- go/flags/endtoend/zkctl.txt | 19 +++++++++- 8 files changed, 248 insertions(+), 61 deletions(-) create mode 100644 go/cmd/zkctl/command/init.go create mode 100644 go/cmd/zkctl/command/root.go create mode 100644 go/cmd/zkctl/command/shutdown.go create mode 100644 go/cmd/zkctl/command/start.go create mode 100644 go/cmd/zkctl/command/teardown.go create mode 100644 go/cmd/zkctl/docgen/main.go diff --git a/go/cmd/zkctl/command/init.go b/go/cmd/zkctl/command/init.go new file mode 100644 index 00000000000..518b4a6239d --- /dev/null +++ b/go/cmd/zkctl/command/init.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import "github.com/spf13/cobra" + +var Init = &cobra.Command{ + Use: "init", + Short: "Generates a new config and then starts zookeeper.", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return zkd.Init() + }, +} + +func init() { + Root.AddCommand(Init) +} diff --git a/go/cmd/zkctl/command/root.go b/go/cmd/zkctl/command/root.go new file mode 100644 index 00000000000..3399ed8c4cb --- /dev/null +++ b/go/cmd/zkctl/command/root.go @@ -0,0 +1,63 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/zkctl" +) + +var ( + zkCfg = "6@:3801:3802:3803" + myID uint + zkExtra []string + + zkd *zkctl.Zkd + + Root = &cobra.Command{ + Use: "zkctl", + Short: "Initializes and controls zookeeper with Vitess-specific configuration.", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if err := servenv.CobraPreRunE(cmd, args); err != nil { + return err + } + + zkConfig := zkctl.MakeZkConfigFromString(zkCfg, uint32(myID)) + zkConfig.Extra = zkExtra + zkd = zkctl.NewZkd(zkConfig) + + return nil + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + logutil.Flush() + }, + } +) + +func init() { + Root.PersistentFlags().StringVar(&zkCfg, "zk.cfg", zkCfg, + "zkid@server1:leaderPort1:electionPort1:clientPort1,...)") + Root.PersistentFlags().UintVar(&myID, "zk.myid", myID, + "which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname") + Root.PersistentFlags().StringArrayVar(&zkExtra, "zk.extra", zkExtra, + "extra config line(s) to append verbatim to config (flag can be specified more than once)") + + servenv.MovePersistentFlagsToCobraCommand(Root) +} diff --git a/go/cmd/zkctl/command/shutdown.go b/go/cmd/zkctl/command/shutdown.go new file mode 100644 index 00000000000..b3166bbd36b --- /dev/null +++ b/go/cmd/zkctl/command/shutdown.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import "github.com/spf13/cobra" + +var Shutdown = &cobra.Command{ + Use: "shutdown", + Short: "Terminates a zookeeper server but keeps its data dir intact.", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return zkd.Shutdown() + }, +} + +func init() { + Root.AddCommand(Shutdown) +} diff --git a/go/cmd/zkctl/command/start.go b/go/cmd/zkctl/command/start.go new file mode 100644 index 00000000000..1ed31d0ed54 --- /dev/null +++ b/go/cmd/zkctl/command/start.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import "github.com/spf13/cobra" + +var Start = &cobra.Command{ + Use: "start", + Short: "Runs an already initialized zookeeper server.", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return zkd.Start() + }, +} + +func init() { + Root.AddCommand(Start) +} diff --git a/go/cmd/zkctl/command/teardown.go b/go/cmd/zkctl/command/teardown.go new file mode 100644 index 00000000000..14fe7278835 --- /dev/null +++ b/go/cmd/zkctl/command/teardown.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package command + +import "github.com/spf13/cobra" + +var Teardown = &cobra.Command{ + Use: "teardown", + Short: "Shuts down the zookeeper server and removes its data dir.", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + return zkd.Teardown() + }, +} + +func init() { + Root.AddCommand(Teardown) +} diff --git a/go/cmd/zkctl/docgen/main.go b/go/cmd/zkctl/docgen/main.go new file mode 100644 index 00000000000..c35da8930e4 --- /dev/null +++ b/go/cmd/zkctl/docgen/main.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/internal/docgen" + "vitess.io/vitess/go/cmd/zkctl/command" +) + +func main() { + var dir string + cmd := cobra.Command{ + Use: "docgen [-d ]", + RunE: func(cmd *cobra.Command, args []string) error { + return docgen.GenerateMarkdownTree(command.Root, dir) + }, + } + + cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation") + _ = cmd.Execute() +} diff --git a/go/cmd/zkctl/zkctl.go b/go/cmd/zkctl/zkctl.go index 566631e8cc2..b00e3eb4812 100644 --- a/go/cmd/zkctl/zkctl.go +++ b/go/cmd/zkctl/zkctl.go @@ -14,75 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -// zkctl initializes and controls ZooKeeper with Vitess-specific configuration. package main import ( - "github.com/spf13/pflag" - + "vitess.io/vitess/go/cmd/zkctl/command" "vitess.io/vitess/go/exit" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/zkctl" ) -var usage = ` -Commands: - - init | start | shutdown | teardown -` - -var ( - zkCfg = "6@:3801:3802:3803" - myID uint - zkExtra []string -) - -func registerZkctlFlags(fs *pflag.FlagSet) { - fs.StringVar(&zkCfg, "zk.cfg", zkCfg, - "zkid@server1:leaderPort1:electionPort1:clientPort1,...)") - fs.UintVar(&myID, "zk.myid", myID, - "which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname") - fs.StringArrayVar(&zkExtra, "zk.extra", zkExtra, - "extra config line(s) to append verbatim to config (flag can be specified more than once)") -} - -func init() { - servenv.OnParse(registerZkctlFlags) -} - func main() { defer exit.Recover() - defer logutil.Flush() - - fs := pflag.NewFlagSet("zkctl", pflag.ExitOnError) - log.RegisterFlags(fs) - logutil.RegisterFlags(fs) - args := servenv.ParseFlagsWithArgs("zkctl") - zkConfig := zkctl.MakeZkConfigFromString(zkCfg, uint32(myID)) - zkConfig.Extra = zkExtra - zkd := zkctl.NewZkd(zkConfig) - - action := args[0] - var err error - switch action { - case "init": - err = zkd.Init() - case "shutdown": - err = zkd.Shutdown() - case "start": - err = zkd.Start() - case "teardown": - err = zkd.Teardown() - default: - log.Errorf("invalid action: %v", action) - log.Errorf(usage) - exit.Return(1) - } - if err != nil { - log.Errorf("failed %v: %v", action, err) + if err := command.Root.Execute(); err != nil { + log.Error(err) exit.Return(1) } } diff --git a/go/flags/endtoend/zkctl.txt b/go/flags/endtoend/zkctl.txt index 6b0473d1cb2..d1aea061ea5 100644 --- a/go/flags/endtoend/zkctl.txt +++ b/go/flags/endtoend/zkctl.txt @@ -1,4 +1,17 @@ -Usage of zkctl: +Initializes and controls zookeeper with Vitess-specific configuration. + +Usage: + zkctl [command] + +Available Commands: + completion Generate the autocompletion script for the specified shell + help Help about any command + init Generates a new config and then starts zookeeper. + shutdown Terminates a zookeeper server but keeps its data dir intact. + start Runs an already initialized zookeeper server. + teardown Shuts down the zookeeper server and removes its data dir. + +Flags: --alsologtostderr log to standard error as well as files --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) @@ -6,7 +19,7 @@ Usage of zkctl: --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). - -h, --help display usage and exit + -h, --help help for zkctl --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) @@ -23,3 +36,5 @@ Usage of zkctl: --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") --zk.extra stringArray extra config line(s) to append verbatim to config (flag can be specified more than once) --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname + +Use "zkctl [command] --help" for more information about a command. From 5d802ee3aed9099ee325ff27425099d05090b0e0 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 25 Sep 2023 17:32:56 -0400 Subject: [PATCH 3/5] migrate zkctld binary to cobra Signed-off-by: Andrew Mason --- go/cmd/zkctld/cli/zkctld.go | 100 +++++++++++++++++++++++++++++++++++ go/cmd/zkctld/docgen/main.go | 37 +++++++++++++ go/cmd/zkctld/zkctld.go | 70 ++---------------------- go/flags/endtoend/zkctld.txt | 33 +++--------- 4 files changed, 148 insertions(+), 92 deletions(-) create mode 100644 go/cmd/zkctld/cli/zkctld.go create mode 100644 go/cmd/zkctld/docgen/main.go diff --git a/go/cmd/zkctld/cli/zkctld.go b/go/cmd/zkctld/cli/zkctld.go new file mode 100644 index 00000000000..101f1013722 --- /dev/null +++ b/go/cmd/zkctld/cli/zkctld.go @@ -0,0 +1,100 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/servenv" + "vitess.io/vitess/go/vt/zkctl" +) + +var ( + zkCfg = "6@:3801:3802:3803" + myID uint + zkExtra []string + + Main = &cobra.Command{ + Use: "zkctld", + Short: "zkctld is a daemon that starts or initializes ZooKeeper with Vitess-specific configuration. It will stay running as long as the underlying ZooKeeper server, and will pass along SIGTERM.", + Args: cobra.NoArgs, + PersistentPreRunE: servenv.CobraPreRunE, + PostRun: func(cmd *cobra.Command, args []string) { + logutil.Flush() + }, + RunE: run, + } +) + +func init() { + servenv.OnParse(registerFlags) +} + +func registerFlags(fs *pflag.FlagSet) { + fs.StringVar(&zkCfg, "zk.cfg", zkCfg, + "zkid@server1:leaderPort1:electionPort1:clientPort1,...)") + fs.UintVar(&myID, "zk.myid", myID, + "which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname") + fs.StringArrayVar(&zkExtra, "zk.extra", zkExtra, + "extra config line(s) to append verbatim to config (flag can be specified more than once)") + acl.RegisterFlags(fs) +} + +func run(cmd *cobra.Command, args []string) error { + servenv.Init() + zkConfig := zkctl.MakeZkConfigFromString(zkCfg, uint32(myID)) + zkConfig.Extra = zkExtra + zkd := zkctl.NewZkd(zkConfig) + + if zkd.Inited() { + log.Infof("already initialized, starting without init...") + if err := zkd.Start(); err != nil { + return fmt.Errorf("failed start: %v", err) + } + } else { + log.Infof("initializing...") + if err := zkd.Init(); err != nil { + return fmt.Errorf("failed init: %v", err) + } + } + + log.Infof("waiting for signal or server shutdown...") + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + select { + case <-zkd.Done(): + log.Infof("server shut down on its own") + case <-sig: + log.Infof("signal received, shutting down server") + + // Action to perform if there is an error + if err := zkd.Shutdown(); err != nil { + return fmt.Errorf("error during shutdown:%v", err) + } + } + + return nil +} diff --git a/go/cmd/zkctld/docgen/main.go b/go/cmd/zkctld/docgen/main.go new file mode 100644 index 00000000000..9915d641352 --- /dev/null +++ b/go/cmd/zkctld/docgen/main.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "vitess.io/vitess/go/cmd/internal/docgen" + "vitess.io/vitess/go/cmd/vttablet/cli" +) + +func main() { + var dir string + cmd := cobra.Command{ + Use: "docgen [-d ]", + RunE: func(cmd *cobra.Command, args []string) error { + return docgen.GenerateMarkdownTree(cli.Main, dir) + }, + } + + cmd.Flags().StringVarP(&dir, "dir", "d", "doc", "output directory to write documentation") + _ = cmd.Execute() +} diff --git a/go/cmd/zkctld/zkctld.go b/go/cmd/zkctld/zkctld.go index 0d1ee413a66..211b63325eb 100644 --- a/go/cmd/zkctld/zkctld.go +++ b/go/cmd/zkctld/zkctld.go @@ -20,77 +20,15 @@ limitations under the License. package main import ( - "os" - "os/signal" - "syscall" - - "github.com/spf13/pflag" - - "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/cmd/zkctld/cli" "vitess.io/vitess/go/exit" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/servenv" - "vitess.io/vitess/go/vt/zkctl" -) - -var ( - zkCfg = "6@:3801:3802:3803" - myID uint - zkExtra []string ) -func init() { - servenv.OnParse(registerFlags) -} - -func registerFlags(fs *pflag.FlagSet) { - fs.StringVar(&zkCfg, "zk.cfg", zkCfg, - "zkid@server1:leaderPort1:electionPort1:clientPort1,...)") - fs.UintVar(&myID, "zk.myid", myID, - "which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname") - fs.StringArrayVar(&zkExtra, "zk.extra", zkExtra, - "extra config line(s) to append verbatim to config (flag can be specified more than once)") - acl.RegisterFlags(fs) -} - func main() { defer exit.Recover() - defer logutil.Flush() - - servenv.ParseFlags("zkctld") - servenv.Init() - zkConfig := zkctl.MakeZkConfigFromString(zkCfg, uint32(myID)) - zkConfig.Extra = zkExtra - zkd := zkctl.NewZkd(zkConfig) - - if zkd.Inited() { - log.Infof("already initialized, starting without init...") - if err := zkd.Start(); err != nil { - log.Errorf("failed start: %v", err) - exit.Return(255) - } - } else { - log.Infof("initializing...") - if err := zkd.Init(); err != nil { - log.Errorf("failed init: %v", err) - exit.Return(255) - } - } - - log.Infof("waiting for signal or server shutdown...") - sig := make(chan os.Signal, 1) - signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - select { - case <-zkd.Done(): - log.Infof("server shut down on its own") - case <-sig: - log.Infof("signal received, shutting down server") - - // Action to perform if there is an error - if err := zkd.Shutdown(); err != nil { - log.Errorf("error during shutdown:%v", err) - exit.Return(1) - } + if err := cli.Main.Execute(); err != nil { + log.Error(err) + exit.Return(1) } } diff --git a/go/flags/endtoend/zkctld.txt b/go/flags/endtoend/zkctld.txt index e957f7a3b3c..d808bd7ce67 100644 --- a/go/flags/endtoend/zkctld.txt +++ b/go/flags/endtoend/zkctld.txt @@ -1,26 +1,7 @@ -Usage of zkctld: - --alsologtostderr log to standard error as well as files - --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) - --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [{{ .Workdir }}]) - --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) - --config-type string Config file type (omit to infer config type from file extension). - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") - --zk.extra stringArray extra config line(s) to append verbatim to config (flag can be specified more than once) - --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname +zkctld is a daemon that starts or initializes ZooKeeper with Vitess-specific configuration. It will stay running as long as the underlying ZooKeeper server, and will pass along SIGTERM. + +Usage: + zkctld [flags] + +Flags: + -h, --help help for zkctld From 0e61ba498e0344d37d6e1cae933ae14aa2804fcd Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 26 Sep 2023 06:26:12 -0400 Subject: [PATCH 4/5] pr feedback Signed-off-by: Andrew Mason --- go/cmd/zk/command/root.go | 4 ++-- go/flags/endtoend/vtcombo.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go/cmd/zk/command/root.go b/go/cmd/zk/command/root.go index 30c2a93fd0a..f3f02e7d4f2 100644 --- a/go/cmd/zk/command/root.go +++ b/go/cmd/zk/command/root.go @@ -32,8 +32,8 @@ var ( Root = &cobra.Command{ Use: "zk", - Short: "zk is a tool for wrangling the zookeeper.", - Long: `zk is a tool for wrangling the zookeeper. + Short: "zk is a tool for wrangling zookeeper.", + Long: `zk is a tool for wrangling zookeeper. It tries to mimic unix file system commands wherever possible, but there are some slight differences in flag handling. diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 89d972f2f6b..ffa2b84970a 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -139,6 +139,7 @@ Flags: --gc_check_interval duration Interval between garbage collection checks (default 1h0m0s) --gc_purge_check_interval duration Interval between purge discovery checks (default 1m0s) --gh-ost-path string override default gh-ost binary full path + --grpc-send-session-in-streaming If set, will send the session as last packet in streaming api to support transactions in streaming --grpc-use-effective-groups If set, and SSL is not used, will set the immediate caller's security groups from the effective caller id's groups. --grpc-use-static-authentication-callerid If set, will set the immediate caller id to the username authenticated by the static auth plugin. --grpc_auth_mode string Which auth plugin implementation to use (eg: static) From 05da3dae80c89cecebc69bdb6086c72725e73720 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 26 Sep 2023 07:54:06 -0400 Subject: [PATCH 5/5] whoops Signed-off-by: Andrew Mason --- go/flags/endtoend/zk.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/flags/endtoend/zk.txt b/go/flags/endtoend/zk.txt index fcae6a479c0..add1b6b6803 100644 --- a/go/flags/endtoend/zk.txt +++ b/go/flags/endtoend/zk.txt @@ -1,4 +1,4 @@ -zk is a tool for wrangling the zookeeper. +zk is a tool for wrangling zookeeper. It tries to mimic unix file system commands wherever possible, but there are some slight differences in flag handling.