feat: implement retina shell CLI command #962
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "time" | ||
|
|
||
| "github.com/microsoft/retina/shell" | ||
| "github.com/spf13/cobra" | ||
| v1 "k8s.io/api/core/v1" | ||
| "k8s.io/cli-runtime/pkg/genericclioptions" | ||
| "k8s.io/cli-runtime/pkg/resource" | ||
| cmdutil "k8s.io/kubectl/pkg/cmd/util" | ||
| "k8s.io/kubectl/pkg/scheme" | ||
| "k8s.io/kubectl/pkg/util/templates" | ||
| ) | ||
|
|
||
| var ( | ||
| configFlags *genericclioptions.ConfigFlags | ||
| matchVersionFlags *cmdutil.MatchVersionFlags | ||
| retinaShellImageRepo string | ||
| retinaShellImageVersion string | ||
| mountHostFilesystem bool | ||
| allowHostFilesystemWrite bool | ||
| hostPID bool | ||
| capabilities []string | ||
| timeout time.Duration | ||
| ) | ||
|
|
||
| const defaultRetinaShellImageRepo = "ghcr.io/microsoft/retina/retina-shell" | ||
|
|
||
| var shellCmd = &cobra.Command{ | ||
| Use: "shell (NODE | TYPE[[.VERSION].GROUP]/NAME)", | ||
| Short: "[EXPERIMENTAL] Interactively debug a node or pod", | ||
| Long: templates.LongDesc(` | ||
| [EXPERIMENTAL] This is an experimental command. The flags and behavior may change in the future. | ||
|
|
||
| Start a shell with networking tools in a node or pod for adhoc debugging. | ||
|
|
||
| * For nodes, this creates a pod on the node in the root network namespace. | ||
| * For pods, this creates an ephemeral container inside the pod's network namespace. | ||
|
|
||
| You can override the default image used for the shell container with either | ||
| CLI flags (--retina-shell-image-repo and --retina-shell-image-version) or | ||
| environment variables (RETINA_SHELL_IMAGE_REPO and RETINA_SHELL_IMAGE_VERSION). | ||
| CLI flags take precedence over env vars. | ||
| `), | ||
|
|
||
| Example: templates.Examples(` | ||
| # start a shell in a node | ||
| kubectl retina shell node0001 | ||
|
|
||
| # start a shell in a node, with debug pod in kube-system namespace | ||
| kubectl retina shell -n kube-system node0001 | ||
|
|
||
| # start a shell as an ephemeral container inside an existing pod | ||
| kubectl retina shell -n kube-system pod/coredns-d459997b4-7cpzx | ||
|
|
||
| # start a shell in a node, mounting the host filesystem to /host with ability to chroot | ||
| kubectl retina shell node001 --mount-host-filesystem --capabilities SYS_CHROOT | ||
|
|
||
| # start a shell in a node, with NET_RAW and NET_ADMIN capabilities | ||
| # (required for iptables and tcpdump) | ||
| kubectl retina shell node001 --capabilities NET_RAW,NET_ADMIN | ||
| `), | ||
| Args: cobra.ExactArgs(1), | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| namespace, explicitNamespace, err := matchVersionFlags.ToRawKubeConfigLoader().Namespace() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // This interprets the first arg as either a node or pod (same as kubectl): | ||
| // "node001" -> node | ||
| // "node/node001" -> node | ||
| // "pod/example-7cpzx" -> pod | ||
| r := resource.NewBuilder(configFlags). | ||
| WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...). | ||
| FilenameParam(explicitNamespace, &resource.FilenameOptions{}). | ||
| NamespaceParam(namespace).DefaultNamespace().ResourceNames("nodes", args[0]). | ||
| Do() | ||
| if err := r.Err(); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| restConfig, err := matchVersionFlags.ToRESTConfig() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| config := shell.Config{ | ||
|
Check failure on line 91 in cli/cmd/shell.go
|
||
|
There was a problem hiding this comment. golang-ci lint is complaining that There was a problem hiding this comment. It's probably upset because it can't prove that the closure gets invoked, and it's trying to call |
||
| RestConfig: restConfig, | ||
| RetinaShellImage: fmt.Sprintf("%s:%s", retinaShellImageRepo, retinaShellImageVersion), | ||
| MountHostFilesystem: mountHostFilesystem, | ||
| AllowHostFilesystemWrite: allowHostFilesystemWrite, | ||
| HostPID: hostPID, | ||
| Capabilities: capabilities, | ||
| Timeout: timeout, | ||
| } | ||
|
|
||
| return r.Visit(func(info *resource.Info, err error) error { | ||
| switch obj := info.Object.(type) { | ||
| case *v1.Node: | ||
| podDebugNamespace := namespace | ||
| nodeName := obj.Name | ||
| return shell.RunInNode(config, nodeName, podDebugNamespace) | ||
| case *v1.Pod: | ||
| return shell.RunInPod(config, obj.Namespace, obj.Name) | ||
| default: | ||
| gvk := obj.GetObjectKind().GroupVersionKind() | ||
| return fmt.Errorf("unsupported resource %s/%s", gvk.GroupVersion(), gvk.Kind) | ||
| } | ||
| }) | ||
| }, | ||
| } | ||
|
|
||
| func init() { | ||
| Retina.AddCommand(shellCmd) | ||
| shellCmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { | ||
| // Avoid printing full usage message if the command exits with an error. | ||
| cmd.SilenceUsage = true | ||
| cmd.SilenceErrors = true | ||
|
|
||
| // Allow setting image repo and version via environment variables (CLI flags still take precedence). | ||
| if !cmd.Flags().Changed("retina-shell-image-repo") { | ||
| if envRepo := os.Getenv("RETINA_SHELL_IMAGE_REPO"); envRepo != "" { | ||
| retinaShellImageRepo = envRepo | ||
| } | ||
| } | ||
| if !cmd.Flags().Changed("retina-shell-image-version") { | ||
| if envVersion := os.Getenv("RETINA_SHELL_IMAGE_VERSION"); envVersion != "" { | ||
| retinaShellImageVersion = envVersion | ||
| } | ||
| } | ||
| } | ||
| shellCmd.Flags().StringVar(&retinaShellImageRepo, "retina-shell-image-repo", defaultRetinaShellImageRepo, "The container registry repository for the image to use for the shell container") | ||
| shellCmd.Flags().StringVar(&retinaShellImageVersion, "retina-shell-image-version", Version, "The version (tag) of the image to use for the shell container") | ||
| shellCmd.Flags().BoolVarP(&mountHostFilesystem, "mount-host-filesystem", "m", false, "Mount the host filesystem to /host. Applies only to nodes, not pods.") | ||
| shellCmd.Flags().BoolVarP(&allowHostFilesystemWrite, "allow-host-filesystem-write", "w", false, "Allow write access to the host filesystem. Implies --mount-host-filesystem. Applies only to nodes, not pods.") | ||
| shellCmd.Flags().BoolVar(&hostPID, "host-pid", false, "Set HostPID on the shell container. Applies only to nodes, not pods.") | ||
| shellCmd.Flags().StringSliceVarP(&capabilities, "capabilities", "c", []string{}, "Add capabilities to the shell container") | ||
| shellCmd.Flags().DurationVar(&timeout, "timeout", 30*time.Second, "The maximum time to wait for the shell container to start") | ||
|
There was a problem hiding this comment. looking for feedback about how to structure these flags. This was the simplest way I could think of that supported all the use cases I wanted to support (see docs/06-Troubleshooting/shell.md) |
||
|
|
||
| // configFlags and matchVersion flags are used to load kubeconfig. | ||
| // This uses the same mechanism as `kubectl debug` to connect to apiserver and attach to containers. | ||
| configFlags = genericclioptions.NewConfigFlags(true) | ||
| configFlags.AddFlags(shellCmd.PersistentFlags()) | ||
| matchVersionFlags = cmdutil.NewMatchVersionFlags(configFlags) | ||
| matchVersionFlags.AddFlags(shellCmd.PersistentFlags()) | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,185 @@ | ||
| # Shell TSG | ||
|
|
||
| **EXPERIMENTAL: `retina shell` is an experimental feature, so the flags and behavior may change in future versions.** | ||
|
|
||
| The `retina shell` command allows you to start an interactive shell on a Kubernetes node or pod. This runs a container image with many common networking tools installed (`ping`, `curl`, etc.). | ||
|
|
||
| ## Testing connectivity | ||
|
|
||
| Start a shell on a node or inside a pod | ||
|
|
||
| ```bash | ||
| # To start a shell in a node (root network namespace): | ||
| kubectl retina shell aks-nodepool1-15232018-vmss000001 | ||
|
|
||
| # To start a shell inside a pod (pod network namespace): | ||
| kubectl retina shell -n kube-system pods/coredns-d459997b4-7cpzx | ||
| ``` | ||
|
|
||
| Check connectivity using `ping`: | ||
|
|
||
| ```text | ||
| root [ / ]# ping 10.224.0.4 | ||
| PING 10.224.0.4 (10.224.0.4) 56(84) bytes of data. | ||
| 64 bytes from 10.224.0.4: icmp_seq=1 ttl=64 time=0.964 ms | ||
| 64 bytes from 10.224.0.4: icmp_seq=2 ttl=64 time=1.13 ms | ||
| 64 bytes from 10.224.0.4: icmp_seq=3 ttl=64 time=0.908 ms | ||
| 64 bytes from 10.224.0.4: icmp_seq=4 ttl=64 time=1.07 ms | ||
| 64 bytes from 10.224.0.4: icmp_seq=5 ttl=64 time=1.01 ms | ||
|
|
||
| --- 10.224.0.4 ping statistics --- | ||
| 5 packets transmitted, 5 received, 0% packet loss, time 4022ms | ||
| rtt min/avg/max/mdev = 0.908/1.015/1.128/0.077 ms | ||
| ``` | ||
|
|
||
| Check DNS resolution using `dig`: | ||
|
|
||
| ```text | ||
| root [ / ]# dig example.com +short | ||
| 93.184.215.14 | ||
| ``` | ||
|
|
||
| The tools `nslookup` and `drill` are also available if you prefer those. | ||
|
|
||
| Check connectivity to apiserver using `nc` and `curl`: | ||
|
|
||
| ```text | ||
| root [ / ]# nc -zv 10.0.0.1 443 | ||
| Ncat: Version 7.95 ( https://nmap.org/ncat ) | ||
| Ncat: Connected to 10.0.0.1:443. | ||
| Ncat: 0 bytes sent, 0 bytes received in 0.06 seconds. | ||
|
|
||
| root [ / ]# curl -k https://10.0.0.1 | ||
| { | ||
| "kind": "Status", | ||
| "apiVersion": "v1", | ||
| "metadata": {}, | ||
| "status": "Failure", | ||
| "message": "Unauthorized", | ||
| "reason": "Unauthorized", | ||
| "code": 401 | ||
| } | ||
| ``` | ||
|
|
||
| ### nftables and iptables | ||
|
|
||
| Accessing nftables and iptables rules requires `NET_RAW` and `NET_ADMIN` capabilities. | ||
|
|
||
| ```bash | ||
| kubectl retina shell aks-nodepool1-15232018-vmss000002 --capabilities NET_ADMIN,NET_RAW | ||
| ``` | ||
|
|
||
| Then you can run `iptables` and `nft`: | ||
|
|
||
| ```text | ||
| root [ / ]# iptables -nvL | head -n 2 | ||
| Chain INPUT (policy ACCEPT 1191K packets, 346M bytes) | ||
| pkts bytes target prot opt in out source destination | ||
| root [ / ]# nft list ruleset | head -n 2 | ||
| # Warning: table ip filter is managed by iptables-nft, do not touch! | ||
| table ip filter { | ||
| ``` | ||
|
|
||
| **If you see the error "Operation not permitted (you must be root)", check that your `kubectl retina shell` command sets `--capabilities NET_RAW,NET_ADMIN`.** | ||
|
|
||
| `iptables` in the shell image uses `iptables-legacy`, which may or may not match the configuration on the node. For example, Ubuntu maps `iptables` to `iptables-nft`. To use the exact same `iptables` binary as installed on the node, you will need to `chroot` into the host filesystem (see below). | ||
|
|
||
| ## Accessing the host filesystem | ||
|
|
||
| On nodes, you can mount the host filesystem to `/host`: | ||
|
|
||
| ```bash | ||
| kubectl retina shell aks-nodepool1-15232018-vmss000002 --mount-host-filesystem | ||
| ``` | ||
|
|
||
| This mounts the host filesystem (`/`) to `/host` in the debug pod: | ||
|
|
||
| ```text | ||
| root [ / ]# ls /host | ||
| NOTICE.txt bin boot dev etc home lib lib64 libx32 lost+found media mnt opt proc root run sbin srv sys tmp usr var | ||
| ``` | ||
|
|
||
| The host filesystem is mounted read-only by default. If you need write access, use the `--allow-host-filesystem-write` flag. | ||
|
|
||
| Symlinks between files on the host filesystem may not resolve correctly. If you see "No such file or directory" errors for symlinks, try following the instructions below to `chroot` to the host filesystem. | ||
|
|
||
| ## Chroot to the host filesystem | ||
|
|
||
| `chroot` requires the `SYS_CHROOT` capability: | ||
|
|
||
| ```bash | ||
| kubectl retina shell aks-nodepool1-15232018-vmss000002 --mount-host-filesystem --capabilities SYS_CHROOT | ||
| ``` | ||
|
|
||
| Then you can use `chroot` to switch to start a shell inside the host filesystem: | ||
|
|
||
| ```text | ||
| root [ / ]# chroot /host bash | ||
| root@aks-nodepool1-15232018-vmss000002:/# cat /etc/resolv.conf | tail -n 2 | ||
| nameserver 168.63.129.16 | ||
| search shncgv2kgepuhm1ls1dwgholsd.cx.internal.cloudapp.net | ||
| ``` | ||
|
|
||
| `chroot` allows you to: | ||
|
|
||
| * Execute binaries installed on the node. | ||
| * Resolve symlinks that point to files in the host filesystem (such as /etc/resolv.conf -> /run/systemd/resolve/resolv.conf) | ||
| * Use `sysctl` to view or modify kernel parameters. | ||
| * Use `journalctl` to view systemd unit and kernel logs. | ||
| * Use `ip netns` to view network namespaces. (However, `ip netns exec` does not work.) | ||
|
|
||
| ## Systemctl | ||
|
|
||
| `systemctl` commands require both `chroot` to the host filesystem and host PID: | ||
|
|
||
| ```bash | ||
| kubectl retina shell aks-nodepool1-15232018-vmss000002 --mount-host-filesystem --capabilities SYS_CHROOT --host-pid | ||
| ``` | ||
|
|
||
| Then `chroot` to the host filesystem and run `systemctl status`: | ||
|
|
||
| ```text | ||
| root [ / ]# chroot /host systemctl status | head -n 2 | ||
| ● aks-nodepool1-15232018-vmss000002 | ||
| State: running | ||
| ``` | ||
|
|
||
| **If `systemctl` shows an error "Failed to connect to bus: No data available", check that the `retina shell` command has `--host-pid` set and that you have chroot'd to /host.** | ||
|
|
||
| ## Troubleshooting | ||
|
|
||
| ### Timeouts | ||
|
|
||
| If `kubectl retina shell` fails with a timeout error, then: | ||
|
|
||
| 1. Increase the timeout by setting `--timeout` flag. | ||
| 2. Check the pod using `kubectl describe pod` to determine why retina shell is failing to start. | ||
|
|
||
| Example: | ||
|
|
||
| ```bash | ||
| kubectl retina shell --timeout 10m node001 # increase timeout to 10 minutes | ||
| ``` | ||
|
|
||
| ### Firewalls and ImagePullBackoff | ||
|
|
||
| Some clusters are behind a firewall that blocks pulling the retina-shell image. To workaround this: | ||
|
|
||
| 1. Replicate the retina-shell images to a container registry accessible from within the cluster. | ||
| 2. Override the image used by Retina CLI with the environment variable `RETINA_SHELL_IMAGE_REPO`. | ||
|
|
||
| Example: | ||
|
|
||
| ```bash | ||
| export RETINA_SHELL_IMAGE_REPO="example.azurecr.io/retina/retina-shell" | ||
| export RETINA_SHELL_IMAGE_VERSION=v0.0.1 # optional, if not set defaults to the Retina CLI version. | ||
| kubectl retina shell node0001 # this will use the image "example.azurecr.io/retina/retina-shell:v0.0.1" | ||
| ``` | ||
|
|
||
| ## Limitations | ||
|
|
||
| * Windows nodes and pods are not yet supported. | ||
| * `bpftool` and `bpftrace` are not supported. | ||
| * The shell image link `iptables` commands to `iptables-legacy`, even if the node itself links to `iptables-nft`. | ||
| * `nsenter` is not supported. | ||
| * `ip netns` will not work without `chroot` to the host filesystem. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| # retina-shell | ||
|
|
||
| Retina CLI provides a command to launch an interactive shell in a node or pod for adhoc debugging. | ||
|
|
||
| * The CLI command `kubectl retina shell` creates a pod with `HostNetwork=true` (for node debugging) or an ephemeral container in an existing pod (for pod debugging). | ||
| * The container runs an image built from the Dockerfile in this directory. The image is based on Azure Linux and includes commonly-used networking tools. | ||
|
|
||
| For testing, you can override the image used by `retina shell` either with CLI arguments | ||
| (`--retina-shell-image-repo` and `--retina-shell-image-version`) or environment variables | ||
| (`RETINA_SHELL_IMAGE_REPO` and `RETINA_SHELL_IMAGE_VERSION`). | ||
|
|
||
| Run `kubectl retina shell -h` for full documentation and examples. | ||
|
|
||
| Currently only Linux is supported; Windows support will be added in the future. |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Few questions about this part:
retina/cli/cmd/version.go
Lines 12 to 14 in 2b109a3
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You could make it a var, have it set by default to GHCR and then patch this var at link time with MCR if needed.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
yeah, that's an option. Do we do any patching like that today when building the retina kubectl plugin? (Ideally the official plugin release would use MCR so it will "just work" in any AKS cluster)