exitCode: '126' when using --tcpdump-image for an airgapped scenario

[alexpothvellapally]

ksniff works without specifying the image, however it fails to launch if --tcpdump-image is specified. I need to be able to use it in an Air gapped environment hence need --tcpdump-image option to work. AFAIK without specifying this option , ksniff pulls the image from the same repository. However, if I specify the location it returns error 126

kubectl sniff shsmf501-n4-57b47c6fbb-zrgsv -n default -c axsvc -o n4.pcap -p -v
INFO[0000] running in verbose mode
DEBU[0000] pod 'shsmf501-n4-57b47c6fbb-zrgsv' status: 'Running'
INFO[0000] sniffing method: privileged pod
INFO[0000] sniffing on pod: 'shsmf501-n4-57b47c6fbb-zrgsv' [namespace: 'default', container: 'axsvc', filter: '', interface: 'any']
INFO[0000] creating privileged pod on node: 'r640-2'
DEBU[0000] creating privileged pod on remote node
INFO[0000] pod: 'ksniff-jzt5f' created successfully in namespace: 'default'
DEBU[0000] created pod details: &Pod{ObjectMeta:{ksniff-jzt5f ksniff- default c8644c87-d023-45ea-b603-f063db41fdad 903904220 0 2023-09-12 12:13:10 +1000 AEST map[app:ksniff] map[] [] [] [{kubectl-sniff Update v1 2023-09-12 12:13:10 +1000 AEST FieldsV1 {"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:app":{}}},"f:spec":{"f:containers":{"k:{"name":"ksniff-privileged"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{"mountPath":"/host"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{"mountPath":"/var/run/docker.sock"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostPID":{},"f:nodeName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{},"f:volumes":{".":{},"k:{"name":"container-socket"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{"name":"host"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}}]},Spec:PodSpec{Volumes:[]Volume{Volume{Name:host,VolumeSource:VolumeSource{HostPath:&HostPathVolumeSource{Path:/,Type:*Directory,},EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:nil,StorageOS:nil,CSI:nil,Ephemeral:nil,},},Volume{Name:container-socket,VolumeSource:VolumeSource{HostPath:&HostPathVolumeSource{Path:/var/run/docker.sock,Type:*Socket,},EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:nil,StorageOS:nil,CSI:nil,Ephemeral:nil,},},Volume{Name:kube-api-access-4q8bh,VolumeSource:VolumeSource{HostPath:nil,EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:&ProjectedVolumeSource{Sources:[]VolumeProjection{VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:nil,ServiceAccountToken:&ServiceAccountTokenProjection{Audience:,ExpirationSeconds:*3607,Path:token,},},VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:&ConfigMapProjection{LocalObjectReference:LocalObjectReference{Name:kube-root-ca.crt,},Items:[]KeyToPath{KeyToPath{Key:ca.crt,Path:ca.crt,Mode:nil,},},Optional:nil,},ServiceAccountToken:nil,},VolumeProjection{Secret:nil,DownwardAPI:&DownwardAPIProjection{Items:[]DownwardAPIVolumeFile{DownwardAPIVolumeFile{Path:namespace,FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,Mode:nil,},},},ConfigMap:nil,ServiceAccountToken:nil,},},DefaultMode:*420,},StorageOS:nil,CSI:nil,Ephemeral:nil,},},},Containers:[]Container{Container{Name:ksniff-privileged,Image:docker,Command:[sh -c sleep 10000000],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:container-socket,ReadOnly:true,MountPath:/var/run/docker.sock,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:host,ReadOnly:false,MountPath:/host,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-4q8bh,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,},},RestartPolicy:Never,TerminationGracePeriodSeconds:*30,ActiveDeadlineSeconds:nil,DNSPolicy:ClusterFirst,NodeSelector:map[string]string{},ServiceAccountName:default,DeprecatedServiceAccount:default,NodeName:r640-2,HostNetwork:false,HostPID:true,HostIPC:false,SecurityContext:&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,},ImagePullSecrets:[]LocalObjectReference{LocalObjectReference{Name:regcredstarhub,},},Hostname:,Subdomain:,Affinity:nil,SchedulerName:default-scheduler,InitContainers:[]Container{},AutomountServiceAccountToken:nil,Tolerations:[]Toleration{Toleration{Key:node.kubernetes.io/not-ready,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},Toleration{Key:node.kubernetes.io/unreachable,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},},HostAliases:[]HostAlias{},PriorityClassName:,Priority:*0,DNSConfig:nil,ShareProcessNamespace:nil,ReadinessGates:[]PodReadinessGate{},RuntimeClassName:nil,EnableServiceLinks:*true,PreemptionPolicy:*PreemptLowerPriority,Overhead:ResourceList{},TopologySpreadConstraints:[]TopologySpreadConstraint{},EphemeralContainers:[]EphemeralContainer{},SetHostnameAsFQDN:nil,},Status:PodStatus{Phase:Pending,Conditions:[]PodCondition{},Message:,Reason:,HostIP:,PodIP:,StartTime:,ContainerStatuses:[]ContainerStatus{},QOSClass:BestEffort,InitContainerStatuses:[]ContainerStatus{},NominatedNodeName:,PodIPs:[]PodIP{},EphemeralContainerStatuses:[]ContainerStatus{},},}
INFO[0000] waiting for pod successful startup
INFO[0002] pod: 'ksniff-jzt5f' created successfully on node: 'r640-2'
INFO[0002] output file option specified, storing output in: 'n4.pcap'
INFO[0002] starting remote sniffing using privileged pod
INFO[0002] executing command: '[docker --host unix:///var/run/docker.sock run --rm --name=ksniff-container-TdoGDfNw --net=container:8b49be29f3b0f9f46674471f904b49afc823e2c20ba8328d125aac2051e3568b maintained/tcpdump -i any -U -w - ]' on container: 'ksniff-privileged', pod: 'ksniff-jzt5f', namespace: 'default'

casa@casa-build-server:~/casa-5gc/platform$ kubectl sniff shsmf501-n4-57b47c6fbb-zrgsv -n default -c axsvc --image docker.io/hamravesh/ksniff-helper:v3 --tcpdump-image docker.io/maintained/tcpdump:latest -o n4.pcap -p -v
INFO[0000] running in verbose mode
DEBU[0000] pod 'shsmf501-n4-57b47c6fbb-zrgsv' status: 'Running'
INFO[0000] sniffing method: privileged pod
INFO[0000] sniffing on pod: 'shsmf501-n4-57b47c6fbb-zrgsv' [namespace: 'default', container: 'axsvc', filter: '', interface: 'any']
INFO[0000] creating privileged pod on node: 'r640-2'
DEBU[0000] creating privileged pod on remote node
INFO[0000] pod: 'ksniff-rzfmr' created successfully in namespace: 'default'
DEBU[0000] created pod details: &Pod{ObjectMeta:{ksniff-rzfmr ksniff- default 35a9c313-49b3-4d05-8657-bfea8d10b047 903904732 0 2023-09-12 12:14:19 +1000 AEST map[app:ksniff] map[] [] [] [{kubectl-sniff Update v1 2023-09-12 12:14:19 +1000 AEST FieldsV1 {"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:app":{}}},"f:spec":{"f:containers":{"k:{"name":"ksniff-privileged"}":{".":{},"f:command":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:resources":{},"f:securityContext":{".":{},"f:privileged":{}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{},"f:volumeMounts":{".":{},"k:{"mountPath":"/host"}":{".":{},"f:mountPath":{},"f:name":{}},"k:{"mountPath":"/var/run/docker.sock"}":{".":{},"f:mountPath":{},"f:name":{},"f:readOnly":{}}}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:hostPID":{},"f:nodeName":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{},"f:volumes":{".":{},"k:{"name":"container-socket"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}},"k:{"name":"host"}":{".":{},"f:hostPath":{".":{},"f:path":{},"f:type":{}},"f:name":{}}}}}}]},Spec:PodSpec{Volumes:[]Volume{Volume{Name:host,VolumeSource:VolumeSource{HostPath:&HostPathVolumeSource{Path:/,Type:*Directory,},EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:nil,StorageOS:nil,CSI:nil,Ephemeral:nil,},},Volume{Name:container-socket,VolumeSource:VolumeSource{HostPath:&HostPathVolumeSource{Path:/var/run/docker.sock,Type:*Socket,},EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:nil,StorageOS:nil,CSI:nil,Ephemeral:nil,},},Volume{Name:kube-api-access-6ktd4,VolumeSource:VolumeSource{HostPath:nil,EmptyDir:nil,GCEPersistentDisk:nil,AWSElasticBlockStore:nil,GitRepo:nil,Secret:nil,NFS:nil,ISCSI:nil,Glusterfs:nil,PersistentVolumeClaim:nil,RBD:nil,FlexVolume:nil,Cinder:nil,CephFS:nil,Flocker:nil,DownwardAPI:nil,FC:nil,AzureFile:nil,ConfigMap:nil,VsphereVolume:nil,Quobyte:nil,AzureDisk:nil,PhotonPersistentDisk:nil,PortworxVolume:nil,ScaleIO:nil,Projected:&ProjectedVolumeSource{Sources:[]VolumeProjection{VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:nil,ServiceAccountToken:&ServiceAccountTokenProjection{Audience:,ExpirationSeconds:*3607,Path:token,},},VolumeProjection{Secret:nil,DownwardAPI:nil,ConfigMap:&ConfigMapProjection{LocalObjectReference:LocalObjectReference{Name:kube-root-ca.crt,},Items:[]KeyToPath{KeyToPath{Key:ca.crt,Path:ca.crt,Mode:nil,},},Optional:nil,},ServiceAccountToken:nil,},VolumeProjection{Secret:nil,DownwardAPI:&DownwardAPIProjection{Items:[]DownwardAPIVolumeFile{DownwardAPIVolumeFile{Path:namespace,FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,Mode:nil,},},},ConfigMap:nil,ServiceAccountToken:nil,},},DefaultMode:*420,},StorageOS:nil,CSI:nil,Ephemeral:nil,},},},Containers:[]Container{Container{Name:ksniff-privileged,Image:docker.io/hamravesh/ksniff-helper:v3,Command:[sh -c sleep 10000000],Args:[],WorkingDir:,Ports:[]ContainerPort{},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{},},VolumeMounts:[]VolumeMount{VolumeMount{Name:container-socket,ReadOnly:true,MountPath:/var/run/docker.sock,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:host,ReadOnly:false,MountPath:/host,SubPath:,MountPropagation:nil,SubPathExpr:,},VolumeMount{Name:kube-api-access-6ktd4,ReadOnly:true,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,SubPath:,MountPropagation:nil,SubPathExpr:,},},LivenessProbe:nil,ReadinessProbe:nil,Lifecycle:nil,TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:nil,Privileged:*true,SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,ReadOnlyRootFilesystem:nil,AllowPrivilegeEscalation:nil,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:File,VolumeDevices:[]VolumeDevice{},StartupProbe:nil,},},RestartPolicy:Never,TerminationGracePeriodSeconds:*30,ActiveDeadlineSeconds:nil,DNSPolicy:ClusterFirst,NodeSelector:map[string]string{},ServiceAccountName:default,DeprecatedServiceAccount:default,NodeName:r640-2,HostNetwork:false,HostPID:true,HostIPC:false,SecurityContext:&PodSecurityContext{SELinuxOptions:nil,RunAsUser:nil,RunAsNonRoot:nil,SupplementalGroups:[],FSGroup:nil,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:nil,},ImagePullSecrets:[]LocalObjectReference{LocalObjectReference{Name:regcredstarhub,},},Hostname:,Subdomain:,Affinity:nil,SchedulerName:default-scheduler,InitContainers:[]Container{},AutomountServiceAccountToken:nil,Tolerations:[]Toleration{Toleration{Key:node.kubernetes.io/not-ready,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},Toleration{Key:node.kubernetes.io/unreachable,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},},HostAliases:[]HostAlias{},PriorityClassName:,Priority:*0,DNSConfig:nil,ShareProcessNamespace:nil,ReadinessGates:[]PodReadinessGate{},RuntimeClassName:nil,EnableServiceLinks:*true,PreemptionPolicy:*PreemptLowerPriority,Overhead:ResourceList{},TopologySpreadConstraints:[]TopologySpreadConstraint{},EphemeralContainers:[]EphemeralContainer{},SetHostnameAsFQDN:nil,},Status:PodStatus{Phase:Pending,Conditions:[]PodCondition{},Message:,Reason:,HostIP:,PodIP:,StartTime:,ContainerStatuses:[]ContainerStatus{},QOSClass:BestEffort,InitContainerStatuses:[]ContainerStatus{},NominatedNodeName:,PodIPs:[]PodIP{},EphemeralContainerStatuses:[]ContainerStatus{},},}
INFO[0000] waiting for pod successful startup
INFO[0002] pod: 'ksniff-rzfmr' created successfully on node: 'r640-2'
INFO[0002] output file option specified, storing output in: 'n4.pcap'
INFO[0002] starting remote sniffing using privileged pod
INFO[0002] executing command: '[docker --host unix:///var/run/docker.sock run --rm --name=ksniff-container-DVlhQPJJ --net=container:8b49be29f3b0f9f46674471f904b49afc823e2c20ba8328d125aac2051e3568b docker.io/maintained/tcpdump:latest -i any -U -w - ]' on container: 'ksniff-privileged', pod: 'ksniff-rzfmr', namespace: 'default'
INFO[0002] command: '[docker --host unix:///var/run/docker.sock run --rm --name=ksniff-container-DVlhQPJJ --net=container:8b49be29f3b0f9f46674471f904b49afc823e2c20ba8328d125aac2051e3568b docker.io/maintained/tcpdump:latest -i any -U -w - ]' executing successfully exitCode: '126', stdErr :''
INFO[0002] remote sniffing using privileged pod completed
INFO[0002] starting sniffer cleanup
INFO[0002] removing privileged container: 'ksniff-privileged'
INFO[0002] executing command: '[docker --host unix:///var/run/docker.sock rm -f ksniff-container-DVlhQPJJ]' on container: 'ksniff-privileged', pod: 'ksniff-rzfmr', namespace: 'default'
INFO[0002] command: '[docker --host unix:///var/run/docker.sock rm -f ksniff-container-DVlhQPJJ]' executing successfully exitCode: '126', stdErr :''
INFO[0002] privileged container: 'ksniff-privileged' removed successfully
INFO[0002] removing pod: 'ksniff-rzfmr'
INFO[0002] removing privileged pod: 'ksniff-rzfmr'
INFO[0002] privileged pod: 'ksniff-rzfmr' removed
INFO[0002] pod: 'ksniff-rzfmr' removed successfully
INFO[0002] sniffer cleanup completed successfully

[Cyril-zip]

I just checked the image docker.io/hamravesh/ksniff-helper:v3; it seems to be missing the docker-cli, so executing the docker command will fail. Unfortunately, I can't find any docker version of the ksniff-helper image. We may need to build the docker image by ourselves. (And I am not sure whether this project is out of support).