Follow logs using overlap

README.md

@@ -57,11 +57,8 @@ Run the following command  (check help string for more details):
 ```
 
 ### Fetching logs
-Run the following command (check help string for more details):
-
-```shell
-./vse-sync-collection-tools logs --kubeconfig="${KUBECONFIG}"
-```
+The log subcommand has been removed. Instead we have implimented at collector which is enabled by default.
+If possible you should use a log aggregator. You can control the collectors running using the `--collector` flag.
 
 ## Running tests
 

doc/implementing_a_collector.md

@@ -145,6 +145,6 @@ func NewAnnouncementCollector(constuctor *CollectionConstuctor) (Collector, erro
 
 func init(){
 	// We'll make this a required collector
-	RegisterCollector(AnnouncementCollectorName, NewAnnouncementCollector, true)
+	RegisterCollector(AnnouncementCollectorName, NewAnnouncementCollector, required)
 }
 ```

pkg/clients/clientset.go

@@ -18,12 +18,13 @@ import (
 
 // A Clientset contains clients for the different k8s API groups in one place
 type Clientset struct {
-	RestConfig    *rest.Config
-	DynamicClient dynamic.Interface
-	OcpClient     ocpconfig.Interface
-	K8sClient     kubernetes.Interface
-	K8sRestClient rest.Interface
-	ready         bool
+	RestConfig      *rest.Config
+	DynamicClient   dynamic.Interface
+	OcpClient       ocpconfig.Interface
+	K8sClient       kubernetes.Interface
+	K8sRestClient   rest.Interface
+	KubeConfigPaths []string
+	ready           bool
 }
 
 var clientset = Clientset{}
@@ -51,6 +52,7 @@ func GetClientset(kubeconfigPaths ...string) (*Clientset, error) {
 // newClientset will initialise the singleton clientset using provided kubeconfigPath
 func newClientset(kubeconfigPaths ...string) (*Clientset, error) {
 	log.Infof("creating new Clientset from %v", kubeconfigPaths)
+	clientset.KubeConfigPaths = kubeconfigPaths
 	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
 
 	loadingRules.Precedence = kubeconfigPaths // This means it will not load the value from $KUBECONFIG

pkg/clients/exec_command.go

@@ -27,7 +27,7 @@ type ContainerContext struct {
 	podNamePrefix string
 }
 
-func (clientsholder *Clientset) findPodNameFromPrefix(namespace, prefix string) (string, error) {
+func (clientsholder *Clientset) FindPodNameFromPrefix(namespace, prefix string) (string, error) {
 	podList, err := clientsholder.K8sClient.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		return "", fmt.Errorf("failed to getting pod list: %w", err)
@@ -53,7 +53,7 @@ func (clientsholder *Clientset) findPodNameFromPrefix(namespace, prefix string)
 }
 
 func (c *ContainerContext) Refresh() error {
-	newPodname, err := c.clientset.findPodNameFromPrefix(c.namespace, c.podNamePrefix)
+	newPodname, err := c.clientset.FindPodNameFromPrefix(c.namespace, c.podNamePrefix)
 	if err != nil {
 		return err
 	}
@@ -65,7 +65,7 @@ func NewContainerContext(
 	clientset *Clientset,
 	namespace, podNamePrefix, containerName string,
 ) (ContainerContext, error) {
-	podName, err := clientset.findPodNameFromPrefix(namespace, podNamePrefix)
+	podName, err := clientset.FindPodNameFromPrefix(namespace, podNamePrefix)
 	if err != nil {
 		return ContainerContext{}, err
 	}

pkg/cmd/collect.go

@@ -3,28 +3,41 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
 	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/redhat-partner-solutions/vse-sync-collection-tools/pkg/collectors"
 	"github.com/redhat-partner-solutions/vse-sync-collection-tools/pkg/runner"
 	"github.com/redhat-partner-solutions/vse-sync-collection-tools/pkg/utils"
 )
 
 const (
-	defaultDuration        string = "1000s"
-	defaultPollInterval    int    = 1
-	defaultDevInfoInterval int    = 60
+	defaultDuration             string = "1000s"
+	defaultPollInterval         int    = 1
+	defaultDevInfoInterval      int    = 60
+	defaultIncludeLogTimestamps bool   = false
+	defaultTempDir              string = "."
+	defaultKeepDebugFiles       bool   = false
+	tempdirPerm                        = 0755
 )
 
 var (
 	requestedDurationStr   string
 	pollInterval           int
 	devInfoAnnouceInterval int
 	collectorNames         []string
+	logsOutputFile         string
+	includeLogTimestamps   bool
+	tempDir                string
+	keepDebugFiles         bool
 )
 
 // collectCmd represents the collect command
@@ -41,6 +54,30 @@ var collectCmd = &cobra.Command{
 		}
 		utils.IfErrorExitOrPanic(err)
 
+		for _, c := range collectorNames {
+			if c == collectors.LogsCollectorName && logsOutputFile == "" {
+				utils.IfErrorExitOrPanic(utils.NewMissingInputError(
+					errors.New("if Logs collector is selected you must also provide a log output file")),
+				)
+			}
+		}
+
+		if strings.Contains(tempDir, "~") {
+			usr, err := user.Current()
+			if err != nil {
+				log.Fatal("Failed to fetch current user so could not resolve tempdir")
+			}
+			if tempDir == "~" {
+				tempDir = usr.HomeDir
+			} else if strings.HasPrefix(tempDir, "~/") {
+				tempDir = filepath.Join(usr.HomeDir, tempDir[2:])
+			}
+		}
+
+		if err := os.MkdirAll(tempDir, tempdirPerm); err != nil {
+			log.Fatal(err)
+		}
+
 		collectionRunner.Run(
 			kubeConfig,
 			outputFile,
@@ -49,6 +86,10 @@ var collectCmd = &cobra.Command{
 			devInfoAnnouceInterval,
 			ptpInterface,
 			useAnalyserJSON,
+			logsOutputFile,
+			includeLogTimestamps,
+			tempDir,
+			keepDebugFiles,
 		)
 	},
 }
@@ -99,4 +140,19 @@ func init() { //nolint:funlen // Allow this to get a little long
 			strings.Join(runner.OptionalCollectorNames, ", "),
 		),
 	)
+
+	collectCmd.Flags().StringVarP(
+		&logsOutputFile,
+		"logs-output", "l", "",
+		"Path to the logs output file. This is required when using the logs collector",
+	)
+	collectCmd.Flags().BoolVar(
+		&includeLogTimestamps,
+		"log-timestamps", defaultIncludeLogTimestamps,
+		"Specifies if collected logs should include timestamps or not. (default is false)",
+	)
+
+	collectCmd.Flags().StringVarP(&tempDir, "tempdir", "t", defaultTempDir,
+		"Directory for storing temp/debug files. Must exist.")
+	collectCmd.Flags().BoolVar(&keepDebugFiles, "keep", defaultKeepDebugFiles, "Keep debug files")
 }

pkg/collectors/collector.go

@@ -23,8 +23,12 @@ type CollectionConstructor struct {
 	ErroredPolls           chan PollResult
 	PTPInterface           string
 	Msg                    string
+	LogsOutputFile         string
+	TempDir                string
 	PollInterval           int
 	DevInfoAnnouceInterval int
+	IncludeLogTimestamps   bool
+	KeepDebugFiles         bool
 }
 
 type PollResult struct {

pkg/collectors/dev_info_collector.go

@@ -204,5 +204,5 @@ func NewDevInfoCollector(constructor *CollectionConstructor) (Collector, error)
 }
 
 func init() {
-	RegisterCollector(DevInfoCollectorName, NewDevInfoCollector, true)
+	RegisterCollector(DevInfoCollectorName, NewDevInfoCollector, required)
 }

pkg/collectors/dpll_collector.go

@@ -108,5 +108,5 @@ func NewDPLLCollector(constructor *CollectionConstructor) (Collector, error) {
 }
 
 func init() {
-	RegisterCollector(DPLLCollectorName, NewDPLLCollector, false)
+	RegisterCollector(DPLLCollectorName, NewDPLLCollector, optional)
 }

pkg/collectors/gps_ubx_collector.go

@@ -94,5 +94,5 @@ func NewGPSCollector(constructor *CollectionConstructor) (Collector, error) {
 }
 
 func init() {
-	RegisterCollector(GPSCollectorName, NewGPSCollector, false)
+	RegisterCollector(GPSCollectorName, NewGPSCollector, optional)
 }