implement stream reading

cmd/logveil/logveil.go

@@ -0,0 +1,103 @@
+package logveil
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+
+	"github.com/logmanager-oss/logveil/internal/anonymizer"
+	"github.com/logmanager-oss/logveil/internal/config"
+	"github.com/logmanager-oss/logveil/internal/loader"
+	"github.com/logmanager-oss/logveil/internal/reader"
+)
+
+type InputReader interface {
+	ReadLine() (map[string]string, error)
+}
+
+func Run() {
+	slog.Info("Anonymization process started...")
+
+	config := &config.Config{}
+	config.LoadAndValidate()
+
+	if *config.IsVerbose {
+		slog.SetLogLoggerLevel(slog.LevelDebug)
+	}
+
+	inputFile, err := os.Open(config.InputPath)
+	if err != nil {
+		slog.Error("opening input file for reading", "error", err)
+		return
+	}
+	defer inputFile.Close()
+
+	var inputReader InputReader
+	if *config.IsLmExport {
+		inputReader, err = reader.NewLmExportReader(inputFile)
+		if err != nil {
+			slog.Error("creating lm export reader", "error", err)
+			return
+		}
+	} else {
+		inputReader, err = reader.NewLmBackupReader(inputFile)
+		slog.Error("creating lm backup reader", "error", err)
+		if err != nil {
+			return
+		}
+	}
+
+	var outputFile *os.File
+	if config.OutputPath != "" {
+		outputFile, err := os.Create(config.OutputPath)
+		if err != nil {
+			slog.Error("opening output file for writing", "error", err)
+			return
+		}
+		defer outputFile.Close()
+	} else {
+		slog.Debug("output path empty - defaulting to stdout")
+		outputFile = os.Stdout
+	}
+
+	outputWriter := bufio.NewWriter(outputFile)
+	defer outputWriter.Flush()
+
+	anonymizingData, err := loader.Load(config.AnonymizationDataPath)
+	if err != nil {
+		slog.Error("loading anonymizing data from dir %s: %v", config.AnonymizationDataPath, err)
+		return
+	}
+
+	anonymizer := anonymizer.New(anonymizingData)
+
+	err = RunAnonymizationLoop(inputReader, outputWriter, anonymizer)
+	if err != nil {
+		slog.Error("running anonymisation loop", "error", err)
+		return
+	}
+
+	slog.Info("All done. Exiting...")
+}
+
+func RunAnonymizationLoop(inputReader InputReader, outputWriter io.Writer, anonymizer *anonymizer.Anonymizer) error {
+	for {
+		logLine, err := inputReader.ReadLine()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return nil
+			}
+			return fmt.Errorf("reading line: %v", err)
+		}
+
+		anonymizedLogLine := anonymizer.Anonymize(logLine)
+
+		_, err = fmt.Fprintln(outputWriter, anonymizedLogLine)
+		if err != nil {
+			return fmt.Errorf("writing log line to buffer: %v", err)
+		}
+	}
+}

cmd/main.go

@@ -1,7 +1,7 @@
 package main
 
-import "github.com/logmanager-oss/logveil/internal/anonymizer"
+import "github.com/logmanager-oss/logveil/cmd/logveil"
 
 func main() {
-	anonymizer.Run()
+	logveil.Run()
 }