bug fixes

.gitignore

@@ -1,2 +1,2 @@
-
+.DS_Store
 dist/

README.md

@@ -105,6 +105,8 @@ If you want to anonymize values in `organization` and `username` keys, you need
 
 Both files should contain appropriate fake data for the values they will be masking.
 
+**Make sure your filenames DOES NOT contain `msg.`**
+
 ### Regexp scanning and dynamic fake data generation
 
 LogVeil implements regular expressions to look for common patterns: IP (v4, v6), Emails, MAC and URL. Once such pattern is found it is replaced with fake data generated on the fly.

cmd/logveil/logveil.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/logmanager-oss/logveil/internal/anonymizer"
 	"github.com/logmanager-oss/logveil/internal/config"
-	"github.com/logmanager-oss/logveil/internal/files"
+	"github.com/logmanager-oss/logveil/internal/handlers"
 	"github.com/logmanager-oss/logveil/internal/proof"
 	"github.com/logmanager-oss/logveil/internal/reader"
 	"github.com/logmanager-oss/logveil/internal/writer"
@@ -25,23 +25,30 @@ func Start() {
 		slog.SetLogLoggerLevel(slog.LevelDebug)
 	}
 
-	filesHandler := &files.FilesHandler{}
+	filesHandler := &handlers.Files{}
 	defer filesHandler.Close()
 
+	buffersHandler := &handlers.Buffers{}
+	defer buffersHandler.Flush()
+
 	inputReader, err := reader.CreateInputReader(config, filesHandler)
 	if err != nil {
+		slog.Error("initializing input reader", "error", err)
 		return
 	}
-	outputWriter, err := writer.CreateOutputWriter(config, filesHandler)
+	outputWriter, err := writer.CreateOutputWriter(config, filesHandler, buffersHandler)
 	if err != nil {
+		slog.Error("initializing output writer", "error", err)
 		return
 	}
-	proofWriter, err := proof.CreateProofWriter(config, filesHandler)
+	proofWriter, err := proof.CreateProofWriter(config, filesHandler, buffersHandler)
 	if err != nil {
+		slog.Error("initializing proof writer", "error", err)
 		return
 	}
 	anonymizerDoer, err := anonymizer.CreateAnonymizer(config, proofWriter)
 	if err != nil {
+		slog.Error("initializing anonymizer", "error", err)
 		return
 	}
 
@@ -55,8 +62,6 @@ func Start() {
 }
 
 func RunAnonymizationLoop(inputReader reader.InputReader, outputWriter *bufio.Writer, anonymizerDoer *anonymizer.Anonymizer) error {
-	defer outputWriter.Flush()
-
 	for {
 		logLine, err := inputReader.ReadLine()
 		if err != nil {
@@ -70,7 +75,7 @@ func RunAnonymizationLoop(inputReader reader.InputReader, outputWriter *bufio.Wr
 
 		_, err = fmt.Fprintln(outputWriter, anonymizedLogLine)
 		if err != nil {
-			return fmt.Errorf("writing log line to buffer: %v", err)
+			return fmt.Errorf("writing log line %s: %v", anonymizedLogLine, err)
 		}
 	}
 }

cmd/main.go

@@ -1,6 +1,8 @@
 package main
 
-import "github.com/logmanager-oss/logveil/cmd/logveil"
+import (
+	"github.com/logmanager-oss/logveil/cmd/logveil"
+)
 
 func main() {
 	logveil.Start()

internal/anonymizer/anonymizer.go

@@ -99,6 +99,7 @@ func (an *Anonymizer) generateAndReplace(rawLog string, replacementMap map[strin
 		}
 
 		replacementMap[value] = generatedData
+		slog.Debug(fmt.Sprintf("Value matched via regexp. Reaplacing from %s to %s.\n", value, generatedData))
 	}
 
 	return replacementMap

internal/config/config.go

@@ -20,13 +20,13 @@ type Config struct {
 
 // LoadAndValidate loads values from user supplied input into Config struct and validates them
 func (c *Config) LoadAndValidate() {
-	flag.Func("d", "Path to directory with anonymizing data", validateDir(c.AnonymizationDataPath))
+	flag.Func("d", "Path to directory with anonymizing data", c.validateDirPath())
 
-	flag.Func("i", "Path to input file containing logs to be anonymized", validateInput(c.InputPath))
+	flag.Func("i", "Path to input file containing logs to be anonymized", c.validateInputPath())
 
-	flag.Func("c", "Path to input file containing custom anonymization mappings", validateInput(c.CustomReplacementMapPath))
+	flag.Func("c", "Path to input file containing custom anonymization mappings", c.validateCustomMappingPath())
 
-	flag.Func("o", "Path to output file (default: Stdout)", validateOutput(c.OutputPath))
+	flag.Func("o", "Path to output file (default: Stdout)", c.validateOutput())
 
 	flag.BoolVar(&c.IsVerbose, "v", false, "Enable verbose logging (default: Disabled)")
 	flag.BoolVar(&c.IsLmExport, "e", false, "Change input file type to LM export (default: LM Backup)")

internal/config/validate.go

@@ -6,7 +6,7 @@ import (
 	"os"
 )
 
-func validateInput(inputPath string) func(string) error {
+func (c *Config) validateInputPath() func(string) error {
 	return func(flagValue string) error {
 		fileInfo, err := os.Stat(flagValue)
 		if err != nil {
@@ -17,33 +17,54 @@ func validateInput(inputPath string) func(string) error {
 			return fmt.Errorf("Input file %s cannot be a directory.\n", flagValue)
 		}
 
-		inputPath = flagValue
+		c.InputPath = flagValue
 
 		return nil
 	}
 }
 
-func validateOutput(outputPath string) func(string) error {
+func (c *Config) validateCustomMappingPath() func(string) error {
 	return func(flagValue string) error {
 		fileInfo, err := os.Stat(flagValue)
 		if err != nil {
+			return err
+		}
+
+		if fileInfo.IsDir() {
+			return fmt.Errorf("Path to custom mapping file %s cannot be a directory.\n", flagValue)
+		}
+
+		c.CustomReplacementMapPath = flagValue
+
+		return nil
+	}
+}
+
+func (c *Config) validateOutput() func(string) error {
+	return func(flagValue string) error {
+		fileInfo, err := os.Stat(flagValue)
+		if err != nil {
+			// If output path does not exist it's ok - we will create it
 			if errors.Is(err, os.ErrNotExist) {
+				c.OutputPath = flagValue
 				return nil
 			}
 			return err
 		}
 
+		// If output path exists check if it's a directory - which would be wrong
 		if fileInfo.IsDir() {
 			return fmt.Errorf("Output file %s cannot be a directory.\n", flagValue)
 		}
 
-		outputPath = flagValue
+		// If output path exists and is not a dir it's ok - file will be truncated
+		c.OutputPath = flagValue
 
 		return nil
 	}
 }
 
-func validateDir(dir string) func(string) error {
+func (c *Config) validateDirPath() func(string) error {
 	return func(flagValue string) error {
 		fileInfo, err := os.Stat(flagValue)
 		if err != nil {
@@ -54,7 +75,7 @@ func validateDir(dir string) func(string) error {
 			return fmt.Errorf("Path to anonymization data %s needs to be a directory.\n", flagValue)
 		}
 
-		dir = flagValue
+		c.AnonymizationDataPath = flagValue
 
 		return nil
 	}

internal/handlers/buffers.go

@@ -0,0 +1,24 @@
+package handlers
+
+import (
+	"bufio"
+	"log/slog"
+)
+
+type Buffers struct {
+	buffers []*bufio.Writer
+}
+
+func (b *Buffers) Add(buffer *bufio.Writer) {
+	b.buffers = append(b.buffers, buffer)
+}
+
+func (b *Buffers) Flush() {
+	slog.Debug("flush buffers on exit...")
+	for _, buffer := range b.buffers {
+		err := buffer.Flush()
+		if err != nil {
+			slog.Error("buffer flush failed", "error", err)
+		}
+	}
+}

internal/handlers/files.go

@@ -0,0 +1,21 @@
+package handlers
+
+import (
+	"log/slog"
+	"os"
+)
+
+type Files struct {
+	files []*os.File
+}
+
+func (f *Files) Add(file *os.File) {
+	f.files = append(f.files, file)
+}
+
+func (f *Files) Close() {
+	slog.Debug("closing files on exit...")
+	for _, file := range f.files {
+		file.Close()
+	}
+}

internal/loader/loader.go

@@ -13,28 +13,30 @@ import (
 func LoadCustomReplacementMap(path string) (map[string]string, error) {
 	customReplacementMap := make(map[string]string)
 
-	file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		line := scanner.Text()
-		values := strings.Split(line, ":")
-		if len(values) == 1 {
-			slog.Error("wrong custom mapping: %s", "error", line)
+	if path != "" {
+		file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
+		if err != nil {
+			return nil, err
 		}
+		defer file.Close()
 
-		originalValue := values[0]
-		newValue := values[1]
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			line := scanner.Text()
+			values := strings.Split(line, ":")
+			if len(values) == 1 {
+				slog.Error("wrong custom mapping: %s", "error", line)
+			}
 
-		customReplacementMap[originalValue] = newValue
-	}
+			originalValue := values[0]
+			newValue := values[1]
 
-	if err := scanner.Err(); err != nil {
-		return nil, fmt.Errorf("error reading custom anonymization mapping: %w", err)
+			customReplacementMap[originalValue] = newValue
+		}
+
+		if err := scanner.Err(); err != nil {
+			return nil, fmt.Errorf("error reading custom anonymization mapping: %w", err)
+		}
 	}
 
 	return customReplacementMap, nil

internal/proof/proof.go

@@ -8,7 +8,7 @@ import (
 	"os"
 
 	"github.com/logmanager-oss/logveil/internal/config"
-	"github.com/logmanager-oss/logveil/internal/files"
+	"github.com/logmanager-oss/logveil/internal/handlers"
 )
 
 type Proof struct {
@@ -22,17 +22,20 @@ type ProofWriter struct {
 	file      *os.File
 }
 
-func CreateProofWriter(config *config.Config, openFiles *files.FilesHandler) (*ProofWriter, error) {
+func CreateProofWriter(config *config.Config, filesHandler *handlers.Files, buffersHandler *handlers.Buffers) (*ProofWriter, error) {
 	if config.IsProofWriter {
 		file, err := os.Create(ProofFilename)
 		if err != nil {
 			return nil, fmt.Errorf("creating/opening proof file: %v", err)
 		}
-		openFiles.Add(file)
+		filesHandler.Add(file)
+
+		proofWriter := bufio.NewWriter(file)
+		buffersHandler.Add(proofWriter)
 
 		return &ProofWriter{
 			IsEnabled: true,
-			writer:    bufio.NewWriter(file),
+			writer:    proofWriter,
 			file:      file,
 		}, nil
 	}

internal/proof/proof_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/logmanager-oss/logveil/internal/config"
-	"github.com/logmanager-oss/logveil/internal/files"
+	"github.com/logmanager-oss/logveil/internal/handlers"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -36,10 +36,13 @@ func TestProof_Write(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			filesHandler := &files.FilesHandler{}
+			filesHandler := &handlers.Files{}
 			defer filesHandler.Close()
 
-			p, err := CreateProofWriter(&config.Config{IsProofWriter: tt.isProofWriter}, filesHandler)
+			buffersHandler := &handlers.Buffers{}
+			defer buffersHandler.Flush()
+
+			p, err := CreateProofWriter(&config.Config{IsProofWriter: tt.isProofWriter}, filesHandler, buffersHandler)
 			if err != nil {
 				t.Fatal(err)
 			}