From f6842be9b989a3943121f88b3d3bf40194fbc1c5 Mon Sep 17 00:00:00 2001 From: solnicki Date: Fri, 8 Nov 2024 10:24:58 +0100 Subject: [PATCH] implement anonymisation proof writer --- cmd/logveil/app.go | 67 ---------------------- cmd/logveil/logveil.go | 6 +- go.mod | 1 + go.sum | 2 + internal/anonymizer/anonymizer.go | 15 +++-- internal/anonymizer/anonymizer_test.go | 4 +- internal/flags/initalize.go | 9 +-- internal/inputs/backup_test.go | 47 +++++++++++----- internal/inputs/export_test.go | 46 ++++++++++----- internal/proof/proof.go | 77 ++++++++++++++++++++++++++ internal/proof/proof_test.go | 61 ++++++++++++++++++++ internal/runner/backup_test.go | 23 +++++++- internal/runner/export_test.go | 24 +++++++- internal/utils/utils.go | 41 ++++++++++++++ 14 files changed, 315 insertions(+), 108 deletions(-) delete mode 100644 cmd/logveil/app.go create mode 100644 internal/proof/proof.go create mode 100644 internal/proof/proof_test.go create mode 100644 internal/utils/utils.go diff --git a/cmd/logveil/app.go b/cmd/logveil/app.go deleted file mode 100644 index 62fe089..0000000 --- a/cmd/logveil/app.go +++ /dev/null @@ -1,67 +0,0 @@ -package logveil - -import ( - "io" - "log/slog" - "os" - - "github.com/logmanager-oss/logveil/internal/anonymizer" - "github.com/logmanager-oss/logveil/internal/flags" - "github.com/logmanager-oss/logveil/internal/inputs" - "github.com/logmanager-oss/logveil/internal/parser" -) - -func Run() { - slog.Info("Anonymization process started...") - - anonDataDir, inputFilename, outputFilename, enableLMexport := flags.Load() - - input, err := os.Open(inputFilename) - if err != nil { - slog.Error("Opening input file", "error", err) - return - } - defer closeFile(input) - - var output io.Writer - if outputFilename != "" { - output, err = os.Create(outputFilename) - if err != nil { - slog.Error("Opening input file", "error", err) - return - } - defer closeFile(output.(*os.File)) - } else { - output = os.Stdout - } - - anonData, err := parser.LoadAnonData(anonDataDir) - if err != nil { - slog.Error("loading anonymizing data from dir %s: %v", anonDataDir, err) - return - } - anonymizer := anonymizer.New(anonData) - - if enableLMexport { - err := inputs.AnonymizeLmExport(input, output, anonymizer) - if err != nil { - slog.Error("reading lm export input file %s: %v", inputFilename, err) - return - } - } else { - err := inputs.AnonymizeLmBackup(input, output, anonymizer) - if err != nil { - slog.Error("reading lm backup input file %s: %v", inputFilename, err) - return - } - } - - slog.Info("All done. Exiting...") -} - -func closeFile(fs *os.File) { - err := fs.Close() - if err != nil { - slog.Error(err.Error()) - } -} diff --git a/cmd/logveil/logveil.go b/cmd/logveil/logveil.go index ebace4f..aa6ec67 100644 --- a/cmd/logveil/logveil.go +++ b/cmd/logveil/logveil.go @@ -8,13 +8,14 @@ import ( "github.com/logmanager-oss/logveil/internal/anonymizer" "github.com/logmanager-oss/logveil/internal/flags" "github.com/logmanager-oss/logveil/internal/loader" + "github.com/logmanager-oss/logveil/internal/proof" "github.com/logmanager-oss/logveil/internal/runner" ) func Run() { slog.Info("Anonymization process started...") - anonymizingDataDir, inputPath, outputPath, isVerbose, isLmExport := flags.LoadAndValidate() + anonymizingDataDir, inputPath, outputPath, isVerbose, isLmExport, isProofWriter := flags.LoadAndValidate() if isVerbose { slog.SetLogLoggerLevel(slog.LevelDebug) @@ -45,7 +46,8 @@ func Run() { slog.Error("loading anonymizing data from dir %s: %v", anonymizingDataDir, err) return } - anonymizer := anonymizer.New(anonymizingData) + proofWriter := proof.New(isProofWriter) + anonymizer := anonymizer.New(anonymizingData, proofWriter) if isLmExport { err := runner.AnonymizeLmExport(inputReader, outputWriter, anonymizer) diff --git a/go.mod b/go.mod index 662393f..dc2e5f8 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require github.com/stretchr/testify v1.9.0 require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/exp v0.0.0-20240716175740-e3f259677ff7 gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index fd1583e..6bc9d5c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 h1:dOYG7LS/WK00RWZc8XGgcUTlTxpp3mKhdR2Q9z9HbXM= +github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= diff --git a/internal/anonymizer/anonymizer.go b/internal/anonymizer/anonymizer.go index 2a96c20..f9cae9e 100644 --- a/internal/anonymizer/anonymizer.go +++ b/internal/anonymizer/anonymizer.go @@ -5,18 +5,21 @@ import ( "log/slog" "strings" + "github.com/logmanager-oss/logveil/internal/proof" "golang.org/x/exp/rand" ) type Anonymizer struct { - anonData map[string][]string - randFunc func(int) int + anonData map[string][]string + randFunc func(int) int + proofWriter *proof.Proof } -func New(anonData map[string][]string) *Anonymizer { +func New(anonData map[string][]string, proofWriter *proof.Proof) *Anonymizer { return &Anonymizer{ - anonData: anonData, - randFunc: rand.Intn, + anonData: anonData, + randFunc: rand.Intn, + proofWriter: proofWriter, } } @@ -33,6 +36,8 @@ func (an *Anonymizer) Anonymize(logLine map[string]string) string { if anonValues, exists := an.anonData[field]; exists { newAnonValue := anonValues[an.randFunc(len(anonValues))] + an.proofWriter.Write(value, newAnonValue) + slog.Debug(fmt.Sprintf("Replacing the values for field %s. From %s to %s.\n", field, value, newAnonValue)) logLine["raw"] = strings.Replace(logLine["raw"], value, newAnonValue, -1) diff --git a/internal/anonymizer/anonymizer_test.go b/internal/anonymizer/anonymizer_test.go index 5e2c69f..75d01a5 100644 --- a/internal/anonymizer/anonymizer_test.go +++ b/internal/anonymizer/anonymizer_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/logmanager-oss/logveil/internal/loader" + "github.com/logmanager-oss/logveil/internal/proof" "github.com/stretchr/testify/assert" ) @@ -29,7 +30,8 @@ func TestAnonimizer_AnonymizeData(t *testing.T) { t.Fatalf("loading anonymizing data from dir %s: %v", tt.anonymizingDataDir, err) } - anonymizer := New(anonymizingData) + proofWriter := proof.New(true) + anonymizer := New(anonymizingData, proofWriter) // Disabling randomization so we know which values to expect anonymizer.SetRandFunc(func(int) int { return 1 }) output := anonymizer.Anonymize(tt.input) diff --git a/internal/flags/initalize.go b/internal/flags/initalize.go index 2f641af..f5f4024 100644 --- a/internal/flags/initalize.go +++ b/internal/flags/initalize.go @@ -4,7 +4,7 @@ import ( "flag" ) -func LoadAndValidate() (string, string, string, bool, bool) { +func LoadAndValidate() (string, string, string, bool, bool, bool) { var anonymizationDataPath string flag.Func("d", "Path to directory with anonymizing data", validateDir(anonymizationDataPath)) @@ -14,10 +14,11 @@ func LoadAndValidate() (string, string, string, bool, bool) { var outputPath string flag.Func("o", "Path to output file (default: Stdout)", validateOutput(outputPath)) - var isVerbose = flag.Bool("v", false, "Enable verbose logging") - var isLmExport = flag.Bool("e", false, "Change input file type to LM export (default input file type is LM Backup)") + var isVerbose = flag.Bool("v", false, "Enable verbose logging (default: Disabled)") + var isLmExport = flag.Bool("e", false, "Change input file type to LM export (default: LM Backup)") + var isProofWriter = flag.Bool("p", true, "Disable proof wrtier (default: Enabled)") flag.Parse() - return anonymizationDataPath, inputPath, outputPath, *isVerbose, *isLmExport + return anonymizationDataPath, inputPath, outputPath, *isVerbose, *isLmExport, *isProofWriter } diff --git a/internal/inputs/backup_test.go b/internal/inputs/backup_test.go index 42f65f9..b1d3cec 100644 --- a/internal/inputs/backup_test.go +++ b/internal/inputs/backup_test.go @@ -7,22 +7,32 @@ import ( "github.com/logmanager-oss/logveil/internal/anonymizer" "github.com/logmanager-oss/logveil/internal/parser" + "github.com/logmanager-oss/logveil/internal/proof" + "github.com/logmanager-oss/logveil/internal/utils" "github.com/stretchr/testify/assert" ) func TestLmBackup(t *testing.T) { tests := []struct { - name string - inputFilename string - outputFilename string - anonDataDir string - expectedOutput string + name string + isProofWriterEnabled bool + inputFilename string + anonDataDir string + expectedOutput string + expectedProof []map[string]interface{} }{ { - name: "Test Test LM Backup Anonymizer", - inputFilename: "../../examples/logs/lm-2024-06-09_0000.gz", - anonDataDir: "../../examples/anon_data", - expectedOutput: "<189>date=2024-11-06 time=12:29:25 devname=\"LM-FW-70F-Praha\" devid=\"FGT70FTK22012016\" eventtime=1730892565525108329 tz=\"+0100\" logid=\"0000000013\" type=\"traffic\" subtype=\"forward\" level=\"notice\" vd=\"root\" srcip=10.20.0.53 srcport=57158 srcintf=\"lan1\" srcintfrole=\"wan\" dstip=227.51.221.89 dstport=80 dstintf=\"lan1\" dstintfrole=\"lan\" srccountry=\"China\" dstcountry=\"Czech Republic\" sessionid=179455916 proto=6 action=\"client-rst\" policyid=9 policytype=\"policy\" poluuid=\"d8ccb3e4-74d4-51ef-69a3-73b41f46df74\" policyname=\"Gitlab web from all\" service=\"HTTP\" trandisp=\"noop\" duration=6 sentbyte=80 rcvdbyte=44 sentpkt=2 rcvdpkt=1 appcat=\"unscanned\" srchwvendor=\"H3C\" devtype=\"Router\" mastersrcmac=\"00:23:89:39:a4:ef\" srcmac=\"00:23:89:39:a4:ef\" srcserver=0 dsthwvendor=\"H3C\" dstdevtype=\"Router\" masterdstmac=\"00:23:89:39:a4:fa\" dstmac=\"00:23:89:39:a4:fa\" dstserver=0\n", + name: "Test Test LM Backup Anonymizer", + isProofWriterEnabled: true, + inputFilename: "../../examples/logs/lm-2024-06-09_0000.gz", + anonDataDir: "../../examples/anon_data", + expectedOutput: "<189>date=2024-11-06 time=12:29:25 devname=\"LM-FW-70F-Praha\" devid=\"FGT70FTK22012016\" eventtime=1730892565525108329 tz=\"+0100\" logid=\"0000000013\" type=\"traffic\" subtype=\"forward\" level=\"notice\" vd=\"root\" srcip=10.20.0.53 srcport=57158 srcintf=\"lan1\" srcintfrole=\"wan\" dstip=227.51.221.89 dstport=80 dstintf=\"lan1\" dstintfrole=\"lan\" srccountry=\"China\" dstcountry=\"Czech Republic\" sessionid=179455916 proto=6 action=\"client-rst\" policyid=9 policytype=\"policy\" poluuid=\"d8ccb3e4-74d4-51ef-69a3-73b41f46df74\" policyname=\"Gitlab web from all\" service=\"HTTP\" trandisp=\"noop\" duration=6 sentbyte=80 rcvdbyte=44 sentpkt=2 rcvdpkt=1 appcat=\"unscanned\" srchwvendor=\"H3C\" devtype=\"Router\" mastersrcmac=\"00:23:89:39:a4:ef\" srcmac=\"00:23:89:39:a4:ef\" srcserver=0 dsthwvendor=\"H3C\" dstdevtype=\"Router\" masterdstmac=\"00:23:89:39:a4:fa\" dstmac=\"00:23:89:39:a4:fa\" dstserver=0\n", + expectedProof: []map[string]interface{}{ + {"original": "dev-uplink", "new": "lan1"}, + {"original": "95.80.197.108", "new": "227.51.221.89"}, + {"original": "27.221.126.209", "new": "10.20.0.53"}, + {"original": "wan1-lm", "new": "lan1"}, + }, }, } for _, tt := range tests { @@ -33,22 +43,33 @@ func TestLmBackup(t *testing.T) { } defer input.Close() - var output bytes.Buffer - anonData, err := parser.LoadAnonData(tt.anonDataDir) if err != nil { t.Fatal(err) } - anonymizer := anonymizer.New(anonData) + + proofWriter := proof.New(tt.isProofWriterEnabled) + anonymizer := anonymizer.New(anonData, proofWriter) // Disabling randomization so we know which values to expect anonymizer.SetRandFunc(func(int) int { return 1 }) + var output bytes.Buffer err = AnonymizeLmBackup(input, &output, anonymizer) if err != nil { t.Fatal(err) } - assert.Equal(t, tt.expectedOutput, output.String()) + + proofWriter.Close() + + actualProof, err := utils.UnpackProofOutput() + if err != nil { + t.Fatal(err) + } + + assert.ElementsMatch(t, tt.expectedProof, actualProof) + + os.Remove("proof.json") }) } } diff --git a/internal/inputs/export_test.go b/internal/inputs/export_test.go index 212d69d..f22caf0 100644 --- a/internal/inputs/export_test.go +++ b/internal/inputs/export_test.go @@ -7,22 +7,31 @@ import ( "github.com/logmanager-oss/logveil/internal/anonymizer" "github.com/logmanager-oss/logveil/internal/parser" + "github.com/logmanager-oss/logveil/internal/proof" + "github.com/logmanager-oss/logveil/internal/utils" "github.com/stretchr/testify/assert" ) func TestLmExport(t *testing.T) { tests := []struct { - name string - inputFilename string - outputFilename string - anonDataDir string - expectedOutput string + name string + isProofWriterEnabled bool + inputFilename string + anonDataDir string + expectedOutput string + expectedProof []map[string]interface{} }{ { - name: "Test LM Export Anonymizer", - inputFilename: "../../examples/logs/example_logs.csv", - anonDataDir: "../../examples/anon_data", - expectedOutput: "{\"@timestamp\": \"2024-06-05T14:59:27.000+00:00\", \"msg.src_ip\":\"10.20.0.53\", \"username\":\"ladislav.dosek\", \"organization\":\"Apple\"}\n", + name: "Test LM Export Anonymizer", + isProofWriterEnabled: true, + inputFilename: "../../examples/logs/example_logs.csv", + anonDataDir: "../../examples/anon_data", + expectedOutput: "{\"@timestamp\": \"2024-06-05T14:59:27.000+00:00\", \"msg.src_ip\":\"10.20.0.53\", \"username\":\"ladislav.dosek\", \"organization\":\"Apple\"}\n", + expectedProof: []map[string]interface{}{ + {"original": "89.239.31.49", "new": "10.20.0.53"}, + {"original": "test.user@test.cz", "new": "ladislav.dosek"}, + {"original": "TESTuser.test.com", "new": "Apple"}, + }, }, } for _, tt := range tests { @@ -33,22 +42,33 @@ func TestLmExport(t *testing.T) { } defer input.Close() - var output bytes.Buffer - anonData, err := parser.LoadAnonData(tt.anonDataDir) if err != nil { t.Fatal(err) } - anonymizer := anonymizer.New(anonData) + + proofWriter := proof.New(tt.isProofWriterEnabled) + anonymizer := anonymizer.New(anonData, proofWriter) // Disabling randomization so we know which values to expect anonymizer.SetRandFunc(func(int) int { return 1 }) + var output bytes.Buffer err = AnonymizeLmExport(input, &output, anonymizer) if err != nil { t.Fatal(err) } - assert.Equal(t, tt.expectedOutput, output.String()) + + proofWriter.Close() + + actualProof, err := utils.UnpackProofOutput() + if err != nil { + t.Fatal(err) + } + + assert.ElementsMatch(t, tt.expectedProof, actualProof) + + os.Remove("proof.json") }) } } diff --git a/internal/proof/proof.go b/internal/proof/proof.go new file mode 100644 index 0000000..8f85d45 --- /dev/null +++ b/internal/proof/proof.go @@ -0,0 +1,77 @@ +package proof + +import ( + "bufio" + "encoding/json" + "fmt" + "log/slog" + "os" + + "github.com/logmanager-oss/logveil/internal/utils" +) + +type Proof struct { + isEnabled bool + writer *bufio.Writer + file *os.File +} + +func New(isEnabled bool) *Proof { + var writer *bufio.Writer + var file *os.File + + if isEnabled { + var err error + proofFile, err := os.OpenFile("proof.json", os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + slog.Error("opening/creating proof file", "error", err) + return nil + } + + writer = bufio.NewWriter(proofFile) + file = proofFile + } + + return &Proof{ + isEnabled: isEnabled, + writer: writer, + file: file, + } +} + +func (p *Proof) Write(originalValue string, maskedValue string) { + if !p.isEnabled { + return + } + + proof := struct { + OriginalValue string `json:"original"` + MaskedValue string `json:"new"` + }{ + OriginalValue: originalValue, + MaskedValue: maskedValue, + } + + bytes, err := json.Marshal(proof) + if err != nil { + slog.Error("marshalling anonymisation proof", "error", err) + } + + _, err = fmt.Fprintf(p.writer, "%s\n", bytes) + if err != nil { + slog.Error("writing anonymisation proof", "error", err) + } +} + +func (p *Proof) Close() { + if !p.isEnabled { + return + } + + err := p.writer.Flush() + if err != nil { + slog.Error("flushing buffer", "error", err) + } + + utils.CloseFile(p.file) +} diff --git a/internal/proof/proof_test.go b/internal/proof/proof_test.go new file mode 100644 index 0000000..ede9f1f --- /dev/null +++ b/internal/proof/proof_test.go @@ -0,0 +1,61 @@ +package proof + +import ( + "bytes" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProof_Write(t *testing.T) { + tests := []struct { + name string + isProofWriterEnabled bool + originalValue string + maskedValue string + expectedOutput string + }{ + { + name: "Test case 1: write proof", + isProofWriterEnabled: true, + originalValue: "test", + maskedValue: "masked", + expectedOutput: "{\"original\":\"test\",\"new\":\"masked\"}\n", + }, + { + name: "Test case 2: proof writer disabled", + isProofWriterEnabled: false, + originalValue: "test", + maskedValue: "masked", + expectedOutput: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := New(tt.isProofWriterEnabled) + + p.Write(tt.originalValue, tt.maskedValue) + + p.Close() + + file, err := os.OpenFile("proof.json", os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + t.Fatal(err) + } + + buf := bytes.NewBuffer(nil) + _, err = io.Copy(buf, file) + if err != nil { + t.Fatal(err) + } + + file.Close() + + assert.Equal(t, tt.expectedOutput, buf.String()) + + os.Remove("proof.json") + }) + } +} diff --git a/internal/runner/backup_test.go b/internal/runner/backup_test.go index 384ef24..9081c5a 100644 --- a/internal/runner/backup_test.go +++ b/internal/runner/backup_test.go @@ -7,6 +7,8 @@ import ( "github.com/logmanager-oss/logveil/internal/anonymizer" "github.com/logmanager-oss/logveil/internal/loader" + "github.com/logmanager-oss/logveil/internal/proof" + "github.com/logmanager-oss/logveil/internal/utils" "github.com/stretchr/testify/assert" ) @@ -16,12 +18,19 @@ func TestLmBackup(t *testing.T) { inputFilename string anonymizingDataDir string expectedOutput string + expectedProof []map[string]interface{} }{ { name: "Test Test LM Backup Anonymizer", inputFilename: "../../examples/logs/lm-2024-06-09_0000.gz", anonymizingDataDir: "../../examples/anon_data", expectedOutput: "<189>date=2024-11-06 time=12:29:25 devname=\"LM-FW-70F-Praha\" devid=\"FGT70FTK22012016\" eventtime=1730892565525108329 tz=\"+0100\" logid=\"0000000013\" type=\"traffic\" subtype=\"forward\" level=\"notice\" vd=\"root\" srcip=10.20.0.53 srcport=57158 srcintf=\"lan1\" srcintfrole=\"wan\" dstip=227.51.221.89 dstport=80 dstintf=\"lan1\" dstintfrole=\"lan\" srccountry=\"China\" dstcountry=\"Czech Republic\" sessionid=179455916 proto=6 action=\"client-rst\" policyid=9 policytype=\"policy\" poluuid=\"d8ccb3e4-74d4-51ef-69a3-73b41f46df74\" policyname=\"Gitlab web from all\" service=\"HTTP\" trandisp=\"noop\" duration=6 sentbyte=80 rcvdbyte=44 sentpkt=2 rcvdpkt=1 appcat=\"unscanned\" srchwvendor=\"H3C\" devtype=\"Router\" mastersrcmac=\"00:23:89:39:a4:ef\" srcmac=\"00:23:89:39:a4:ef\" srcserver=0 dsthwvendor=\"H3C\" dstdevtype=\"Router\" masterdstmac=\"00:23:89:39:a4:fa\" dstmac=\"00:23:89:39:a4:fa\" dstserver=0\n", + expectedProof: []map[string]interface{}{ + {"original": "dev-uplink", "new": "lan1"}, + {"original": "95.80.197.108", "new": "227.51.221.89"}, + {"original": "27.221.126.209", "new": "10.20.0.53"}, + {"original": "wan1-lm", "new": "lan1"}, + }, }, } for _, tt := range tests { @@ -38,7 +47,8 @@ func TestLmBackup(t *testing.T) { if err != nil { t.Fatal(err) } - anonymizer := anonymizer.New(anonymizingData) + proofWriter := proof.New(true) + anonymizer := anonymizer.New(anonymizingData, proofWriter) // Disabling randomization so we know which values to expect anonymizer.SetRandFunc(func(int) int { return 1 }) @@ -47,6 +57,17 @@ func TestLmBackup(t *testing.T) { t.Fatal(err) } + proofWriter.Close() + + actualProof, err := utils.UnpackProofOutput() + if err != nil { + t.Fatal(err) + } + + assert.ElementsMatch(t, tt.expectedProof, actualProof) + + os.Remove("proof.json") + assert.Equal(t, tt.expectedOutput, output.String()) }) } diff --git a/internal/runner/export_test.go b/internal/runner/export_test.go index 3a22878..5906509 100644 --- a/internal/runner/export_test.go +++ b/internal/runner/export_test.go @@ -7,6 +7,8 @@ import ( "github.com/logmanager-oss/logveil/internal/anonymizer" "github.com/logmanager-oss/logveil/internal/loader" + "github.com/logmanager-oss/logveil/internal/proof" + "github.com/logmanager-oss/logveil/internal/utils" "github.com/stretchr/testify/assert" ) @@ -17,12 +19,18 @@ func TestLmExport(t *testing.T) { outputFilename string anonymizingData string expectedOutput string + expectedProof []map[string]interface{} }{ { name: "Test LM Export Anonymizer", inputFilename: "../../examples/logs/example_logs.csv", anonymizingData: "../../examples/anon_data", expectedOutput: "{\"@timestamp\": \"2024-06-05T14:59:27.000+00:00\", \"msg.src_ip\":\"10.20.0.53\", \"username\":\"ladislav.dosek\", \"organization\":\"Apple\"}\n", + expectedProof: []map[string]interface{}{ + {"original": "89.239.31.49", "new": "10.20.0.53"}, + {"original": "test.user@test.cz", "new": "ladislav.dosek"}, + {"original": "TESTuser.test.com", "new": "Apple"}, + }, }, } for _, tt := range tests { @@ -35,11 +43,12 @@ func TestLmExport(t *testing.T) { var output bytes.Buffer - anonData, err := loader.Load(tt.anonymizingData) + anonymizingData, err := loader.Load(tt.anonymizingData) if err != nil { t.Fatal(err) } - anonymizer := anonymizer.New(anonData) + proofWriter := proof.New(true) + anonymizer := anonymizer.New(anonymizingData, proofWriter) // Disabling randomization so we know which values to expect anonymizer.SetRandFunc(func(int) int { return 1 }) @@ -49,6 +58,17 @@ func TestLmExport(t *testing.T) { } assert.Equal(t, tt.expectedOutput, output.String()) + + proofWriter.Close() + + actualProof, err := utils.UnpackProofOutput() + if err != nil { + t.Fatal(err) + } + + assert.ElementsMatch(t, tt.expectedProof, actualProof) + + os.Remove("proof.json") }) } } diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..2775fab --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,41 @@ +package utils + +import ( + "bufio" + "encoding/json" + "log/slog" + "os" +) + +func CloseFile(fs *os.File) { + err := fs.Close() + if err != nil { + slog.Error(err.Error()) + } +} + +func UnpackProofOutput() ([]map[string]interface{}, error) { + outputFile, err := os.OpenFile("proof.json", os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return nil, err + } + + var output []map[string]interface{} + scanner := bufio.NewScanner(outputFile) + for scanner.Scan() { + var unpackedLine map[string]interface{} + line := scanner.Bytes() + err := json.Unmarshal(line, &unpackedLine) + if err != nil { + return nil, err + } + output = append(output, unpackedLine) + } + + err = scanner.Err() + if err != nil { + return nil, err + } + + return output, nil +}