From 8597bf964eb6af47fade78d9f1d8d4991bdbe986 Mon Sep 17 00:00:00 2001
From: Tarun Koyalwar <tarun@projectdiscovery.io>
Date: Mon, 16 Oct 2023 20:20:32 +0530
Subject: [PATCH] cli global dedupe with -dns flag

---
 go.mod               |  2 +-
 pkg/output/output.go | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 5cf97a86..a693d20a 100644
--- a/go.mod
+++ b/go.mod
@@ -90,7 +90,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
 	github.com/projectdiscovery/cdncheck v1.0.9 // indirect
-	github.com/projectdiscovery/hmap v0.0.22
+	github.com/projectdiscovery/hmap v0.0.22 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.6 // indirect
 	github.com/projectdiscovery/retryabledns v1.0.38 // indirect
 	github.com/projectdiscovery/retryablehttp-go v1.0.31
diff --git a/pkg/output/output.go b/pkg/output/output.go
index 2866cc46..0ad65237 100644
--- a/pkg/output/output.go
+++ b/pkg/output/output.go
@@ -12,9 +12,18 @@ import (
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	mapsutil "github.com/projectdiscovery/utils/maps"
 	"golang.org/x/exp/maps"
 )
 
+var (
+	// when unique domains are displayed with `-dns` flag. tlsx json/struct already
+	// contains unique domains for each certificate
+	// globalDedupe is meant to be used when running in cli mode with multiple inputs
+	// ex: google.com and youtube.com may have same wildcard certificate or some overlapping domains
+	globalDedupe = mapsutil.NewSyncLockMap[string, struct{}]()
+)
+
 // Writer is an interface which writes output to somewhere for katana events.
 type Writer interface {
 	// Close closes the output writer interface
@@ -69,6 +78,10 @@ func (w *StandardWriter) Write(event *clients.Response) error {
 		return errorutil.NewWithErr(err).Msgf("could not format output")
 	}
 	data = bytes.TrimSuffix(data, []byte("\n")) // remove last newline
+	if len(data) == 0 {
+		// this happens when -dns flag is used and two domains have same certificate hence deduped
+		return nil
+	}
 
 	w.outputMutex.Lock()
 	defer w.outputMutex.Unlock()
@@ -113,6 +126,10 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error
 
 	if w.options.DisplayDns {
 		for _, hname := range cert.Domains {
+			if _, ok := globalDedupe.Get(hname); ok {
+				continue
+			}
+			_ = globalDedupe.Set(hname, struct{}{})
 			builder.WriteString(hname)
 			builder.WriteString("\n")
 		}