cli global dedupe with -dns flag

go.mod

@@ -90,7 +90,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
 	github.com/projectdiscovery/cdncheck v1.0.9 // indirect
-	github.com/projectdiscovery/hmap v0.0.22
+	github.com/projectdiscovery/hmap v0.0.22 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.6 // indirect
 	github.com/projectdiscovery/retryabledns v1.0.38 // indirect
 	github.com/projectdiscovery/retryablehttp-go v1.0.31

pkg/output/output.go

@@ -12,9 +12,18 @@ import (
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
 	errorutil "github.com/projectdiscovery/utils/errors"
+	mapsutil "github.com/projectdiscovery/utils/maps"
 	"golang.org/x/exp/maps"
 )
 
+var (
+	// when unique domains are displayed with `-dns` flag. tlsx json/struct already
+	// contains unique domains for each certificate
+	// globalDedupe is meant to be used when running in cli mode with multiple inputs
+	// ex: google.com and youtube.com may have same wildcard certificate or some overlapping domains
+	globalDedupe = mapsutil.NewSyncLockMap[string, struct{}]()
+)
+
 // Writer is an interface which writes output to somewhere for katana events.
 type Writer interface {
 	// Close closes the output writer interface
@@ -69,6 +78,10 @@ func (w *StandardWriter) Write(event *clients.Response) error {
 		return errorutil.NewWithErr(err).Msgf("could not format output")
 	}
 	data = bytes.TrimSuffix(data, []byte("\n")) // remove last newline
+	if len(data) == 0 {
+		// this happens when -dns flag is used and two domains have same certificate hence deduped
+		return nil
+	}
 
 	w.outputMutex.Lock()
 	defer w.outputMutex.Unlock()
@@ -113,6 +126,10 @@ func (w *StandardWriter) formatStandard(output *clients.Response) ([]byte, error
 
 	if w.options.DisplayDns {
 		for _, hname := range cert.Domains {
+			if _, ok := globalDedupe.Get(hname); ok {
+				continue
+			}
+			_ = globalDedupe.Set(hname, struct{}{})
 			builder.WriteString(hname)
 			builder.WriteString("\n")
 		}