Feature: optional flag to skip a host if it has a private IP address.

common/httpx/option.go

@@ -8,13 +8,14 @@ import (
 
 // Options contains configuration options for the client
 type Options struct {
-	RandomAgent      bool
-	DefaultUserAgent string
-	HTTPProxy        string
-	SocksProxy       string
-	Threads          int
-	CdnCheck         bool
-	ExcludeCdn       bool
+	RandomAgent         bool
+	DefaultUserAgent    string
+	HTTPProxy           string
+	SocksProxy          string
+	Threads             int
+	CdnCheck            bool
+	ExcludeCdn          bool
+	ExcludePrivateHosts bool
 	// Timeout is the maximum time to wait for the request
 	Timeout time.Duration
 	// RetryMax is the maximum number of retries
@@ -48,14 +49,15 @@ type Options struct {
 
 // DefaultOptions contains the default options
 var DefaultOptions = Options{
-	RandomAgent:  true,
-	Threads:      25,
-	Timeout:      30 * time.Second,
-	RetryMax:     5,
-	MaxRedirects: 10,
-	Unsafe:       false,
-	CdnCheck:     true,
-	ExcludeCdn:   false,
+	RandomAgent:         true,
+	Threads:             25,
+	Timeout:             30 * time.Second,
+	RetryMax:            5,
+	MaxRedirects:        10,
+	Unsafe:              false,
+	CdnCheck:            true,
+	ExcludeCdn:          false,
+	ExcludePrivateHosts: false,
 	// VHOSTs options
 	VHostIgnoreStatusCode:    false,
 	VHostIgnoreContentLength: true,

runner/options.go

@@ -77,6 +77,7 @@ type ScanOptions struct {
 	OutputExtractRegex        string
 	extractRegexps            map[string]*regexp.Regexp
 	ExcludeCDN                bool
+	ExcludePrivateHosts       bool
 	HostMaxErrors             int
 	ProbeAllIPS               bool
 	Favicon                   bool
@@ -241,6 +242,7 @@ type Options struct {
 	Resume                    bool
 	resumeCfg                 *ResumeCfg
 	ExcludeCDN                bool
+	ExcludePrivateHosts       bool
 	HostMaxErrors             int
 	Stream                    bool
 	SkipDedupe                bool
@@ -450,6 +452,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "),
 		flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"),
 		flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDN/WAF (only checks for 80,443)"),
+		flagSet.BoolVarP(&options.ExcludePrivateHosts, "exclude-private-hosts", "eph", false, "skip any hosts which have a private ip address"),
 		flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"),
 		flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"),
 		flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"),

runner/runner.go

@@ -113,6 +113,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.UnsafeURI = options.RequestURI
 	httpxOptions.CdnCheck = options.OutputCDN
 	httpxOptions.ExcludeCdn = options.ExcludeCDN
+	httpxOptions.ExcludePrivateHosts = options.ExcludePrivateHosts
 	if options.CustomHeaders.Has("User-Agent:") {
 		httpxOptions.RandomAgent = false
 	} else {
@@ -277,6 +278,7 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	scanopts.ExcludeCDN = options.ExcludeCDN
+	scanopts.ExcludePrivateHosts = options.ExcludePrivateHosts
 	scanopts.HostMaxErrors = options.HostMaxErrors
 	scanopts.ProbeAllIPS = options.ProbeAllIPS
 	scanopts.Favicon = options.Favicon
@@ -1242,6 +1244,11 @@ retry:
 		}
 	}
 
+	if r.skipPrivateHosts(URL.Host) {
+		gologger.Debug().Msgf("Skipping private host %s\n", URL.Host)
+		return Result{URL: target.Host, Input: origInput, Err: errors.New("target has a private ip and will only connect within same local network")}
+	}
+
 	// check if the combination host:port should be skipped if belonging to a cdn
 	if r.skipCDNPort(URL.Host, URL.Port()) {
 		gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port())
@@ -2135,6 +2142,39 @@ func (r *Runner) skipCDNPort(host string, port string) bool {
 	return false
 }
 
+func (r *Runner) skipPrivateHosts(host string) bool {
+	// if the option is not enabled we don't skip
+	if !r.options.ExcludePrivateHosts {
+		return false
+	}
+	// use the dealer to pre-resolve the target
+	dnsData, err := r.hp.Dialer.GetDNSData(host)
+	// if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly
+	if err != nil {
+		return false
+	}
+
+	if len(dnsData.A) == 0 && len(dnsData.AAAA) == 0 {
+		return false
+	}
+
+	var ipsToCheck []string
+	ipsToCheck = make([]string, 0, len(dnsData.A)+len(dnsData.AAAA))
+	ipsToCheck = append(ipsToCheck, dnsData.A...)
+	ipsToCheck = append(ipsToCheck, dnsData.AAAA...)
+
+	for _, ipAddr := range ipsToCheck {
+		ip := net.ParseIP(ipAddr)
+		if ip == nil {
+			continue //skip any bad ip addresses
+		}
+		if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+			return true
+		}
+	}
+	return false
+}
+
 // parseURL parses url based on cli option(unsafe)
 func (r *Runner) parseURL(url string) (*urlutil.URL, error) {
 	urlx, err := urlutil.ParseURL(url, r.options.Unsafe)