From f1b28cd2f5e85832bdf9cd215b8693f9d94fa6d1 Mon Sep 17 00:00:00 2001 From: CodyCline <20631166+CodyCline@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:48:54 -0700 Subject: [PATCH 1/6] feat: Implement flag which when enabled, will skip any host with a private ip address instead of timing out. --- common/httpx/option.go | 32 +++++++++++++++++--------------- resume.cfg | 2 -- runner/options.go | 3 +++ runner/runner.go | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 17 deletions(-) delete mode 100644 resume.cfg diff --git a/common/httpx/option.go b/common/httpx/option.go index f1de106d7..5e3a98b55 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -8,13 +8,14 @@ import ( // Options contains configuration options for the client type Options struct { - RandomAgent bool - DefaultUserAgent string - HTTPProxy string - SocksProxy string - Threads int - CdnCheck bool - ExcludeCdn bool + RandomAgent bool + DefaultUserAgent string + HTTPProxy string + SocksProxy string + Threads int + CdnCheck bool + ExcludeCdn bool + ExcludePrivateHosts bool // Timeout is the maximum time to wait for the request Timeout time.Duration // RetryMax is the maximum number of retries @@ -48,14 +49,15 @@ type Options struct { // DefaultOptions contains the default options var DefaultOptions = Options{ - RandomAgent: true, - Threads: 25, - Timeout: 30 * time.Second, - RetryMax: 5, - MaxRedirects: 10, - Unsafe: false, - CdnCheck: true, - ExcludeCdn: false, + RandomAgent: true, + Threads: 25, + Timeout: 30 * time.Second, + RetryMax: 5, + MaxRedirects: 10, + Unsafe: false, + CdnCheck: true, + ExcludeCdn: false, + ExcludePrivateHosts: false, // VHOSTs options VHostIgnoreStatusCode: false, VHostIgnoreContentLength: true, diff --git a/resume.cfg b/resume.cfg deleted file mode 100644 index b84a48b69..000000000 --- a/resume.cfg +++ /dev/null @@ -1,2 +0,0 @@ -index=21 -resume_from=www.hackerone.com diff --git a/runner/options.go b/runner/options.go index cdb9cd413..8cb8e4d3e 100644 --- a/runner/options.go +++ b/runner/options.go @@ -77,6 +77,7 @@ type ScanOptions struct { OutputExtractRegex string extractRegexps map[string]*regexp.Regexp ExcludeCDN bool + ExcludePrivateHosts bool HostMaxErrors int ProbeAllIPS bool Favicon bool @@ -241,6 +242,7 @@ type Options struct { Resume bool resumeCfg *ResumeCfg ExcludeCDN bool + ExcludePrivateHosts bool HostMaxErrors int Stream bool SkipDedupe bool @@ -450,6 +452,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "), flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"), flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDN/WAF (only checks for 80,443)"), + flagSet.BoolVarP(&options.ExcludePrivateHosts, "exclude-private-hosts", "eph", false, "skip any hosts which have a private ip address"), flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"), flagSet.IntVar(&options.Timeout, "timeout", 10, "timeout in seconds"), flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"), diff --git a/runner/runner.go b/runner/runner.go index 1bb91b9a5..aeab74b83 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -113,6 +113,7 @@ func New(options *Options) (*Runner, error) { httpxOptions.UnsafeURI = options.RequestURI httpxOptions.CdnCheck = options.OutputCDN httpxOptions.ExcludeCdn = options.ExcludeCDN + httpxOptions.ExcludePrivateHosts = options.ExcludePrivateHosts if options.CustomHeaders.Has("User-Agent:") { httpxOptions.RandomAgent = false } else { @@ -277,6 +278,7 @@ func New(options *Options) (*Runner, error) { } scanopts.ExcludeCDN = options.ExcludeCDN + scanopts.ExcludePrivateHosts = options.ExcludePrivateHosts scanopts.HostMaxErrors = options.HostMaxErrors scanopts.ProbeAllIPS = options.ProbeAllIPS scanopts.Favicon = options.Favicon @@ -1242,6 +1244,11 @@ retry: } } + if r.skipPrivateHosts(URL.Host) { + gologger.Debug().Msgf("Skipping private host %s\n", URL.Host) + return Result{URL: target.Host, Input: origInput, Err: errors.New("target has a private ip address and will not connect")} + } + // check if the combination host:port should be skipped if belonging to a cdn if r.skipCDNPort(URL.Host, URL.Port()) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port()) @@ -2135,6 +2142,39 @@ func (r *Runner) skipCDNPort(host string, port string) bool { return false } +func (r *Runner) skipPrivateHosts(host string) bool { + // if the option is not enabled we don't skip + if !r.options.ExcludePrivateHosts { + return false + } + // use the dealer to pre-resolve the target + dnsData, err := r.hp.Dialer.GetDNSData(host) + // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly + if err != nil { + return false + } + + if len(dnsData.A) == 0 && len(dnsData.AAAA) == 0 { + return false + } + + var ipsToCheck []string + ipsToCheck = make([]string, 0, len(dnsData.A)+len(dnsData.AAAA)) + ipsToCheck = append(ipsToCheck, dnsData.A...) + ipsToCheck = append(ipsToCheck, dnsData.AAAA...) + + for _, ipAddr := range ipsToCheck { + ip := net.ParseIP(ipAddr) + if ip == nil { + continue //skip any bad ip addresses + } + if ip.IsPrivate() { + return true + } + } + return false +} + // parseURL parses url based on cli option(unsafe) func (r *Runner) parseURL(url string) (*urlutil.URL, error) { urlx, err := urlutil.ParseURL(url, r.options.Unsafe) From 6c22e012f54ea69fe732cf3f09d0cb7717f6b424 Mon Sep 17 00:00:00 2001 From: CodyCline <20631166+CodyCline@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:25:00 -0700 Subject: [PATCH 2/6] fix: Amend private ip check to include loopback and link-local addresses --- runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index aeab74b83..08ea80a64 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -2168,7 +2168,7 @@ func (r *Runner) skipPrivateHosts(host string) bool { if ip == nil { continue //skip any bad ip addresses } - if ip.IsPrivate() { + if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { return true } } From 75d49d459804b90588af6f252d47bb5d3ff5294a Mon Sep 17 00:00:00 2001 From: CodyCline <20631166+CodyCline@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:29:35 -0700 Subject: [PATCH 3/6] fix: Amend debug message to be more concise --- runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 08ea80a64..103f64fc4 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1246,7 +1246,7 @@ retry: if r.skipPrivateHosts(URL.Host) { gologger.Debug().Msgf("Skipping private host %s\n", URL.Host) - return Result{URL: target.Host, Input: origInput, Err: errors.New("target has a private ip address and will not connect")} + return Result{URL: target.Host, Input: origInput, Err: errors.New("target has a private ip and will only connect within same local network")} } // check if the combination host:port should be skipped if belonging to a cdn From 47b5825e570d62b2841aea38a6ff3a923929f139 Mon Sep 17 00:00:00 2001 From: CodyCline <20631166+CodyCline@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:20:58 -0700 Subject: [PATCH 4/6] fix: Account for port numbers in skipPrivateHosts feature --- runner/runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 103f64fc4..01f3007c7 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1244,7 +1244,7 @@ retry: } } - if r.skipPrivateHosts(URL.Host) { + if r.skipPrivateHosts(URL.Hostname()) { gologger.Debug().Msgf("Skipping private host %s\n", URL.Host) return Result{URL: target.Host, Input: origInput, Err: errors.New("target has a private ip and will only connect within same local network")} } @@ -2147,13 +2147,13 @@ func (r *Runner) skipPrivateHosts(host string) bool { if !r.options.ExcludePrivateHosts { return false } - // use the dealer to pre-resolve the target dnsData, err := r.hp.Dialer.GetDNSData(host) + // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly if err != nil { return false } - + gologger.Debug().Msgf("%v, %v", dnsData.A, dnsData.AAAA) if len(dnsData.A) == 0 && len(dnsData.AAAA) == 0 { return false } From 7e19413938f2eac55ee1f7aa3ff648dc40d718ea Mon Sep 17 00:00:00 2001 From: CodyCline <20631166+CodyCline@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:23:52 -0700 Subject: [PATCH 5/6] docs: Remove uncessary debug statement --- runner/runner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 01f3007c7..13360c2d6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -2153,7 +2153,6 @@ func (r *Runner) skipPrivateHosts(host string) bool { if err != nil { return false } - gologger.Debug().Msgf("%v, %v", dnsData.A, dnsData.AAAA) if len(dnsData.A) == 0 && len(dnsData.AAAA) == 0 { return false } From 07cc402a2a7a6bbaccfedcecaeebb6ccb2eb7212 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 2 Nov 2023 14:08:35 +0530 Subject: [PATCH 6/6] readme update --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 83e46452e..74a49534c 100644 --- a/README.md +++ b/README.md @@ -230,6 +230,7 @@ OPTIMIZATIONS: -nfs, -no-fallback-scheme probe with protocol scheme specified in input -maxhr, -max-host-error int max error count per host before skipping remaining path/s (default 30) -ec, -exclude-cdn skip full port scans for CDN/WAF (only checks for 80,443) + -eph, -exclude-private-hosts skip any hosts which have a private ip address -retries int number of retries -timeout int timeout in seconds (default 10) -delay value duration between each http request (eg: 200ms, 1s) (default -1ns)