diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..dbc790e5 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,30 @@ +name: Build Docker Images + +on: + push: + branches: + - main + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +jobs: + Dockerhub: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: checkout sources + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push + uses: docker/build-push-action@v4 + with: + push: true + tags: | + ghcr.io/OJ/gobuster:latest diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 81c02bf2..ad185642 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -6,10 +6,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: ["1.18", "1.19"] + go: ["1.18", "1.19", "stable"] steps: - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 010bc9c9..7919e38f 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -7,9 +7,9 @@ jobs: steps: - uses: actions/checkout@v3.3.0 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: - go-version: "^1.19" + go-version: "stable" - name: golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 835c57b8..9f345ecf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,20 +13,23 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.2.0 with: fetch-depth: 0 + - name: Fetch all tags run: git fetch --force --tags + - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: "stable" + - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 + uses: goreleaser/goreleaser-action@v4.4.0 with: distribution: goreleaser version: latest - args: release --rm-dist + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 6b8cb77e..a65edfee 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -13,24 +13,33 @@ builds: - linux - windows - darwin + archives: - format: tar.gz + # this name template makes the OS and Arch compatible with the results of uname. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives format_overrides: - - goos: windows - format: zip - replacements: - darwin: Darwin - linux: Linux - windows: Windows - 386: i386 - amd64: x86_64 + - goos: windows + format: zip checksum: - name_template: "checksums.txt" + name_template: 'checksums.txt' snapshot: - name_template: "{{ incpatch .Version }}-dev" + name_template: "{{ incpatch .Version }}-next" changelog: sort: asc filters: exclude: - - "^docs:" - - "^test:" + - '^docs:' + - '^test:' + +# The lines beneath this are called `modelines`. See `:help modeline` +# Feel free to remove those if you don't want/use them. +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/README.md b/README.md index 6c28dd5c..8a8a2091 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,13 @@ All funds that are donated to this project will be donated to charity. A full lo # Changes +## 3.6 + +- Wordlist offset parameter to skip x lines from the wordlist +- prevent double slashes when building up an url in dir mode +- allow for multiple values and ranges on `--exclude-length` +- `no-fqdn` parameter on dns bruteforce to disable the use of the systems search domains. This should speed up the run if you have configured some search domains. [https://github.com/OJ/gobuster/pull/418](https://github.com/OJ/gobuster/pull/418) + ## 3.5 - Allow Ranges in status code and status code blacklist. Example: 200,300-305,404 diff --git a/cli/cmd/dir.go b/cli/cmd/dir.go index 194a75ae..0ea9834b 100644 --- a/cli/cmd/dir.go +++ b/cli/cmd/dir.go @@ -7,7 +7,6 @@ import ( "github.com/OJ/gobuster/v3/cli" "github.com/OJ/gobuster/v3/gobusterdir" - "github.com/OJ/gobuster/v3/helper" "github.com/OJ/gobuster/v3/libgobuster" "github.com/spf13/cobra" ) @@ -26,11 +25,13 @@ func runDir(cmd *cobra.Command, args []string) error { return fmt.Errorf("error on creating gobusterdir: %w", err) } - if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { + log := libgobuster.NewLogger(globalopts.Debug) + if err := cli.Gobuster(mainContext, globalopts, plugin, log); err != nil { var wErr *gobusterdir.ErrWildcard if errors.As(err, &wErr) { return fmt.Errorf("%w. To continue please exclude the status code or the length", wErr) } + log.Debugf("%#v", err) return fmt.Errorf("error on running gobuster: %w", err) } return nil @@ -69,7 +70,7 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { return nil, nil, fmt.Errorf("invalid value for extensions: %w", err) } - ret, err := helper.ParseExtensions(pluginOpts.Extensions) + ret, err := libgobuster.ParseExtensions(pluginOpts.Extensions) if err != nil { return nil, nil, fmt.Errorf("invalid value for extensions: %w", err) } @@ -81,7 +82,7 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { } if pluginOpts.ExtensionsFile != "" { - extensions, err := helper.ParseExtensionsFile(pluginOpts.ExtensionsFile) + extensions, err := libgobuster.ParseExtensionsFile(pluginOpts.ExtensionsFile) if err != nil { return nil, nil, fmt.Errorf("invalid value for extensions file: %w", err) } @@ -93,7 +94,7 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { if err != nil { return nil, nil, fmt.Errorf("invalid value for status-codes: %w", err) } - ret2, err := helper.ParseCommaSeparatedInt(pluginOpts.StatusCodes) + ret2, err := libgobuster.ParseCommaSeparatedInt(pluginOpts.StatusCodes) if err != nil { return nil, nil, fmt.Errorf("invalid value for status-codes: %w", err) } @@ -104,7 +105,7 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { if err != nil { return nil, nil, fmt.Errorf("invalid value for status-codes-blacklist: %w", err) } - ret3, err := helper.ParseCommaSeparatedInt(pluginOpts.StatusCodesBlacklist) + ret3, err := libgobuster.ParseCommaSeparatedInt(pluginOpts.StatusCodesBlacklist) if err != nil { return nil, nil, fmt.Errorf("invalid value for status-codes-blacklist: %w", err) } @@ -144,10 +145,15 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { return nil, nil, fmt.Errorf("invalid value for discover-backup: %w", err) } - pluginOpts.ExcludeLength, err = cmdDir.Flags().GetIntSlice("exclude-length") + pluginOpts.ExcludeLength, err = cmdDir.Flags().GetString("exclude-length") if err != nil { - return nil, nil, fmt.Errorf("invalid value for excludelength: %w", err) + return nil, nil, fmt.Errorf("invalid value for exclude-length: %w", err) } + ret4, err := libgobuster.ParseCommaSeparatedInt(pluginOpts.ExcludeLength) + if err != nil { + return nil, nil, fmt.Errorf("invalid value for exclude-length: %w", err) + } + pluginOpts.ExcludeLengthParsed = ret4 return globalopts, pluginOpts, nil } @@ -172,7 +178,7 @@ func init() { cmdDir.Flags().Bool("hide-length", false, "Hide the length of the body in the output") cmdDir.Flags().BoolP("add-slash", "f", false, "Append / to each request") cmdDir.Flags().BoolP("discover-backup", "d", false, "Also search for backup files by appending multiple backup extensions") - cmdDir.Flags().IntSlice("exclude-length", []int{}, "exclude the following content length (completely ignores the status). Supply multiple times to exclude multiple sizes.") + cmdDir.Flags().String("exclude-length", "", "exclude the following content lengths (completely ignores the status). You can separate multiple lengths by comma and it also supports ranges like 203-206") cmdDir.PersistentPreRun = func(cmd *cobra.Command, args []string) { configureGlobalOptions() diff --git a/cli/cmd/dir_test.go b/cli/cmd/dir_test.go index 51c1f0d1..568fd487 100644 --- a/cli/cmd/dir_test.go +++ b/cli/cmd/dir_test.go @@ -3,8 +3,6 @@ package cmd import ( "context" "fmt" - "io" - "log" "net/http" "net/http/httptest" "os" @@ -13,7 +11,6 @@ import ( "github.com/OJ/gobuster/v3/cli" "github.com/OJ/gobuster/v3/gobusterdir" - "github.com/OJ/gobuster/v3/helper" "github.com/OJ/gobuster/v3/libgobuster" ) @@ -33,14 +30,14 @@ func BenchmarkDirMode(b *testing.B) { pluginopts.Timeout = 10 * time.Second pluginopts.Extensions = ".php,.csv" - tmpExt, err := helper.ParseExtensions(pluginopts.Extensions) + tmpExt, err := libgobuster.ParseExtensions(pluginopts.Extensions) if err != nil { b.Fatalf("could not parse extensions: %v", err) } pluginopts.ExtensionsParsed = tmpExt pluginopts.StatusCodes = "200,204,301,302,307,401,403" - tmpStat, err := helper.ParseCommaSeparatedInt(pluginopts.StatusCodes) + tmpStat, err := libgobuster.ParseCommaSeparatedInt(pluginopts.StatusCodes) if err != nil { b.Fatalf("could not parse status codes: %v", err) } @@ -71,8 +68,7 @@ func BenchmarkDirMode(b *testing.B) { b.Fatalf("could not get devnull %v", err) } defer devnull.Close() - log.SetFlags(0) - log.SetOutput(io.Discard) + log := libgobuster.NewLogger(false) // Run the real benchmark for x := 0; x < b.N; x++ { @@ -83,7 +79,7 @@ func BenchmarkDirMode(b *testing.B) { b.Fatalf("error on creating gobusterdir: %v", err) } - if err := cli.Gobuster(ctx, &globalopts, plugin); err != nil { + if err := cli.Gobuster(ctx, &globalopts, plugin, log); err != nil { b.Fatalf("error on running gobuster: %v", err) } os.Stdout = oldStdout diff --git a/cli/cmd/dns.go b/cli/cmd/dns.go index 9116db9e..c37fb0c1 100644 --- a/cli/cmd/dns.go +++ b/cli/cmd/dns.go @@ -27,11 +27,13 @@ func runDNS(cmd *cobra.Command, args []string) error { return fmt.Errorf("error on creating gobusterdns: %w", err) } - if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { + log := libgobuster.NewLogger(globalopts.Debug) + if err := cli.Gobuster(mainContext, globalopts, plugin, log); err != nil { var wErr *gobusterdns.ErrWildcard if errors.As(err, &wErr) { return fmt.Errorf("%w. To force processing of Wildcard DNS, specify the '--wildcard' switch", wErr) } + log.Debugf("%#v", err) return fmt.Errorf("error on running gobuster: %w", err) } return nil @@ -74,6 +76,11 @@ func parseDNSOptions() (*libgobuster.Options, *gobusterdns.OptionsDNS, error) { return nil, nil, fmt.Errorf("invalid value for resolver: %w", err) } + pluginOpts.NoFQDN, err = cmdDNS.Flags().GetBool("no-fqdn") + if err != nil { + return nil, nil, fmt.Errorf("invalid value for no-fqdn: %w", err) + } + if pluginOpts.Resolver != "" && runtime.GOOS == "windows" { return nil, nil, fmt.Errorf("currently can not set custom dns resolver on windows. See https://golang.org/pkg/net/#hdr-Name_Resolution") } @@ -94,6 +101,7 @@ func init() { cmdDNS.Flags().BoolP("show-cname", "c", false, "Show CNAME records (cannot be used with '-i' option)") cmdDNS.Flags().DurationP("timeout", "", time.Second, "DNS resolver timeout") cmdDNS.Flags().BoolP("wildcard", "", false, "Force continued operation when wildcard found") + cmdDNS.Flags().BoolP("no-fqdn", "", false, "Do not automatically add a trailing dot to the domain, so the resolver uses the DNS search domain") cmdDNS.Flags().StringP("resolver", "r", "", "Use custom DNS server (format server.com or server.com:port)") if err := cmdDNS.MarkFlagRequired("domain"); err != nil { log.Fatalf("error on marking flag as required: %v", err) diff --git a/cli/cmd/fuzz.go b/cli/cmd/fuzz.go index ae0a841b..b763b7d5 100644 --- a/cli/cmd/fuzz.go +++ b/cli/cmd/fuzz.go @@ -8,7 +8,6 @@ import ( "github.com/OJ/gobuster/v3/cli" "github.com/OJ/gobuster/v3/gobusterfuzz" - "github.com/OJ/gobuster/v3/helper" "github.com/OJ/gobuster/v3/libgobuster" "github.com/spf13/cobra" ) @@ -31,11 +30,13 @@ func runFuzz(cmd *cobra.Command, args []string) error { return fmt.Errorf("error on creating gobusterfuzz: %w", err) } - if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { + log := libgobuster.NewLogger(globalopts.Debug) + if err := cli.Gobuster(mainContext, globalopts, plugin, log); err != nil { var wErr *gobusterfuzz.ErrWildcard if errors.As(err, &wErr) { return fmt.Errorf("%w. To continue please exclude the status code or the length", wErr) } + log.Debugf("%#v", err) return fmt.Errorf("error on running gobuster: %w", err) } return nil @@ -74,16 +75,21 @@ func parseFuzzOptions() (*libgobuster.Options, *gobusterfuzz.OptionsFuzz, error) if err != nil { return nil, nil, fmt.Errorf("invalid value for excludestatuscodes: %w", err) } - ret, err := helper.ParseCommaSeparatedInt(pluginOpts.ExcludedStatusCodes) + ret, err := libgobuster.ParseCommaSeparatedInt(pluginOpts.ExcludedStatusCodes) if err != nil { return nil, nil, fmt.Errorf("invalid value for excludestatuscodes: %w", err) } pluginOpts.ExcludedStatusCodesParsed = ret - pluginOpts.ExcludeLength, err = cmdFuzz.Flags().GetIntSlice("exclude-length") + pluginOpts.ExcludeLength, err = cmdFuzz.Flags().GetString("exclude-length") if err != nil { - return nil, nil, fmt.Errorf("invalid value for excludelength: %w", err) + return nil, nil, fmt.Errorf("invalid value for exclude-length: %w", err) } + ret2, err := libgobuster.ParseCommaSeparatedInt(pluginOpts.ExcludeLength) + if err != nil { + return nil, nil, fmt.Errorf("invalid value for exclude-length: %w", err) + } + pluginOpts.ExcludeLengthParsed = ret2 pluginOpts.RequestBody, err = cmdFuzz.Flags().GetString("body") if err != nil { @@ -105,7 +111,7 @@ func init() { log.Fatalf("%v", err) } cmdFuzz.Flags().StringP("excludestatuscodes", "b", "", "Excluded status codes. Can also handle ranges like 200,300-400,404.") - cmdFuzz.Flags().IntSlice("exclude-length", []int{}, "exclude the following content length (completely ignores the status). Supply multiple times to exclude multiple sizes.") + cmdFuzz.Flags().String("exclude-length", "", "exclude the following content lengths (completely ignores the status). You can separate multiple lengths by comma and it also supports ranges like 203-206") cmdFuzz.Flags().StringP("body", "B", "", "Request body") cmdFuzz.PersistentPreRun = func(cmd *cobra.Command, args []string) { diff --git a/cli/cmd/gcs.go b/cli/cmd/gcs.go index 1b7512e3..bdec7394 100644 --- a/cli/cmd/gcs.go +++ b/cli/cmd/gcs.go @@ -23,7 +23,9 @@ func runGCS(cmd *cobra.Command, args []string) error { return fmt.Errorf("error on creating gobustergcs: %w", err) } - if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { + log := libgobuster.NewLogger(globalopts.Debug) + if err := cli.Gobuster(mainContext, globalopts, plugin, log); err != nil { + log.Debugf("%#v", err) return fmt.Errorf("error on running gobuster: %w", err) } return nil diff --git a/cli/cmd/http.go b/cli/cmd/http.go index 4f8c1f85..c0747fe0 100644 --- a/cli/cmd/http.go +++ b/cli/cmd/http.go @@ -11,7 +11,6 @@ import ( "syscall" "time" - "github.com/OJ/gobuster/v3/helper" "github.com/OJ/gobuster/v3/libgobuster" "github.com/spf13/cobra" "golang.org/x/crypto/pkcs12" @@ -21,7 +20,7 @@ import ( func addBasicHTTPOptions(cmd *cobra.Command) { cmd.Flags().StringP("useragent", "a", libgobuster.DefaultUserAgent(), "Set the User-Agent string") cmd.Flags().BoolP("random-agent", "", false, "Use a random User-Agent string") - cmd.Flags().StringP("proxy", "", "", "Proxy to use for requests [http(s)://host:port]") + cmd.Flags().StringP("proxy", "", "", "Proxy to use for requests [http(s)://host:port] or [socks5://host:port]") cmd.Flags().DurationP("timeout", "", 10*time.Second, "HTTP Timeout") cmd.Flags().BoolP("no-tls-validation", "k", false, "Skip TLS certificate verification") cmd.Flags().BoolP("retry", "", false, "Should retry on request timeout") @@ -64,7 +63,7 @@ func parseBasicHTTPOptions(cmd *cobra.Command) (libgobuster.BasicHTTPOptions, er return options, fmt.Errorf("invalid value for random-agent: %w", err) } if randomUA { - ua, err := helper.GetRandomUserAgent() + ua, err := libgobuster.GetRandomUserAgent() if err != nil { return options, err } diff --git a/cli/cmd/root.go b/cli/cmd/root.go index ef95ecb8..cf27b782 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -88,6 +88,20 @@ func parseGlobalOptions() (*libgobuster.Options, error) { return nil, fmt.Errorf("wordlist file %q does not exist: %w", globalopts.Wordlist, err2) } + offset, err := rootCmd.Flags().GetInt("wordlist-offset") + if err != nil { + return nil, fmt.Errorf("invalid value for wordlist-offset: %w", err) + } + + if offset < 0 { + return nil, fmt.Errorf("wordlist-offset must be bigger or equal to 0") + } + globalopts.WordlistOffset = offset + + if globalopts.Wordlist == "-" && globalopts.WordlistOffset > 0 { + return nil, fmt.Errorf("wordlist-offset is not supported when reading from STDIN") + } + globalopts.PatternFile, err = rootCmd.Flags().GetString("pattern") if err != nil { return nil, fmt.Errorf("invalid value for pattern: %w", err) @@ -145,6 +159,11 @@ func parseGlobalOptions() (*libgobuster.Options, error) { color.NoColor = true } + globalopts.Debug, err = rootCmd.Flags().GetBool("debug") + if err != nil { + return nil, fmt.Errorf("invalid value for debug: %w", err) + } + return globalopts, nil } @@ -162,7 +181,8 @@ func configureGlobalOptions() { func init() { rootCmd.PersistentFlags().DurationP("delay", "", 0, "Time each thread waits between requests (e.g. 1500ms)") rootCmd.PersistentFlags().IntP("threads", "t", 10, "Number of concurrent threads") - rootCmd.PersistentFlags().StringP("wordlist", "w", "", "Path to the wordlist") + rootCmd.PersistentFlags().StringP("wordlist", "w", "", "Path to the wordlist. Set to - to use STDIN.") + rootCmd.PersistentFlags().IntP("wordlist-offset", "", 0, "Resume from a given position in the wordlist (defaults to 0)") rootCmd.PersistentFlags().StringP("output", "o", "", "Output file to write results to (defaults to stdout)") rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Verbose output (errors)") rootCmd.PersistentFlags().BoolP("quiet", "q", false, "Don't print the banner and other noise") @@ -170,4 +190,5 @@ func init() { rootCmd.PersistentFlags().Bool("no-error", false, "Don't display errors") rootCmd.PersistentFlags().StringP("pattern", "p", "", "File containing replacement patterns") rootCmd.PersistentFlags().Bool("no-color", false, "Disable color output") + rootCmd.PersistentFlags().Bool("debug", false, "Enable debug output") } diff --git a/cli/cmd/s3.go b/cli/cmd/s3.go index c8aaccc4..9823d50b 100644 --- a/cli/cmd/s3.go +++ b/cli/cmd/s3.go @@ -23,7 +23,9 @@ func runS3(cmd *cobra.Command, args []string) error { return fmt.Errorf("error on creating gobusters3: %w", err) } - if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { + log := libgobuster.NewLogger(globalopts.Debug) + if err := cli.Gobuster(mainContext, globalopts, plugin, log); err != nil { + log.Debugf("%#v", err) return fmt.Errorf("error on running gobuster: %w", err) } return nil diff --git a/cli/cmd/tftp.go b/cli/cmd/tftp.go index a7ddfed3..7180abe8 100644 --- a/cli/cmd/tftp.go +++ b/cli/cmd/tftp.go @@ -26,7 +26,9 @@ func runTFTP(cmd *cobra.Command, args []string) error { return fmt.Errorf("error on creating gobustertftp: %w", err) } - if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { + log := libgobuster.NewLogger(globalopts.Debug) + if err := cli.Gobuster(mainContext, globalopts, plugin, log); err != nil { + log.Debugf("%#v", err) return fmt.Errorf("error on running gobuster: %w", err) } return nil diff --git a/cli/cmd/vhost.go b/cli/cmd/vhost.go index bc7be05b..c707750b 100644 --- a/cli/cmd/vhost.go +++ b/cli/cmd/vhost.go @@ -24,7 +24,9 @@ func runVhost(cmd *cobra.Command, args []string) error { return fmt.Errorf("error on creating gobustervhost: %w", err) } - if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { + log := libgobuster.NewLogger(globalopts.Debug) + if err := cli.Gobuster(mainContext, globalopts, plugin, log); err != nil { + log.Debugf("%#v", err) return fmt.Errorf("error on running gobuster: %w", err) } return nil @@ -63,10 +65,15 @@ func parseVhostOptions() (*libgobuster.Options, *gobustervhost.OptionsVhost, err return nil, nil, fmt.Errorf("invalid value for append-domain: %w", err) } - pluginOpts.ExcludeLength, err = cmdVhost.Flags().GetIntSlice("exclude-length") + pluginOpts.ExcludeLength, err = cmdVhost.Flags().GetString("exclude-length") if err != nil { - return nil, nil, fmt.Errorf("invalid value for excludelength: %w", err) + return nil, nil, fmt.Errorf("invalid value for exclude-length: %w", err) } + ret, err := libgobuster.ParseCommaSeparatedInt(pluginOpts.ExcludeLength) + if err != nil { + return nil, nil, fmt.Errorf("invalid value for exclude-length: %w", err) + } + pluginOpts.ExcludeLengthParsed = ret pluginOpts.Domain, err = cmdVhost.Flags().GetString("domain") if err != nil { @@ -87,7 +94,7 @@ func init() { log.Fatalf("%v", err) } cmdVhost.Flags().BoolP("append-domain", "", false, "Append main domain from URL to words from wordlist. Otherwise the fully qualified domains need to be specified in the wordlist.") - cmdVhost.Flags().IntSlice("exclude-length", []int{}, "exclude the following content length (completely ignores the status). Supply multiple times to exclude multiple sizes.") + cmdVhost.Flags().String("exclude-length", "", "exclude the following content lengths (completely ignores the status). You can separate multiple lengths by comma and it also supports ranges like 203-206") cmdVhost.Flags().String("domain", "", "the domain to append when using an IP address as URL. If left empty and you specify a domain based URL the hostname from the URL is extracted") cmdVhost.PersistentPreRun = func(cmd *cobra.Command, args []string) { diff --git a/cli/cmd/vhost_test.go b/cli/cmd/vhost_test.go index 9f6a2ea5..722c8730 100644 --- a/cli/cmd/vhost_test.go +++ b/cli/cmd/vhost_test.go @@ -3,8 +3,6 @@ package cmd import ( "context" "fmt" - "io" - "log" "os" "testing" "time" @@ -47,8 +45,7 @@ func BenchmarkVhostMode(b *testing.B) { b.Fatalf("could not get devnull %v", err) } defer devnull.Close() - log.SetFlags(0) - log.SetOutput(io.Discard) + log := libgobuster.NewLogger(false) // Run the real benchmark for x := 0; x < b.N; x++ { @@ -59,7 +56,7 @@ func BenchmarkVhostMode(b *testing.B) { b.Fatalf("error on creating gobusterdir: %v", err) } - if err := cli.Gobuster(ctx, &globalopts, plugin); err != nil { + if err := cli.Gobuster(ctx, &globalopts, plugin, log); err != nil { b.Fatalf("error on running gobuster: %v", err) } os.Stdout = oldStdout diff --git a/cli/gobuster.go b/cli/gobuster.go index e04a4d2d..e32febab 100644 --- a/cli/gobuster.go +++ b/cli/gobuster.go @@ -14,11 +14,6 @@ import ( const ruler = "===============================================================" const cliProgressUpdate = 500 * time.Millisecond -func banner() { - fmt.Printf("Gobuster v%s\n", libgobuster.VERSION) - fmt.Println("by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart)") -} - // resultWorker outputs the results as they come in. This needs to be a range and should not handle // the context so the channel always has a receiver and libgobuster will not block. func resultWorker(g *libgobuster.Gobuster, filename string, wg *sync.WaitGroup) { @@ -29,7 +24,7 @@ func resultWorker(g *libgobuster.Gobuster, filename string, wg *sync.WaitGroup) if filename != "" { f, err = os.Create(filename) if err != nil { - g.LogError.Fatalf("error on creating output file: %v", err) + g.Logger.Fatalf("error on creating output file: %v", err) } defer f.Close() } @@ -37,7 +32,7 @@ func resultWorker(g *libgobuster.Gobuster, filename string, wg *sync.WaitGroup) for r := range g.Progress.ResultChan { s, err := r.ResultToString() if err != nil { - g.LogError.Fatal(err) + g.Logger.Fatal(err) } if s != "" { s = strings.TrimSpace(s) @@ -45,7 +40,7 @@ func resultWorker(g *libgobuster.Gobuster, filename string, wg *sync.WaitGroup) if f != nil { err = writeToFile(f, s) if err != nil { - g.LogError.Fatalf("error on writing output file: %v", err) + g.Logger.Fatalf("error on writing output file: %v", err) } } } @@ -59,7 +54,44 @@ func errorWorker(g *libgobuster.Gobuster, wg *sync.WaitGroup) { for e := range g.Progress.ErrorChan { if !g.Opts.Quiet && !g.Opts.NoError { - g.LogError.Printf("[!] %s\n", e.Error()) + g.Logger.Error(e.Error()) + g.Logger.Debugf("%#v", e) + } + } +} + +// messageWorker outputs messages as they come in. This needs to be a range and should not handle +// the context so the channel always has a receiver and libgobuster will not block. +func messageWorker(g *libgobuster.Gobuster, wg *sync.WaitGroup) { + defer wg.Done() + + for msg := range g.Progress.MessageChan { + if !g.Opts.Quiet { + switch msg.Level { + case libgobuster.LevelDebug: + g.Logger.Debug(msg.Message) + case libgobuster.LevelError: + g.Logger.Error(msg.Message) + case libgobuster.LevelInfo: + g.Logger.Info(msg.Message) + default: + panic(fmt.Sprintf("invalid level %d", msg.Level)) + } + } + } +} + +func printProgress(g *libgobuster.Gobuster) { + if !g.Opts.Quiet && !g.Opts.NoProgress { + requestsIssued := g.Progress.RequestsIssued() + requestsExpected := g.Progress.RequestsExpected() + if g.Opts.Wordlist == "-" { + s := fmt.Sprintf("%sProgress: %d", TERMINAL_CLEAR_LINE, requestsIssued) + _, _ = fmt.Fprint(os.Stderr, s) + // only print status if we already read in the wordlist + } else if requestsExpected > 0 { + s := fmt.Sprintf("%sProgress: %d / %d (%3.2f%%)", TERMINAL_CLEAR_LINE, requestsIssued, requestsExpected, float32(requestsIssued)*100.0/float32(requestsExpected)) + _, _ = fmt.Fprint(os.Stderr, s) } } } @@ -74,19 +106,10 @@ func progressWorker(ctx context.Context, g *libgobuster.Gobuster, wg *sync.WaitG for { select { case <-tick.C: - if !g.Opts.Quiet && !g.Opts.NoProgress { - requestsIssued := g.Progress.RequestsIssued() - requestsExpected := g.Progress.RequestsExpected() - if g.Opts.Wordlist == "-" { - s := fmt.Sprintf("%sProgress: %d", TERMINAL_CLEAR_LINE, requestsIssued) - _, _ = fmt.Fprint(os.Stderr, s) - // only print status if we already read in the wordlist - } else if requestsExpected > 0 { - s := fmt.Sprintf("%sProgress: %d / %d (%3.2f%%)", TERMINAL_CLEAR_LINE, requestsIssued, requestsExpected, float32(requestsIssued)*100.0/float32(requestsExpected)) - _, _ = fmt.Fprint(os.Stderr, s) - } - } + printProgress(g) case <-ctx.Done(): + // print the final progress so we end at 100% + printProgress(g) fmt.Println() return } @@ -102,7 +125,7 @@ func writeToFile(f *os.File, output string) error { } // Gobuster is the main entry point for the CLI -func Gobuster(ctx context.Context, opts *libgobuster.Options, plugin libgobuster.GobusterPlugin) error { +func Gobuster(ctx context.Context, opts *libgobuster.Options, plugin libgobuster.GobusterPlugin, log libgobuster.Logger) error { // Sanity checks if opts == nil { return fmt.Errorf("please provide valid options") @@ -115,23 +138,27 @@ func Gobuster(ctx context.Context, opts *libgobuster.Options, plugin libgobuster ctxCancel, cancel := context.WithCancel(ctx) defer cancel() - gobuster, err := libgobuster.NewGobuster(opts, plugin) + gobuster, err := libgobuster.NewGobuster(opts, plugin, log) if err != nil { return err } if !opts.Quiet { - fmt.Println(ruler) - banner() - fmt.Println(ruler) + log.Println(ruler) + log.Printf("Gobuster v%s\n", libgobuster.VERSION) + log.Println("by OJ Reeves (@TheColonial) & Christian Mehlmauer (@firefart)") + log.Println(ruler) c, err := gobuster.GetConfigString() if err != nil { return fmt.Errorf("error on creating config string: %w", err) } - fmt.Println(c) - fmt.Println(ruler) - gobuster.LogInfo.Printf("Starting gobuster in %s mode", plugin.Name()) - fmt.Println(ruler) + log.Println(c) + log.Println(ruler) + gobuster.Logger.Printf("Starting gobuster in %s mode", plugin.Name()) + if opts.WordlistOffset > 0 { + gobuster.Logger.Printf("Skipping the first %d elements...", opts.WordlistOffset) + } + log.Println(ruler) } // our waitgroup for all goroutines @@ -145,6 +172,9 @@ func Gobuster(ctx context.Context, opts *libgobuster.Options, plugin libgobuster wg.Add(1) go errorWorker(gobuster, &wg) + wg.Add(1) + go messageWorker(gobuster, &wg) + if !opts.Quiet && !opts.NoProgress { // if not quiet add a new workgroup entry and start the goroutine wg.Add(1) @@ -165,9 +195,9 @@ func Gobuster(ctx context.Context, opts *libgobuster.Options, plugin libgobuster } if !opts.Quiet { - fmt.Println(ruler) - gobuster.LogInfo.Println("Finished") - fmt.Println(ruler) + log.Println(ruler) + gobuster.Logger.Println("Finished") + log.Println(ruler) } return nil } diff --git a/go.mod b/go.mod index 1e3e5904..a155e1d9 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,19 @@ module github.com/OJ/gobuster/v3 go 1.19 require ( - github.com/fatih/color v1.14.1 + github.com/fatih/color v1.15.0 github.com/google/uuid v1.3.0 github.com/pin/tftp/v3 v3.0.0 - github.com/spf13/cobra v1.6.1 - golang.org/x/crypto v0.6.0 - golang.org/x/term v0.5.0 + github.com/spf13/cobra v1.7.0 + golang.org/x/crypto v0.12.0 + golang.org/x/term v0.11.0 ) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.11.0 // indirect ) diff --git a/go.sum b/go.sum index c5b360e3..e21bd206 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,35 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pin/tftp/v3 v3.0.0 h1:o9cQpmWBSbgiaYXuN+qJAB12XBIv4dT7OuOONucn2l0= github.com/pin/tftp/v3 v3.0.0/go.mod h1:xwQaN4viYL019tM4i8iecm++5cGxSqen6AJEOEyEI0w= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/gobusterdir/gobusterdir.go b/gobusterdir/gobusterdir.go index 6599990b..5138d6c7 100644 --- a/gobusterdir/gobusterdir.go +++ b/gobusterdir/gobusterdir.go @@ -9,8 +9,8 @@ import ( "net/http" "strings" "text/tabwriter" + "unicode/utf8" - "github.com/OJ/gobuster/v3/helper" "github.com/OJ/gobuster/v3/libgobuster" "github.com/google/uuid" ) @@ -91,7 +91,7 @@ func (d *GobusterDir) Name() string { } // PreRun is the pre run implementation of gobusterdir -func (d *GobusterDir) PreRun(ctx context.Context) error { +func (d *GobusterDir) PreRun(ctx context.Context, progress *libgobuster.Progress) error { // add trailing slash if !strings.HasSuffix(d.options.URL, "/") { d.options.URL = fmt.Sprintf("%s/", d.options.URL) @@ -113,7 +113,7 @@ func (d *GobusterDir) PreRun(ctx context.Context) error { return err } - if helper.SliceContains(d.options.ExcludeLength, int(wildcardLength)) { + if d.options.ExcludeLengthParsed.Contains(int(wildcardLength)) { // we are done and ignore the request as the length is excluded return nil } @@ -174,6 +174,17 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li suffix = "/" } entity := fmt.Sprintf("%s%s", word, suffix) + + // make sure the url ends with a slash + if !strings.HasSuffix(d.options.URL, "/") { + d.options.URL = fmt.Sprintf("%s/", d.options.URL) + } + // prevent double slashes by removing leading / + if strings.HasPrefix(entity, "/") { + // get size of first rune and trim it + _, i := utf8.DecodeRuneInString(entity) + entity = entity[i:] + } url := fmt.Sprintf("%s%s", d.options.URL, entity) tries := 1 @@ -220,7 +231,7 @@ func (d *GobusterDir) ProcessWord(ctx context.Context, word string, progress *li return fmt.Errorf("StatusCodes and StatusCodesBlacklist are both not set which should not happen") } - if (resultStatus && !helper.SliceContains(d.options.ExcludeLength, int(size))) || d.globalopts.Verbose { + if (resultStatus && !d.options.ExcludeLengthParsed.Contains(int(size))) || d.globalopts.Verbose { progress.ResultChan <- Result{ URL: d.options.URL, Path: entity, @@ -288,7 +299,7 @@ func (d *GobusterDir) GetConfigString() (string, error) { } if len(o.ExcludeLength) > 0 { - if _, err := fmt.Fprintf(tw, "[+] Exclude Length:\t%s\n", helper.JoinIntSlice(d.options.ExcludeLength)); err != nil { + if _, err := fmt.Fprintf(tw, "[+] Exclude Length:\t%s\n", d.options.ExcludeLengthParsed.Stringify()); err != nil { return "", err } } diff --git a/gobusterdir/options.go b/gobusterdir/options.go index b0d700c9..c1309b8b 100644 --- a/gobusterdir/options.go +++ b/gobusterdir/options.go @@ -19,7 +19,8 @@ type OptionsDir struct { Expanded bool NoStatus bool DiscoverBackup bool - ExcludeLength []int + ExcludeLength string + ExcludeLengthParsed libgobuster.Set[int] } // NewOptionsDir returns a new initialized OptionsDir @@ -28,5 +29,6 @@ func NewOptionsDir() *OptionsDir { StatusCodesParsed: libgobuster.NewSet[int](), StatusCodesBlacklistParsed: libgobuster.NewSet[int](), ExtensionsParsed: libgobuster.NewSet[string](), + ExcludeLengthParsed: libgobuster.NewSet[int](), } } diff --git a/gobusterdns/gobusterdns.go b/gobusterdns/gobusterdns.go index 3d9e6229..abb6302c 100644 --- a/gobusterdns/gobusterdns.go +++ b/gobusterdns/gobusterdns.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "fmt" - "log" "net" "net/netip" "strings" @@ -78,7 +77,7 @@ func (d *GobusterDNS) Name() string { } // PreRun is the pre run implementation of gobusterdns -func (d *GobusterDNS) PreRun(ctx context.Context) error { +func (d *GobusterDNS) PreRun(ctx context.Context, progress *libgobuster.Progress) error { // Resolve a subdomain that probably shouldn't exist guid := uuid.New() wildcardIps, err := d.dnsLookup(ctx, fmt.Sprintf("%s.%s", guid, d.options.Domain)) @@ -95,7 +94,14 @@ func (d *GobusterDNS) PreRun(ctx context.Context) error { _, err = d.dnsLookup(ctx, d.options.Domain) if err != nil { // Not an error, just a warning. Eg. `yp.to` doesn't resolve, but `cr.yp.to` does! - log.Printf("[-] Unable to validate base domain: %s (%v)", d.options.Domain, err) + progress.MessageChan <- libgobuster.Message{ + Level: libgobuster.LevelInfo, + Message: fmt.Sprintf("[-] Unable to validate base domain: %s (%v)", d.options.Domain, err), + } + progress.MessageChan <- libgobuster.Message{ + Level: libgobuster.LevelDebug, + Message: fmt.Sprintf("%#v", err), + } } } @@ -105,6 +111,10 @@ func (d *GobusterDNS) PreRun(ctx context.Context) error { // ProcessWord is the process implementation of gobusterdns func (d *GobusterDNS) ProcessWord(ctx context.Context, word string, progress *libgobuster.Progress) error { subdomain := fmt.Sprintf("%s.%s", word, d.options.Domain) + if !d.options.NoFQDN && !strings.HasSuffix(subdomain, ".") { + // add a . to indicate this is the full domain and we do not want to traverse the search domains on the system + subdomain = fmt.Sprintf("%s.", subdomain) + } ips, err := d.dnsLookup(ctx, subdomain) if err == nil { if !d.isWildcard || !d.wildcardIps.ContainsAny(ips) { @@ -113,6 +123,7 @@ func (d *GobusterDNS) ProcessWord(ctx context.Context, word string, progress *li Found: true, ShowIPs: d.options.ShowIPs, ShowCNAME: d.options.ShowCNAME, + NoFQDN: d.options.NoFQDN, } if d.options.ShowIPs { result.IPs = ips diff --git a/gobusterdns/options.go b/gobusterdns/options.go index b5b276e4..1c6cfd1d 100644 --- a/gobusterdns/options.go +++ b/gobusterdns/options.go @@ -11,6 +11,7 @@ type OptionsDNS struct { ShowCNAME bool WildcardForced bool Resolver string + NoFQDN bool Timeout time.Duration } diff --git a/gobusterdns/result.go b/gobusterdns/result.go index b93501d6..b55a5ed4 100644 --- a/gobusterdns/result.go +++ b/gobusterdns/result.go @@ -19,6 +19,7 @@ type Result struct { ShowCNAME bool Found bool Subdomain string + NoFQDN bool IPs []netip.Addr CNAME string } @@ -29,6 +30,9 @@ func (r Result) ResultToString() (string, error) { c := green + if !r.NoFQDN { + r.Subdomain = strings.TrimSuffix(r.Subdomain, ".") + } if r.Found { c(buf, "Found: ") } else { diff --git a/gobusterfuzz/gobusterfuzz.go b/gobusterfuzz/gobusterfuzz.go index 10825255..5dd39316 100644 --- a/gobusterfuzz/gobusterfuzz.go +++ b/gobusterfuzz/gobusterfuzz.go @@ -9,7 +9,6 @@ import ( "strings" "text/tabwriter" - "github.com/OJ/gobuster/v3/helper" "github.com/OJ/gobuster/v3/libgobuster" ) @@ -83,7 +82,7 @@ func (d *GobusterFuzz) Name() string { } // PreRun is the pre run implementation of gobusterfuzz -func (d *GobusterFuzz) PreRun(ctx context.Context) error { +func (d *GobusterFuzz) PreRun(ctx context.Context, progress *libgobuster.Progress) error { return nil } @@ -146,7 +145,7 @@ func (d *GobusterFuzz) ProcessWord(ctx context.Context, word string, progress *l if statusCode != 0 { resultStatus := true - if helper.SliceContains(d.options.ExcludeLength, int(size)) { + if d.options.ExcludeLengthParsed.Contains(int(size)) { resultStatus = false } @@ -163,6 +162,7 @@ func (d *GobusterFuzz) ProcessWord(ctx context.Context, word string, progress *l Path: url, StatusCode: statusCode, Size: size, + Word: word, } } } @@ -218,7 +218,7 @@ func (d *GobusterFuzz) GetConfigString() (string, error) { } if len(o.ExcludeLength) > 0 { - if _, err := fmt.Fprintf(tw, "[+] Exclude Length:\t%s\n", helper.JoinIntSlice(d.options.ExcludeLength)); err != nil { + if _, err := fmt.Fprintf(tw, "[+] Exclude Length:\t%s\n", d.options.ExcludeLengthParsed.Stringify()); err != nil { return "", err } } diff --git a/gobusterfuzz/options.go b/gobusterfuzz/options.go index de91b603..f5327cac 100644 --- a/gobusterfuzz/options.go +++ b/gobusterfuzz/options.go @@ -9,13 +9,15 @@ type OptionsFuzz struct { libgobuster.HTTPOptions ExcludedStatusCodes string ExcludedStatusCodesParsed libgobuster.Set[int] - ExcludeLength []int - RequestBody string + ExcludeLength string + ExcludeLengthParsed libgobuster.Set[int] + RequestBody string } // NewOptionsFuzz returns a new initialized OptionsFuzz func NewOptionsFuzz() *OptionsFuzz { return &OptionsFuzz{ ExcludedStatusCodesParsed: libgobuster.NewSet[int](), + ExcludeLengthParsed: libgobuster.NewSet[int](), } } diff --git a/gobusterfuzz/result.go b/gobusterfuzz/result.go index b4e48f8f..00932550 100644 --- a/gobusterfuzz/result.go +++ b/gobusterfuzz/result.go @@ -13,6 +13,7 @@ var ( // Result represents a single result type Result struct { + Word string Verbose bool Found bool Path string @@ -38,7 +39,7 @@ func (r Result) ResultToString() (string, error) { c(buf, "Found: ") } - c(buf, "[Status=%d] [Length=%d] %s", r.StatusCode, r.Size, r.Path) + c(buf, "[Status=%d] [Length=%d] [Word=%s] %s", r.StatusCode, r.Size, r.Word, r.Path) c(buf, "\n") s := buf.String() diff --git a/gobustergcs/gobustersgcs.go b/gobustergcs/gobustersgcs.go index 87056221..50b0f687 100644 --- a/gobustergcs/gobustersgcs.go +++ b/gobustergcs/gobustersgcs.go @@ -71,7 +71,7 @@ func (s *GobusterGCS) Name() string { } // PreRun is the pre run implementation of GobusterS3 -func (s *GobusterGCS) PreRun(ctx context.Context) error { +func (s *GobusterGCS) PreRun(ctx context.Context, progress *libgobuster.Progress) error { return nil } diff --git a/gobusters3/gobusters3.go b/gobusters3/gobusters3.go index 4c371818..5a183528 100644 --- a/gobusters3/gobusters3.go +++ b/gobusters3/gobusters3.go @@ -70,7 +70,7 @@ func (s *GobusterS3) Name() string { } // PreRun is the pre run implementation of GobusterS3 -func (s *GobusterS3) PreRun(ctx context.Context) error { +func (s *GobusterS3) PreRun(ctx context.Context, progress *libgobuster.Progress) error { return nil } diff --git a/gobustertftp/gobustertftp.go b/gobustertftp/gobustertftp.go index 3598391a..19f20573 100644 --- a/gobustertftp/gobustertftp.go +++ b/gobustertftp/gobustertftp.go @@ -42,7 +42,7 @@ func (d *GobusterTFTP) Name() string { } // PreRun is the pre run implementation of gobustertftp -func (d *GobusterTFTP) PreRun(ctx context.Context) error { +func (d *GobusterTFTP) PreRun(ctx context.Context, progress *libgobuster.Progress) error { _, err := tftp.NewClient(d.options.Server) if err != nil { return err diff --git a/gobustervhost/gobustervhost.go b/gobustervhost/gobustervhost.go index 166ba9fd..d3d8059b 100644 --- a/gobustervhost/gobustervhost.go +++ b/gobustervhost/gobustervhost.go @@ -11,7 +11,6 @@ import ( "strings" "text/tabwriter" - "github.com/OJ/gobuster/v3/helper" "github.com/OJ/gobuster/v3/libgobuster" "github.com/google/uuid" ) @@ -76,7 +75,7 @@ func (v *GobusterVhost) Name() string { } // PreRun is the pre run implementation of gobusterdir -func (v *GobusterVhost) PreRun(ctx context.Context) error { +func (v *GobusterVhost) PreRun(ctx context.Context, progress *libgobuster.Progress) error { // add trailing slash if !strings.HasSuffix(v.options.URL, "/") { v.options.URL = fmt.Sprintf("%s/", v.options.URL) @@ -152,7 +151,7 @@ func (v *GobusterVhost) ProcessWord(ctx context.Context, word string, progress * // subdomain must not match default vhost and non existent vhost // or verbose mode is enabled found := body != nil && !bytes.Equal(body, v.normalBody) && !bytes.Equal(body, v.abnormalBody) - if (found && !helper.SliceContains(v.options.ExcludeLength, int(size))) || v.globalopts.Verbose { + if (found && !v.options.ExcludeLengthParsed.Contains(int(size))) || v.globalopts.Verbose { resultStatus := false if found { resultStatus = true @@ -249,7 +248,7 @@ func (v *GobusterVhost) GetConfigString() (string, error) { } if len(o.ExcludeLength) > 0 { - if _, err := fmt.Fprintf(tw, "[+] Exclude Length:\t%s\n", helper.JoinIntSlice(v.options.ExcludeLength)); err != nil { + if _, err := fmt.Fprintf(tw, "[+] Exclude Length:\t%s\n", v.options.ExcludeLengthParsed.Stringify()); err != nil { return "", err } } diff --git a/gobustervhost/options.go b/gobustervhost/options.go index 50db56e5..a5090308 100644 --- a/gobustervhost/options.go +++ b/gobustervhost/options.go @@ -7,12 +7,15 @@ import ( // OptionsVhost is the struct to hold all options for this plugin type OptionsVhost struct { libgobuster.HTTPOptions - AppendDomain bool - ExcludeLength []int - Domain string + AppendDomain bool + ExcludeLength string + ExcludeLengthParsed libgobuster.Set[int] + Domain string } // NewOptionsVhost returns a new initialized OptionsVhost func NewOptionsVhost() *OptionsVhost { - return &OptionsVhost{} + return &OptionsVhost{ + ExcludeLengthParsed: libgobuster.NewSet[int](), + } } diff --git a/helper/helper.go b/helper/helper.go deleted file mode 100644 index 28f68b2e..00000000 --- a/helper/helper.go +++ /dev/null @@ -1,117 +0,0 @@ -package helper - -import ( - "bufio" - "fmt" - "os" - "regexp" - "strconv" - "strings" - - "github.com/OJ/gobuster/v3/libgobuster" -) - -// ParseExtensions parses the extensions provided as a comma separated list -func ParseExtensions(extensions string) (libgobuster.Set[string], error) { - ret := libgobuster.NewSet[string]() - - if extensions == "" { - return ret, nil - } - - for _, e := range strings.Split(extensions, ",") { - e = strings.TrimSpace(e) - // remove leading . from extensions - ret.Add(strings.TrimPrefix(e, ".")) - } - return ret, nil -} - -func ParseExtensionsFile(file string) ([]string, error) { - var ret []string - - stream, err := os.Open(file) - if err != nil { - return ret, err - } - defer stream.Close() - - scanner := bufio.NewScanner(stream) - for scanner.Scan() { - e := scanner.Text() - e = strings.TrimSpace(e) - // remove leading . from extensions - ret = append(ret, (strings.TrimPrefix(e, "."))) - } - - if err := scanner.Err(); err != nil { - return nil, err - } - - return ret, nil -} - -// ParseCommaSeparatedInt parses the status codes provided as a comma separated list -func ParseCommaSeparatedInt(inputString string) (libgobuster.Set[int], error) { - ret := libgobuster.NewSet[int]() - - if inputString == "" { - return ret, nil - } - - for _, part := range strings.Split(inputString, ",") { - part = strings.TrimSpace(part) - // check for range - if strings.Contains(part, "-") { - re := regexp.MustCompile(`^\s*(\d+)\s*-\s*(\d+)\s*$`) - match := re.FindStringSubmatch(part) - if match == nil || len(match) != 3 { - return libgobuster.NewSet[int](), fmt.Errorf("invalid range given: %s", part) - } - from := strings.TrimSpace(match[1]) - to := strings.TrimSpace(match[2]) - fromI, err := strconv.Atoi(from) - if err != nil { - return libgobuster.NewSet[int](), fmt.Errorf("invalid string in range %s: %s", part, from) - } - toI, err := strconv.Atoi(to) - if err != nil { - return libgobuster.NewSet[int](), fmt.Errorf("invalid string in range %s: %s", part, to) - } - if toI < fromI { - return libgobuster.NewSet[int](), fmt.Errorf("invalid range given: %s", part) - } - for i := fromI; i <= toI; i++ { - ret.Add(i) - } - } else { - i, err := strconv.Atoi(part) - if err != nil { - return libgobuster.NewSet[int](), fmt.Errorf("invalid string given: %s", part) - } - ret.Add(i) - } - } - return ret, nil -} - -// SliceContains checks if an integer slice contains a specific value -func SliceContains(s []int, e int) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - -// JoinIntSlice joins an int slice by , -func JoinIntSlice(s []int) string { - valuesText := make([]string, len(s)) - for i, number := range s { - text := strconv.Itoa(number) - valuesText[i] = text - } - result := strings.Join(valuesText, ",") - return result -} diff --git a/helper/helper_test.go b/helper/helper_test.go deleted file mode 100644 index 783e47d0..00000000 --- a/helper/helper_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package helper - -import ( - "reflect" - "testing" - - "github.com/OJ/gobuster/v3/libgobuster" -) - -func TestParseExtensions(t *testing.T) { - t.Parallel() - var tt = []struct { - testName string - extensions string - expectedExtensions libgobuster.Set[string] - expectedError string - }{ - {"Valid extensions", "php,asp,txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Spaces", "php, asp , txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Double extensions", "php,asp,txt,php,asp,txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Leading dot", ".php,asp,.txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Empty string", "", libgobuster.NewSet[string](), "invalid extension string provided"}, - } - - for _, x := range tt { - x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables - t.Run(x.testName, func(t *testing.T) { - t.Parallel() - ret, err := ParseExtensions(x.extensions) - if x.expectedError != "" { - if err != nil && err.Error() != x.expectedError { - t.Fatalf("Expected error %q but got %q", x.expectedError, err.Error()) - } - } else if !reflect.DeepEqual(x.expectedExtensions, ret) { - t.Fatalf("Expected %v but got %v", x.expectedExtensions, ret) - } - }) - } -} - -func TestParseCommaSeparatedInt(t *testing.T) { - t.Parallel() - var tt = []struct { - stringCodes string - expectedCodes []int - expectedError string - }{ - {"200,100,202", []int{200, 100, 202}, ""}, - {"200, 100 , 202", []int{200, 100, 202}, ""}, - {"200, 100, 202, 100", []int{200, 100, 202}, ""}, - {"200,AAA", []int{}, "invalid string given: AAA"}, - {"2000000000000000000000000000000", []int{}, "invalid string given: 2000000000000000000000000000000"}, - {"", []int{}, "invalid string provided"}, - {"200-205", []int{200, 201, 202, 203, 204, 205}, ""}, - {"200-202,203-205", []int{200, 201, 202, 203, 204, 205}, ""}, - {"200-202,204-205", []int{200, 201, 202, 204, 205}, ""}, - {"200-202,205", []int{200, 201, 202, 205}, ""}, - {"205,200,100-101,103-105", []int{100, 101, 103, 104, 105, 200, 205}, ""}, - {"200-200", []int{200}, ""}, - {"200 - 202", []int{200, 201, 202}, ""}, - {"200 -202", []int{200, 201, 202}, ""}, - {"200- 202", []int{200, 201, 202}, ""}, - {"200 - 202", []int{200, 201, 202}, ""}, - {"230-200", []int{}, "invalid range given: 230-200"}, - {"A-200", []int{}, "invalid range given: A-200"}, - {"230-A", []int{}, "invalid range given: 230-A"}, - {"200,202-205,A,206-210", []int{}, "invalid string given: A"}, - {"200,202-205,A-1,206-210", []int{}, "invalid range given: A-1"}, - {"200,202-205,1-A,206-210", []int{}, "invalid range given: 1-A"}, - } - - for _, x := range tt { - x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables - t.Run(x.stringCodes, func(t *testing.T) { - t.Parallel() - want := libgobuster.NewSet[int]() - want.AddRange(x.expectedCodes) - ret, err := ParseCommaSeparatedInt(x.stringCodes) - if x.expectedError != "" { - if err != nil && err.Error() != x.expectedError { - t.Fatalf("Expected error %q but got %q", x.expectedError, err.Error()) - } - } else if !reflect.DeepEqual(want, ret) { - t.Fatalf("Expected %v but got %v", want, ret) - } - }) - } -} - -func BenchmarkParseExtensions(b *testing.B) { - var tt = []struct { - testName string - extensions string - expectedExtensions libgobuster.Set[string] - expectedError string - }{ - {"Valid extensions", "php,asp,txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Spaces", "php, asp , txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Double extensions", "php,asp,txt,php,asp,txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Leading dot", ".php,asp,.txt", libgobuster.Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, - {"Empty string", "", libgobuster.NewSet[string](), "invalid extension string provided"}, - } - - for _, x := range tt { - x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables - b.Run(x.testName, func(b2 *testing.B) { - for y := 0; y < b2.N; y++ { - _, _ = ParseExtensions(x.extensions) - } - }) - } -} - -func BenchmarkParseCommaSeparatedInt(b *testing.B) { - var tt = []struct { - testName string - stringCodes string - expectedCodes libgobuster.Set[int] - expectedError string - }{ - {"Valid codes", "200,100,202", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Spaces", "200, 100 , 202", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Double codes", "200, 100, 202, 100", libgobuster.Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, - {"Invalid code", "200,AAA", libgobuster.NewSet[int](), "invalid string given: AAA"}, - {"Invalid integer", "2000000000000000000000000000000", libgobuster.NewSet[int](), "invalid string given: 2000000000000000000000000000000"}, - {"Empty string", "", libgobuster.NewSet[int](), "invalid string string provided"}, - } - - for _, x := range tt { - x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables - b.Run(x.testName, func(b2 *testing.B) { - for y := 0; y < b2.N; y++ { - _, _ = ParseCommaSeparatedInt(x.stringCodes) - } - }) - } -} diff --git a/libgobuster/helpers.go b/libgobuster/helpers.go index fbc054ac..6df5c8ee 100644 --- a/libgobuster/helpers.go +++ b/libgobuster/helpers.go @@ -1,10 +1,14 @@ package libgobuster import ( + "bufio" "bytes" "errors" "fmt" "io" + "os" + "regexp" + "strconv" "strings" ) @@ -87,3 +91,108 @@ func lineCounter(r io.Reader) (int, error) { func DefaultUserAgent() string { return fmt.Sprintf("gobuster/%s", VERSION) } + +// ParseExtensions parses the extensions provided as a comma separated list +func ParseExtensions(extensions string) (Set[string], error) { + ret := NewSet[string]() + + if extensions == "" { + return ret, nil + } + + for _, e := range strings.Split(extensions, ",") { + e = strings.TrimSpace(e) + // remove leading . from extensions + ret.Add(strings.TrimPrefix(e, ".")) + } + return ret, nil +} + +func ParseExtensionsFile(file string) ([]string, error) { + var ret []string + + stream, err := os.Open(file) + if err != nil { + return ret, err + } + defer stream.Close() + + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + e := scanner.Text() + e = strings.TrimSpace(e) + // remove leading . from extensions + ret = append(ret, (strings.TrimPrefix(e, "."))) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return ret, nil +} + +// ParseCommaSeparatedInt parses the status codes provided as a comma separated list +func ParseCommaSeparatedInt(inputString string) (Set[int], error) { + ret := NewSet[int]() + + if inputString == "" { + return ret, nil + } + + for _, part := range strings.Split(inputString, ",") { + part = strings.TrimSpace(part) + // check for range + if strings.Contains(part, "-") { + re := regexp.MustCompile(`^\s*(\d+)\s*-\s*(\d+)\s*$`) + match := re.FindStringSubmatch(part) + if match == nil || len(match) != 3 { + return NewSet[int](), fmt.Errorf("invalid range given: %s", part) + } + from := strings.TrimSpace(match[1]) + to := strings.TrimSpace(match[2]) + fromI, err := strconv.Atoi(from) + if err != nil { + return NewSet[int](), fmt.Errorf("invalid string in range %s: %s", part, from) + } + toI, err := strconv.Atoi(to) + if err != nil { + return NewSet[int](), fmt.Errorf("invalid string in range %s: %s", part, to) + } + if toI < fromI { + return NewSet[int](), fmt.Errorf("invalid range given: %s", part) + } + for i := fromI; i <= toI; i++ { + ret.Add(i) + } + } else { + i, err := strconv.Atoi(part) + if err != nil { + return NewSet[int](), fmt.Errorf("invalid string given: %s", part) + } + ret.Add(i) + } + } + return ret, nil +} + +// SliceContains checks if an integer slice contains a specific value +func SliceContains(s []int, e int) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +// JoinIntSlice joins an int slice by , +func JoinIntSlice(s []int) string { + valuesText := make([]string, len(s)) + for i, number := range s { + text := strconv.Itoa(number) + valuesText[i] = text + } + result := strings.Join(valuesText, ",") + return result +} diff --git a/libgobuster/helpers_test.go b/libgobuster/helpers_test.go index 900543b1..496898d7 100644 --- a/libgobuster/helpers_test.go +++ b/libgobuster/helpers_test.go @@ -3,6 +3,7 @@ package libgobuster import ( "errors" "fmt" + "reflect" "strings" "testing" "testing/iotest" @@ -191,3 +192,132 @@ func TestLineCounterError(t *testing.T) { t.Fatalf("Got wrong error! %v", err) } } + +func TestParseExtensions(t *testing.T) { + t.Parallel() + var tt = []struct { + testName string + extensions string + expectedExtensions Set[string] + expectedError string + }{ + {"Valid extensions", "php,asp,txt", Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Spaces", "php, asp , txt", Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Double extensions", "php,asp,txt,php,asp,txt", Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Leading dot", ".php,asp,.txt", Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Empty string", "", NewSet[string](), "invalid extension string provided"}, + } + + for _, x := range tt { + x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables + t.Run(x.testName, func(t *testing.T) { + t.Parallel() + ret, err := ParseExtensions(x.extensions) + if x.expectedError != "" { + if err != nil && err.Error() != x.expectedError { + t.Fatalf("Expected error %q but got %q", x.expectedError, err.Error()) + } + } else if !reflect.DeepEqual(x.expectedExtensions, ret) { + t.Fatalf("Expected %v but got %v", x.expectedExtensions, ret) + } + }) + } +} + +func TestParseCommaSeparatedInt(t *testing.T) { + t.Parallel() + var tt = []struct { + stringCodes string + expectedCodes []int + expectedError string + }{ + {"200,100,202", []int{200, 100, 202}, ""}, + {"200, 100 , 202", []int{200, 100, 202}, ""}, + {"200, 100, 202, 100", []int{200, 100, 202}, ""}, + {"200,AAA", []int{}, "invalid string given: AAA"}, + {"2000000000000000000000000000000", []int{}, "invalid string given: 2000000000000000000000000000000"}, + {"", []int{}, "invalid string provided"}, + {"200-205", []int{200, 201, 202, 203, 204, 205}, ""}, + {"200-202,203-205", []int{200, 201, 202, 203, 204, 205}, ""}, + {"200-202,204-205", []int{200, 201, 202, 204, 205}, ""}, + {"200-202,205", []int{200, 201, 202, 205}, ""}, + {"205,200,100-101,103-105", []int{100, 101, 103, 104, 105, 200, 205}, ""}, + {"200-200", []int{200}, ""}, + {"200 - 202", []int{200, 201, 202}, ""}, + {"200 -202", []int{200, 201, 202}, ""}, + {"200- 202", []int{200, 201, 202}, ""}, + {"200 - 202", []int{200, 201, 202}, ""}, + {"230-200", []int{}, "invalid range given: 230-200"}, + {"A-200", []int{}, "invalid range given: A-200"}, + {"230-A", []int{}, "invalid range given: 230-A"}, + {"200,202-205,A,206-210", []int{}, "invalid string given: A"}, + {"200,202-205,A-1,206-210", []int{}, "invalid range given: A-1"}, + {"200,202-205,1-A,206-210", []int{}, "invalid range given: 1-A"}, + } + + for _, x := range tt { + x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables + t.Run(x.stringCodes, func(t *testing.T) { + t.Parallel() + want := NewSet[int]() + want.AddRange(x.expectedCodes) + ret, err := ParseCommaSeparatedInt(x.stringCodes) + if x.expectedError != "" { + if err != nil && err.Error() != x.expectedError { + t.Fatalf("Expected error %q but got %q", x.expectedError, err.Error()) + } + } else if !reflect.DeepEqual(want, ret) { + t.Fatalf("Expected %v but got %v", want, ret) + } + }) + } +} + +func BenchmarkParseExtensions(b *testing.B) { + var tt = []struct { + testName string + extensions string + expectedExtensions Set[string] + expectedError string + }{ + {"Valid extensions", "php,asp,txt", Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Spaces", "php, asp , txt", Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Double extensions", "php,asp,txt,php,asp,txt", Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Leading dot", ".php,asp,.txt", Set[string]{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Empty string", "", NewSet[string](), "invalid extension string provided"}, + } + + for _, x := range tt { + x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables + b.Run(x.testName, func(b2 *testing.B) { + for y := 0; y < b2.N; y++ { + _, _ = ParseExtensions(x.extensions) + } + }) + } +} + +func BenchmarkParseCommaSeparatedInt(b *testing.B) { + var tt = []struct { + testName string + stringCodes string + expectedCodes Set[int] + expectedError string + }{ + {"Valid codes", "200,100,202", Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Spaces", "200, 100 , 202", Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Double codes", "200, 100, 202, 100", Set[int]{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Invalid code", "200,AAA", NewSet[int](), "invalid string given: AAA"}, + {"Invalid integer", "2000000000000000000000000000000", NewSet[int](), "invalid string given: 2000000000000000000000000000000"}, + {"Empty string", "", NewSet[int](), "invalid string string provided"}, + } + + for _, x := range tt { + x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables + b.Run(x.testName, func(b2 *testing.B) { + for y := 0; y < b2.N; y++ { + _, _ = ParseCommaSeparatedInt(x.stringCodes) + } + }) + } +} diff --git a/libgobuster/interfaces.go b/libgobuster/interfaces.go index 28d7e2b8..4a39dd7a 100644 --- a/libgobuster/interfaces.go +++ b/libgobuster/interfaces.go @@ -5,7 +5,7 @@ import "context" // GobusterPlugin is an interface which plugins must implement type GobusterPlugin interface { Name() string - PreRun(context.Context) error + PreRun(context.Context, *Progress) error ProcessWord(context.Context, string, *Progress) error AdditionalWords(string) []string GetConfigString() (string, error) diff --git a/libgobuster/libgobuster.go b/libgobuster/libgobuster.go index e188ebc9..d9105890 100644 --- a/libgobuster/libgobuster.go +++ b/libgobuster/libgobuster.go @@ -4,13 +4,10 @@ import ( "bufio" "context" "fmt" - "log" "os" "strings" "sync" "time" - - "github.com/fatih/color" ) // PATTERN is the pattern for wordlist replacements in pattern file @@ -28,19 +25,17 @@ type ResultToStringFunc func(*Gobuster, *Result) (*string, error) // Gobuster is the main object when creating a new run type Gobuster struct { Opts *Options + Logger Logger plugin GobusterPlugin - LogInfo *log.Logger - LogError *log.Logger Progress *Progress } // NewGobuster returns a new Gobuster object -func NewGobuster(opts *Options, plugin GobusterPlugin) (*Gobuster, error) { +func NewGobuster(opts *Options, plugin GobusterPlugin, logger Logger) (*Gobuster, error) { var g Gobuster g.Opts = opts g.plugin = plugin - g.LogInfo = log.New(os.Stdout, "", log.LstdFlags) - g.LogError = log.New(os.Stderr, color.New(color.FgRed).Sprint("[ERROR] "), log.LstdFlags) + g.Logger = logger g.Progress = NewProgress() return &g, nil @@ -97,9 +92,16 @@ func (g *Gobuster) getWordlist() (*bufio.Scanner, error) { return nil, fmt.Errorf("failed to get number of lines: %w", err) } + if lines-g.Opts.WordlistOffset <= 0 { + return nil, fmt.Errorf("offset is greater than the number of lines in the wordlist") + } + // calcutate expected requests g.Progress.IncrementTotalRequests(lines) + // add offset if needed (offset defaults to 0) + g.Progress.incrementRequestsIssues(g.Opts.WordlistOffset) + // call the function once with a dummy entry to receive the number // of custom words per wordlist word customWordsLen := len(g.plugin.AdditionalWords("dummy")) @@ -114,7 +116,20 @@ func (g *Gobuster) getWordlist() (*bufio.Scanner, error) { if err != nil { return nil, fmt.Errorf("failed to rewind wordlist: %w", err) } - return bufio.NewScanner(wordlist), nil + + wordlistScanner := bufio.NewScanner(wordlist) + + // skip lines + for i := 0; i < g.Opts.WordlistOffset; i++ { + if !wordlistScanner.Scan() { + if err := wordlistScanner.Err(); err != nil { + return nil, fmt.Errorf("failed to skip lines in wordlist: %w", err) + } + return nil, fmt.Errorf("failed to skip lines in wordlist") + } + } + + return wordlistScanner, nil } // Run the busting of the website with the given @@ -122,8 +137,9 @@ func (g *Gobuster) getWordlist() (*bufio.Scanner, error) { func (g *Gobuster) Run(ctx context.Context) error { defer close(g.Progress.ResultChan) defer close(g.Progress.ErrorChan) + defer close(g.Progress.MessageChan) - if err := g.plugin.PreRun(ctx); err != nil { + if err := g.plugin.PreRun(ctx, g.Progress); err != nil { return err } diff --git a/libgobuster/logger.go b/libgobuster/logger.go new file mode 100644 index 00000000..41c00110 --- /dev/null +++ b/libgobuster/logger.go @@ -0,0 +1,80 @@ +package libgobuster + +import ( + "log" + "os" + + "github.com/fatih/color" +) + +type Logger struct { + log *log.Logger + errorLog *log.Logger + debugLog *log.Logger + infoLog *log.Logger + debug bool +} + +func NewLogger(debug bool) Logger { + return Logger{ + log: log.New(os.Stdout, "", 0), + errorLog: log.New(os.Stderr, color.New(color.FgRed).Sprint("[ERROR] "), 0), + debugLog: log.New(os.Stderr, color.New(color.FgBlue).Sprint("[DEBUG] "), 0), + infoLog: log.New(os.Stderr, color.New(color.FgCyan).Sprint("[INFO] "), 0), + debug: debug, + } +} + +func (l Logger) Debug(v ...any) { + if !l.debug { + return + } + l.debugLog.Print(v...) +} + +func (l Logger) Debugf(format string, v ...any) { + if !l.debug { + return + } + l.debugLog.Printf(format, v...) +} + +func (l Logger) Info(v ...any) { + l.infoLog.Print(v...) +} + +func (l Logger) Infof(format string, v ...any) { + l.infoLog.Printf(format, v...) +} + +func (l Logger) Print(v ...any) { + l.log.Print(v...) +} + +func (l Logger) Printf(format string, v ...any) { + l.log.Printf(format, v...) +} + +func (l Logger) Println(v ...any) { + l.log.Println(v...) +} + +func (l Logger) Error(v ...any) { + l.errorLog.Print(v...) +} + +func (l Logger) Errorf(format string, v ...any) { + l.errorLog.Printf(format, v...) +} + +func (l Logger) Fatal(v ...any) { + l.errorLog.Fatal(v...) +} + +func (l Logger) Fatalf(format string, v ...any) { + l.errorLog.Fatalf(format, v...) +} + +func (l Logger) Fatalln(v ...any) { + l.errorLog.Fatalln(v...) +} diff --git a/libgobuster/options.go b/libgobuster/options.go index 377dc8e2..156d9663 100644 --- a/libgobuster/options.go +++ b/libgobuster/options.go @@ -5,7 +5,9 @@ import "time" // Options holds all options that can be passed to libgobuster type Options struct { Threads int + Debug bool Wordlist string + WordlistOffset int PatternFile string Patterns []string OutputFilename string diff --git a/libgobuster/progress.go b/libgobuster/progress.go index 64a3182d..f37bb963 100644 --- a/libgobuster/progress.go +++ b/libgobuster/progress.go @@ -2,6 +2,19 @@ package libgobuster import "sync" +type MessageLevel int + +const ( + LevelDebug MessageLevel = iota + LevelInfo + LevelError +) + +type Message struct { + Level MessageLevel + Message string +} + type Progress struct { requestsExpectedMutex *sync.RWMutex requestsExpected int @@ -9,6 +22,7 @@ type Progress struct { requestsIssued int ResultChan chan Result ErrorChan chan error + MessageChan chan Message } func NewProgress() *Progress { @@ -18,6 +32,7 @@ func NewProgress() *Progress { p.requestsCountMutex = new(sync.RWMutex) p.ResultChan = make(chan Result) p.ErrorChan = make(chan error) + p.MessageChan = make(chan Message) return &p } @@ -33,6 +48,12 @@ func (p *Progress) RequestsIssued() int { return p.requestsIssued } +func (p *Progress) incrementRequestsIssues(by int) { + p.requestsCountMutex.Lock() + defer p.requestsCountMutex.Unlock() + p.requestsIssued += by +} + func (p *Progress) incrementRequests() { p.requestsCountMutex.Lock() defer p.requestsCountMutex.Unlock() diff --git a/helper/useragents.go b/libgobuster/useragents.go similarity index 99% rename from helper/useragents.go rename to libgobuster/useragents.go index 5d9f1dd3..fbbe4738 100644 --- a/helper/useragents.go +++ b/libgobuster/useragents.go @@ -1,4 +1,4 @@ -package helper +package libgobuster import ( "crypto/rand" diff --git a/libgobuster/version.go b/libgobuster/version.go index 6d03d6fe..4a253cd7 100644 --- a/libgobuster/version.go +++ b/libgobuster/version.go @@ -2,5 +2,5 @@ package libgobuster const ( // VERSION contains the current gobuster version - VERSION = "3.5" + VERSION = "3.6" )