From 585c8adb857a952c4e8f20eb3175251c31596686 Mon Sep 17 00:00:00 2001 From: Jon Hadfield Date: Sat, 4 May 2024 16:13:39 +0100 Subject: [PATCH] initial. --- .github/dependabot.yml | 11 + .github/workflows/codeql-analysis.yml | 38 + .github/workflows/golangci-lint.yml | 24 + .gitignore | 1 + .goreleaser.yml | 148 ++++ Dockerfile | 40 + LICENSE | 21 + Makefile | 89 ++ README.md | 32 + cache/cache.go | 158 ++++ cache/cache_test.go | 159 ++++ cmd/cache.go | 109 +++ cmd/root.go | 282 +++++++ cmd/root_test.go | 1 + cmd/util.go | 13 + cmd/version.go | 21 + go.mod | 74 ++ go.sum | 268 ++++++ main.go | 9 + manager/manager.go | 207 +++++ out | 1 + present/present.go | 106 +++ process/process.go | 401 +++++++++ process/process_test.go | 1 + providers/abuseipdb/abuseipdb.go | 366 ++++++++ providers/annotated/annotated.go | 495 +++++++++++ providers/annotated/annotated_test.go | 105 +++ providers/annotated/testdata/small/small.yml | 21 + providers/aws/aws.go | 395 +++++++++ .../aws/testdata/aws_18_164_52_75_report.json | 16 + providers/azure/azure.go | 359 ++++++++ .../testdata/azure_40_126_12_192_report.json | 1 + providers/criminalip/criminalip.go | 792 ++++++++++++++++++ .../testdata/criminalip_1_1_1_1_report.json | 264 ++++++ .../testdata/criminalip_9_9_9_9_report.json | 606 ++++++++++++++ providers/digitalocean/digitalocean.go | 346 ++++++++ .../digitalocean_165_232_46_239_report.json | 13 + providers/gcp/gcp.go | 370 ++++++++ .../gcp/testdata/gcp_34_128_62_0_report.json | 8 + providers/icloudpr/icloudpr.go | 351 ++++++++ .../icloudpr_172_224_224_60_report.json | 10 + providers/ipurl/ipurl.go | 390 +++++++++ .../testdata/ipurl_5_105_62_0_report.json | 1 + providers/linode/linode.go | 350 ++++++++ .../testdata/linode_69_164_198_1_report.json | 10 + providers/providers.go | 276 ++++++ providers/providers_test.go | 232 +++++ providers/ptr/ptr.go | 338 ++++++++ .../ptr/testdata/ptr_8_8_8_8_report.json | 34 + providers/shodan/shodan.go | 663 +++++++++++++++ providers/shodan/shodan_test.go | 153 ++++ .../testdata/shodan_google_dns_resp.json | 1 + providers/whois/whois.go | 1 + session/config.yaml | 48 ++ session/session.go | 244 ++++++ session/session_test.go | 126 +++ 56 files changed, 9599 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/golangci-lint.yml create mode 100644 .gitignore create mode 100644 .goreleaser.yml create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 cache/cache.go create mode 100644 cache/cache_test.go create mode 100644 cmd/cache.go create mode 100644 cmd/root.go create mode 100644 cmd/root_test.go create mode 100644 cmd/util.go create mode 100644 cmd/version.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 manager/manager.go create mode 100644 out create mode 100644 present/present.go create mode 100644 process/process.go create mode 100644 process/process_test.go create mode 100644 providers/abuseipdb/abuseipdb.go create mode 100644 providers/annotated/annotated.go create mode 100644 providers/annotated/annotated_test.go create mode 100644 providers/annotated/testdata/small/small.yml create mode 100644 providers/aws/aws.go create mode 100644 providers/aws/testdata/aws_18_164_52_75_report.json create mode 100644 providers/azure/azure.go create mode 100644 providers/azure/testdata/azure_40_126_12_192_report.json create mode 100644 providers/criminalip/criminalip.go create mode 100644 providers/criminalip/testdata/criminalip_1_1_1_1_report.json create mode 100644 providers/criminalip/testdata/criminalip_9_9_9_9_report.json create mode 100644 providers/digitalocean/digitalocean.go create mode 100644 providers/digitalocean/testdata/digitalocean_165_232_46_239_report.json create mode 100644 providers/gcp/gcp.go create mode 100644 providers/gcp/testdata/gcp_34_128_62_0_report.json create mode 100644 providers/icloudpr/icloudpr.go create mode 100644 providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json create mode 100644 providers/ipurl/ipurl.go create mode 100644 providers/ipurl/testdata/ipurl_5_105_62_0_report.json create mode 100644 providers/linode/linode.go create mode 100644 providers/linode/testdata/linode_69_164_198_1_report.json create mode 100644 providers/providers.go create mode 100644 providers/providers_test.go create mode 100644 providers/ptr/ptr.go create mode 100644 providers/ptr/testdata/ptr_8_8_8_8_report.json create mode 100644 providers/shodan/shodan.go create mode 100644 providers/shodan/shodan_test.go create mode 100644 providers/shodan/testdata/shodan_google_dns_resp.json create mode 100644 providers/whois/whois.go create mode 100644 session/config.yaml create mode 100644 session/session.go create mode 100644 session/session_test.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a0da861 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..d6b8d52 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,38 @@ +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '25 23 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..a3d4c8f --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,24 @@ +name: golangci-lint +on: + push: + tags: + - '*' + branches: + - main + pull_request: +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + cache: false + - uses: actions/checkout@v4 + - name: golangci-lint + uses: golangci/golangci-lint-action@v5 + with: + version: latest + args: -v --disable lll --disable interfacer --disable gochecknoglobals diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..849ddff --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +dist/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..411f747 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,148 @@ +project_name: ipscout + +env: + - GO111MODULE=on + - GOPROXY=https://proxy.golang.org + - CGO_ENABLED=0 + +before: + hooks: + - make clean + - go mod tidy +builds: + - + id: macos-ipscout + binary: ipscout + goos: + - darwin + goarch: + - amd64 + - arm64 + flags: + - -trimpath + ldflags: + - "-s -w -X main.version={{ .Version }} -X main.sha={{ .ShortCommit }} -X main.buildDate={{ .Date }} -X main.tag={{ .Tag }}" + hooks: + post: + - | + sh -c ' + cat > /tmp/ipscout_gon_arm64.hcl << EOF + source = ["./dist/macos-ipscout_darwin_arm64/ipscout"] + bundle_id = "uk.co.lessknown.ipscout" + apple_id { + username = "jon@lessknown.co.uk" + provider = "VBZY8FBYR5" + } + sign { + application_identity = "Developer ID Application: Jonathan Hadfield (VBZY8FBYR5)" + } + zip { + output_path = "./dist/ipscout_darwin_arm64.zip" + } + EOF + gon -log-level=info -log-json /tmp/ipscout_gon_arm64.hcl + echo $? + ' + echo $? + - | + sh -c ' + cat > /tmp/ipscout_gon_amd64.hcl << EOF + source = ["./dist/macos-ipscout_darwin_amd64_v1/ipscout"] + bundle_id = "uk.co.lessknown.ipscout" + apple_id { + username = "jon@lessknown.co.uk" + provider = "VBZY8FBYR5" + } + sign { + application_identity = "Developer ID Application: Jonathan Hadfield (VBZY8FBYR5)" + } + zip { + output_path = "./dist/ipscout_darwin_amd64_v1.zip" + } + EOF + echo $? + gon -log-level=info -log-json /tmp/ipscout_gon_amd64.hcl + echo $? + ' + - + id: ipscout + binary: ipscout + env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - freebsd + goarch: + - amd64 + - arm + - arm64 + flags: + - -trimpath + ldflags: + - "-s -w -X main.version={{ .Version }} -X main.sha={{ .ShortCommit }} -X main.buildDate={{ .Date }} -X main.tag={{ .Tag }}" + +brews: + - + name: ipscout + homepage: 'https://github.com/jonhadfield/ipscout' + description: 'A command line tool useful for network administrators and security analysts to quickly identify the origin and threat of an IP address.' + repository: + owner: jonhadfield + name: homebrew-ipscout + +archives: + - name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}" + builds: + - macos-ipscout + - ipscout + format: tar.gz + format_overrides: + - goos: windows + format: zip + files: + - none* + +release: + github: + owner: jonhadfield + name: ipscout + prerelease: auto + name_template: '{{ .Tag }}' + extra_files: + - glob: ./dist/ipscout_darwin*.zip + +announce: + skip: true + +snapshot: + name_template: "{{ .Tag }}-devel" + +changelog: + sort: asc + filters: + exclude: + - README + - test + - ignore + +checksum: + name_template: 'checksums.txt' +# +#universal_binaries: +# - +# id: ipscout +# replace: true + +#notarize: +# macos: +# - enabled: '{{ isEnvSet "MACOS_SIGN_P12" }}' +# ids: +# - ipscout +# sign: +# certificate: "{{.Env.MACOS_SIGN_P12}}" +# password: "{{.Env.MACOS_SIGN_PASSWORD}}" +# notarize: +# issuer_id: "{{.Env.MACOS_NOTARY_ISSUER_ID}}" +# key_id: "{{.Env.MACOS_NOTARY_KEY_ID}}" +# key: "{{.Env.MACOS_NOTARY_KEY}}" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f93529e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +FROM --platform=linux/amd64 golang:1.22-bookworm AS base +ARG BUILD_SHA +ARG BUILD_TAG +ARG BUILD_DATE +WORKDIR /src + +COPY ./ . + +RUN apt-get update && \ + apt-get install -y git coreutils && \ + apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV GOPROXY=https://proxy.golang.org +RUN --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go mod download + +FROM base AS builder +ARG BUILD_SHA +ARG BUILD_TAG +ARG BUILD_DATE +ENV CGO_ENABLED=0 + +RUN mkdir /app +COPY ./ /app/ +WORKDIR /app +RUN echo "Building version: [$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" +RUN --mount=target=. \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X \"github.com/jonhadfield/ipscout/cmd.version=[${BUILD_TAG}-${BUILD_SHA}] ${BUILD_DATE} UTC\" -X \"github.com/jonhadfield/ipscout/cmd.semver=${BUILD_TAG}\"" -o /out/ipscout \ + && chmod +x /out/ipscout + +FROM --platform=linux/amd64 gcr.io/distroless/static-debian12:nonroot-amd64 +LABEL maintainer="Jon Hadfield jon@lessknown.co.uk" + +WORKDIR /app +COPY --from=builder /out/ipscout /app/ipscout + +ENTRYPOINT ["/app/ipscout"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f144312 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jon Hadfield + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..608ef6f --- /dev/null +++ b/Makefile @@ -0,0 +1,89 @@ +SOURCE_FILES?=$$(go list ./...) +TEST_PATTERN?=. +TEST_OPTIONS?=-race -v + +setup: + go get -u github.com/go-critic/go-critic/... + go get -u github.com/alecthomas/gometalinter + go get -u golang.org/x/tools/cmd/cover + gometalinter --install + +clean: + rm -rf ./dist + +# This requires credentials are set for all providers!!! +test: + echo 'mode: atomic' > coverage.txt && go list ./... | xargs -n1 -I{} sh -c 'go test -v -timeout=600s -covermode=atomic -coverprofile=coverage.tmp {} && tail -n +2 coverage.tmp >> coverage.txt' && rm coverage.tmp + +cover: test + go tool cover -html=coverage.txt + +fmt: + goimports -w . && gofumpt -l -w . + +lint: + golangci-lint run --disable lll --disable interfacer --disable gochecknoglobals --disable gochecknoinits --enable wsl --enable revive --enable gosec --enable unused --enable gocritic --enable gofmt --enable goimports --enable misspell --enable unparam --enable goconst --enable wrapcheck +ci: lint test + +BUILD_TAG := $(shell git describe --tags 2>/dev/null) +BUILD_SHA := $(shell git rev-parse --short HEAD) +BUILD_DATE := $(shell date -u '+%Y/%m/%d:%H:%M:%S') +LATEST_TAG := $(shell git describe --abbrev=0 2>/dev/null) + +build: + CGO_ENABLED=0 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout" + +build-all: fmt + GOOS=darwin CGO_ENABLED=0 GOARCH=amd64 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_darwin_amd64" + GOOS=darwin CGO_ENABLED=0 GOARCH=arm64 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_darwin_arm64" + GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_linux_amd64" + GOOS=linux CGO_ENABLED=0 GOARCH=386 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_linux_386" + GOOS=linux CGO_ENABLED=0 GOARCH=arm go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_linux_arm" + GOOS=linux CGO_ENABLED=0 GOARCH=arm64 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_linux_arm64" + GOOS=netbsd CGO_ENABLED=0 GOARCH=amd64 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_netbsd_amd64" + GOOS=openbsd CGO_ENABLED=0 GOARCH=amd64 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_openbsd_amd64" + GOOS=freebsd CGO_ENABLED=0 GOARCH=amd64 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_freebsd_amd64" + GOOS=windows CGO_ENABLED=0 GOARCH=amd64 go build -ldflags '-s -w -X "github.com/jonhadfield/ipscout/cmd.version=[$(BUILD_TAG)-$(BUILD_SHA)] $(BUILD_DATE) UTC" -X "github.com/jonhadfield/ipscout/cmd.semver=$(BUILD_TAG)"' -o ".local_dist/ipscout_windows_amd64.exe" + +critic: + gocritic check ./... + +mac-install: build + install .local_dist/ipscout /usr/local/bin/ipscout + +linux-install: build + sudo install .local_dist/ipscout /usr/local/bin/ipscout + +install: build + go install ./... + +find-updates: + go list -u -m -json all | go-mod-outdated -update -direct + +NAME := ghcr.io/jonhadfield/ipscout +TAG := $(shell git rev-parse --short HEAD) +IMG := ${NAME}:${TAG} +LATEST := ${NAME}:latest + +build-docker: + docker build --platform=linux/amd64 --build-arg BUILD_TAG="$(BUILD_TAG)" --build-arg BUILD_SHA="$(BUILD_SHA)" --build-arg BUILD_DATE="$(BUILD_DATE) UTC" -t ${IMG} . + docker tag ${IMG} ${LATEST} + docker tag ${LATEST} ipscout:latest + docker tag ${LATEST} docker.io/jonhadfield/ipscout:latest + +pull-image: + docker pull jonhadfield/ipscout:latest + +scan-image: pull-image + trivy image jonhadfield/ipscout:latest + +build-latest-docker-tag: + docker build --build-arg="TAG=$(LATEST_TAG)" -f ./docker/Dockerfile -t ipscout ./docker + +release: + goreleaser && git push --follow-tags + +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.DEFAULT_GOAL := build diff --git a/README.md b/README.md new file mode 100644 index 0000000..f62946a --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# Overview + +IPScout is a command line tool useful for network administrators and security analysts to quickly identify the origin +and threat of +an IP address. Results can be cached to reduce API calls and improve performance. + +## Providers + +IPScout supports multiple well known sources. You can also provide custom sources +with the [Annotated](#Annotated) and [IPURL](#IPURL) providers. + +The following are currently supported: + +| Provider | Category | Needs Registration | +|:-------------------------------------------------------------------------------------------|:--------------:|:------------------:| +| [AWS](https://docs.aws.amazon.com/vpc/latest/userguide/aws-ip-ranges.html#aws-ip-download) | Cloud Provider | | +| [AbuseIPDB](https://www.abuseipdb.com/) | IP Reputation | 🔑 | +| [Annotated](#Annotated) | User Provided | | +| [Azure](https://www.microsoft.com/en-gb/download/details.aspx?id=56519) | Cloud Provider | | +| [CriminalIP](https://www.criminalip.io/) | IP Reputation | 🔑 | +| [DigitalOcean](https://www.criminalip.io/) | Cloud Provider | | +| [IPURL](#IPURL) | User Provided | | +| [PTR](https://www.cloudflare.com/en-gb/learning/dns/dns-records/dns-ptr-record/) | DNS | | +| [Shodan](https://www.shodan.io/) | IP Reputation | 🔑 | + +## Getting Started + +### Installation + +### + +### Sources diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..778c808 --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,158 @@ +package cache + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "path/filepath" + "time" + + "github.com/dgraph-io/badger/v4" +) + +type Item struct { + Key string + Value []byte + Version string + AppVersion string + Created time.Time +} + +var ( + ErrUpsertFailed = errors.New("upsert failed") + ErrCreateCacheFailed = errors.New("create cache failed") + ErrKeyNotFound = badger.ErrKeyNotFound + ErrCreateKeyFailed = errors.New("create key failed") + ErrDeleteKeyFailed = errors.New("delete key failed") +) + +func Create(logger *slog.Logger, path string) (*badger.DB, error) { + logger.Info("creating cache", "path", filepath.Join(path, "cache")) + + if path == "" { + return nil, errors.New("path is empty") + } + + db, err := badger.Open(badger.DefaultOptions(filepath.Join(path, "cache")).WithLogger(nil)) + if err != nil { + return nil, fmt.Errorf("error creating cache: %w", err) + } + + return db, nil +} + +func UpsertWithTTL(logger *slog.Logger, db *badger.DB, item Item, ttl time.Duration) error { + mItem, err := json.Marshal(item) + if err != nil { + return fmt.Errorf("error marshalling cache item: %w", err) + } + + logger.Info("upserting item", "key", item.Key, "value len", len(mItem), "ttl", ttl.String()) + + err = db.Update(func(txn *badger.Txn) error { + e := badger.NewEntry([]byte(item.Key), mItem).WithTTL(ttl) + return txn.SetEntry(e) + }) + if err != nil { + return fmt.Errorf("error upserting cache item: %w", err) + } + + return nil +} + +func Read(logger *slog.Logger, db *badger.DB, key string) (*Item, error) { + logger.Debug("reading cache item", "key", key) + + var item *Item + + err := db.View(func(txn *badger.Txn) error { + itemFound, tErr := txn.Get([]byte(key)) + if tErr != nil { + return fmt.Errorf("error getting cache item: %w", tErr) + } + + return itemFound.Value(func(val []byte) error { + logger.Debug("read cache item", "key", key, "value len", len(val)) + + if uErr := json.Unmarshal(val, &item); uErr != nil { + return fmt.Errorf("error unmarshalling cache item: %w", uErr) + } + + return nil + }) + }) + if err != nil { + return nil, fmt.Errorf("error reading cache item: %w", err) + } + + return item, nil +} + +func CheckExists(logger *slog.Logger, db *badger.DB, key string) (bool, error) { + logger.Debug("checking cache item exists", "key", key) + + var found bool + + err := db.View(func(txn *badger.Txn) error { + _, tErr := txn.Get([]byte(key)) + if tErr != nil { + if errors.Is(tErr, badger.ErrKeyNotFound) { + return nil + } + + return fmt.Errorf("error getting cache item: %w", tErr) + } + + found = true + + return nil + }) + if err != nil { + return false, fmt.Errorf("error checking cache item exists: %w", err) + } + + return found, nil +} + +func Delete(logger *slog.Logger, db *badger.DB, key string) error { + logger.Info("deleting cache item", "key", key) + + if err := db.Update(func(txn *badger.Txn) error { + return txn.Delete([]byte(key)) + }); err != nil { + return fmt.Errorf("error deleting cache item: %w", err) + } + + return nil +} + +func DeleteMultiple(logger *slog.Logger, db *badger.DB, keys []string) error { + logger.Info("deleting cache items", "keys", keys) + + var deleted int + + var notfound int + + for _, key := range keys { + found, err := CheckExists(logger, db, key) + if err != nil { + return err + } + + if !found { + notfound++ + continue + } + + if err = Delete(logger, db, key); err != nil { + return err + } + + deleted++ + } + + fmt.Printf("cache items deleted: %d not found: %d\n", deleted, notfound) + + return nil +} diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 0000000..cb094cc --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,159 @@ +package cache + +import ( + "log/slog" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCreate(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + c, err := Create(slog.New(slog.NewTextHandler(os.Stdout, nil)), tempDir) + require.NotNil(t, c) + require.NoError(t, err) + require.NoError(t, c.Close()) +} + +func TestCreateMissingPath(t *testing.T) { + t.Parallel() + + c, err := Create(slog.New(slog.NewTextHandler(os.Stdout, nil)), "") + require.Nil(t, c) + require.Error(t, err) +} + +func TestUpsert(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + + l := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + d, err := Create(l, tempDir) + require.NoError(t, err) + + now := time.Now() + require.NoError(t, UpsertWithTTL(l, d, Item{ + Key: "test-key", + Value: []byte("test value"), + Version: "Item Version x1.2.3", + AppVersion: "App Version x1.2.3", + Created: now, + }, 1*time.Second)) + + item, err := Read(l, d, "test-key") + require.NoError(t, err) + require.Equal(t, "test-key", item.Key) + require.Equal(t, "test value", string(item.Value)) + require.Equal(t, "Item Version x1.2.3", item.Version) + require.Equal(t, "App Version x1.2.3", item.AppVersion) + require.True(t, item.Created.Equal(now)) + + time.Sleep(2 * time.Second) + + exists, err := CheckExists(l, d, "test-key") + require.NoError(t, err) + // check it's expired + require.False(t, exists) +} + +func TestCheckExists(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + + l := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + d, err := Create(l, tempDir) + require.NoError(t, err) + + now := time.Now() + require.NoError(t, UpsertWithTTL(l, d, Item{ + Key: "test-key", + Value: []byte("test value"), + Version: "Item Version x1.2.3", + AppVersion: "App Version x1.2.3", + Created: now, + }, 10*time.Second)) + + exists, err := CheckExists(l, d, "test-key") + require.NoError(t, err) + require.True(t, exists) +} + +func TestDeleteOne(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + + l := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + d, err := Create(l, tempDir) + require.NoError(t, err) + + now := time.Now() + require.NoError(t, UpsertWithTTL(l, d, Item{ + Key: "test-key", + Value: []byte("test value"), + Version: "Item Version x1.2.3", + AppVersion: "App Version x1.2.3", + Created: now, + }, 10*time.Second)) + + require.NoError(t, UpsertWithTTL(l, d, Item{ + Key: "test-key2", + Value: []byte("test value2"), + Version: "Item Version x1.2.3", + AppVersion: "App Version x1.2.3", + Created: now, + }, 10*time.Second)) + + require.NoError(t, Delete(l, d, "test-key")) + exists, err := CheckExists(l, d, "test-key") + require.NoError(t, err) + require.False(t, exists) + exists, err = CheckExists(l, d, "test-key2") + require.NoError(t, err) + require.True(t, exists) +} + +func TestDeleteMultiple(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + + l := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + d, err := Create(l, tempDir) + require.NoError(t, err) + + now := time.Now() + require.NoError(t, UpsertWithTTL(l, d, Item{ + Key: "test-key", + Value: []byte("test value"), + Version: "Item Version x1.2.3", + AppVersion: "App Version x1.2.3", + Created: now, + }, 10*time.Second)) + + require.NoError(t, UpsertWithTTL(l, d, Item{ + Key: "test-key2", + Value: []byte("test value2"), + Version: "Item Version x1.2.3", + AppVersion: "App Version x1.2.3", + Created: now, + }, 10*time.Second)) + + require.NoError(t, DeleteMultiple(l, d, []string{"test-key", "test-key2", "missing-key"})) + exists, err := CheckExists(l, d, "test-key") + require.NoError(t, err) + require.False(t, exists) + exists, err = CheckExists(l, d, "test-key2") + require.NoError(t, err) + require.False(t, exists) +} diff --git a/cmd/cache.go b/cmd/cache.go new file mode 100644 index 0000000..2a1918c --- /dev/null +++ b/cmd/cache.go @@ -0,0 +1,109 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/jonhadfield/ipscout/manager" + + "github.com/spf13/cobra" +) + +func newCacheCommand() *cobra.Command { + cacheCmd := &cobra.Command{ + Use: "cache", + Short: "manage cached data", + Long: `manage cached data.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + _ = cmd.Help() + os.Exit(0) + } + + return nil + }, + } + + cacheCmd.AddCommand(newCacheDelCommand()) + cacheCmd.AddCommand(newCacheGetCommand()) + cacheCmd.AddCommand(newCacheListCommand()) + + return cacheCmd +} + +func newCacheListCommand() *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "list cached items", + Long: `list outputs all of the currently cached items.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // nolint:revive + return initConfig(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { // nolint:revive + mgr, err := manager.NewClient(sess) + if err != nil { + os.Exit(1) + } + + if err = mgr.List(); err != nil { + return fmt.Errorf("error listing cache items: %w", err) + } + + return nil + }, + } +} + +func newCacheDelCommand() *cobra.Command { + return &cobra.Command{ + Use: "delete", + Short: "delete items from cache", + Long: `delete one or more items from cache by specifying their keys.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // nolint:revive + return initConfig(cmd) + }, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { // nolint:revive + mgr, err := manager.NewClient(sess) + if err != nil { + os.Exit(1) + } + + if err = mgr.Delete(args); err != nil { + return fmt.Errorf("error deleting item from cache: %w", err) + } + + return nil + }, + } +} + +func newCacheGetCommand() *cobra.Command { + var raw bool + + cmd := &cobra.Command{ + Use: "get", + Short: "get item from cache", + Long: `get a cached item by providing its key.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // nolint:revive + return initConfig(cmd) + }, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { // nolint:revive + mgr, err := manager.NewClient(sess) + if err != nil { + os.Exit(1) + } + + if err = mgr.Get(args[0], raw); err != nil { + return fmt.Errorf("error getting item from cache: %w", err) + } + + return nil + }, + } + + cmd.PersistentFlags().BoolVar(&raw, "raw", false, "raw data only") + + return cmd +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..1b1583d --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,282 @@ +package cmd + +import ( + "fmt" + "log/slog" + "net/netip" + "os" + "strings" + + "github.com/jonhadfield/ipscout/process" + "github.com/jonhadfield/ipscout/session" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +const ( + AppName = "ipscout" +) + +var sess *session.Session + +func newRootCommand() *cobra.Command { + var ( + useTestData bool + ports []string + maxValueChars int32 + maxAge string + maxReports int + logLevel string + output string + disableCache bool + ) + + rootCmd := &cobra.Command{ + Use: "ipscout", + Short: "ipscout", + Long: `IPScout is a CLI application to prod to an IP address.`, + Args: cobra.MinimumNArgs(0), + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { // nolint:revive + return initConfig(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + // using test data doesn't require a host be provided + // but command does so use placeholder + if useTestData { + args = []string{"8.8.8.8"} + } + + if len(args) == 0 { + _ = cmd.Help() + os.Exit(0) + } + + var err error + + if sess.Host, err = netip.ParseAddr(args[0]); err != nil { + return fmt.Errorf("invalid host: %w", err) + } + + processor, err := process.New(sess) + if err != nil { + os.Exit(1) + } + processor.Run() + + return nil + }, + } + + // Define cobra flags, the default value has the lowest (least significant) precedence + rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "WARN", "set log level as: ERROR, WARN, INFO, DEBUG") + rootCmd.PersistentFlags().StringVar(&output, "output", "table", "output format: table, json") + rootCmd.PersistentFlags().StringVar(&maxAge, "max-age", "", "max age of data to consider") + rootCmd.PersistentFlags().IntVar(&maxReports, "max-reports", session.DefaultMaxReports, "max reports to output for each provider") + rootCmd.PersistentFlags().BoolVar(&useTestData, "use-test-data", false, "use test data") + rootCmd.PersistentFlags().BoolVar(&disableCache, "disable-cache", false, "disable cache") + rootCmd.PersistentFlags().StringSliceVarP(&ports, "ports", "p", nil, "limit ports") + rootCmd.PersistentFlags().Int32Var(&maxValueChars, "max-value-chars", 0, "max characters to output for any value") + + cacheCommand := newCacheCommand() + + rootCmd.AddCommand(cacheCommand) + rootCmd.AddCommand(versionCmd) + + return rootCmd +} + +func Execute() { + // setup session + rootCmd := newRootCommand() + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} + +func bindFlags(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().VisitAll(func(flg *pflag.Flag) { + configName := flg.Name + v.Set(configName, flg.Value) + + if !flg.Changed && v.IsSet(configName) { + val := v.Get(configName) + if err := cmd.Flags().Set(flg.Name, fmt.Sprintf("%v", val)); err != nil { + fmt.Printf("error setting flag %s: %v\n", flg.Name, err) + } + } + }) +} + +func initConfig(cmd *cobra.Command) error { + v := viper.New() + + sess = session.New() + configRoot := session.GetConfigRoot("", AppName) + sess.App.Version = version + sess.App.SemVer = semver + + if err := session.CreateDefaultConfigIfMissing(configRoot); err != nil { + fmt.Printf("can't create default session: %v\n", err) + + os.Exit(1) + } + + v.AddConfigPath(configRoot) + v.SetConfigName("config") + + if err := v.ReadInConfig(); err != nil { + fmt.Println("can't read session:", err) + os.Exit(1) + } + + v.AutomaticEnv() + + if err := session.CreateConfigPathStructure(configRoot); err != nil { + fmt.Printf("can't create cache directory: %v\n", err) + + os.Exit(1) + } + + readProviderAuthKeys(v) + + // set cmd flags to those learned by viper if cmd flag is not set and viper's is + bindFlags(cmd, v) + + sess.Target = os.Stderr + + sess.Providers.AbuseIPDB.Enabled = v.GetBool("providers.abuseipdb.enabled") + sess.Providers.AbuseIPDB.MaxAge = v.GetInt("providers.abuseipdb.max_age") + sess.Providers.Annotated.Enabled = v.GetBool("providers.annotated.enabled") + sess.Providers.Annotated.Paths = v.GetStringSlice("providers.annotated.paths") + sess.Providers.AWS.Enabled = v.GetBool("providers.aws.enabled") + sess.Providers.AWS.URL = v.GetString("providers.aws.url") + sess.Providers.Azure.Enabled = v.GetBool("providers.azure.enabled") + sess.Providers.Azure.URL = v.GetString("providers.azure.url") + sess.Providers.CriminalIP.Enabled = v.GetBool("providers.criminalip.enabled") + sess.Providers.DigitalOcean.Enabled = v.GetBool("providers.digitalocean.enabled") + sess.Providers.DigitalOcean.URL = v.GetString("providers.digitalocean.url") + sess.Providers.GCP.Enabled = v.GetBool("providers.gcp.enabled") + sess.Providers.GCP.URL = v.GetString("providers.gcp.url") + sess.Providers.ICloudPR.Enabled = v.GetBool("providers.icloudpr.enabled") + sess.Providers.ICloudPR.URL = v.GetString("providers.icloudpr.url") + sess.Providers.IPURL.Enabled = v.GetBool("providers.ipurl.enabled") + sess.Providers.IPURL.URLs = v.GetStringSlice("providers.ipurl.urls") + sess.Providers.Linode.Enabled = v.GetBool("providers.linode.enabled") + sess.Providers.Linode.URL = v.GetString("providers.linode.url") + sess.Providers.Shodan.Enabled = v.GetBool("providers.shodan.enabled") + sess.Providers.PTR.Enabled = v.GetBool("providers.ptr.enabled") + sess.Config.Global.Ports = v.GetStringSlice("global.ports") + sess.Config.Global.MaxValueChars = v.GetInt32("global.max_value_chars") + sess.Config.Global.MaxAge = v.GetString("global.max_age") + sess.Config.Global.MaxReports = v.GetInt("global.max_reports") + // TODO: Nasty Hack Alert + // if not specified, ports is returned as a string: "[]" + // set to nil if that's the case + if len(sess.Config.Global.Ports) == 0 || sess.Config.Global.Ports[0] == "[]" { + sess.Config.Global.Ports = nil + } + + sess.Config.Global.MaxAge = v.GetString("global.max_age") + + // initialise logging + initLogging(cmd) + + sess.HTTPClient = getHTTPClient() + + utd, err := cmd.Flags().GetBool("use-test-data") + if err != nil { + os.Exit(1) + } + + sess.UseTestData = utd + + ports, _ := cmd.Flags().GetStringSlice("ports") + // TODO: Nasty Hack Alert + // if not specified, ports is returned as a string: "[]" + // set to nil if that's the case + if len(ports) == 0 || ports[0] == "[]" { + ports = nil + } + // if no ports specified on cli then default to global ports + if len(ports) > 0 { + sess.Config.Global.Ports = ports + } + + maxAge, _ := cmd.Flags().GetString("max-age") + if maxAge != "" { + sess.Config.Global.MaxAge = maxAge + } + + disableCache, _ := cmd.Flags().GetBool("disable-cache") + if disableCache { + sess.Config.Global.DisableCache = disableCache + } + + output, _ := cmd.Flags().GetString("output") + if output != "" { + sess.Config.Global.Output = output + } + + maxValueChars, _ := cmd.Flags().GetInt32("max-value-chars") + if maxValueChars > 0 { + sess.Config.Global.MaxValueChars = maxValueChars + } + + sess.Config.Global.IndentSpaces = session.DefaultIndentSpaces + + return nil +} + +var ProgramLevel = new(slog.LevelVar) // Info by default + +func initLogging(cmd *cobra.Command) { + hOptions := slog.HandlerOptions{AddSource: false} + + ll, err := cmd.Flags().GetString("log-level") + if err != nil { + fmt.Println("error getting log-level:", err) + os.Exit(1) + } + + sess.Config.Global.LogLevel = ll + + // set log level + switch strings.ToUpper(ll) { + case "ERROR": + ProgramLevel.Set(slog.LevelError) + + sess.HideProgress = false + case "WARN": + ProgramLevel.Set(slog.LevelWarn) + + sess.HideProgress = false + case "INFO": + ProgramLevel.Set(slog.LevelInfo) + + sess.HideProgress = true + case "DEBUG": + ProgramLevel.Set(slog.LevelDebug) + + sess.HideProgress = true + } + + hOptions.Level = ProgramLevel + + sess.Logger = slog.New(slog.NewTextHandler(sess.Target, &hOptions)) +} + +func readProviderAuthKeys(v *viper.Viper) { + // read provider auth keys from env if not set in session + if sess.Providers.AbuseIPDB.APIKey == "" { + sess.Providers.AbuseIPDB.APIKey = v.GetString("abuseipdb_api_key") + } + + if sess.Providers.Shodan.APIKey == "" { + sess.Providers.Shodan.APIKey = v.GetString("shodan_api_key") + } + + if sess.Providers.CriminalIP.APIKey == "" { + sess.Providers.CriminalIP.APIKey = v.GetString("criminal_ip_api_key") + } +} diff --git a/cmd/root_test.go b/cmd/root_test.go new file mode 100644 index 0000000..1d619dd --- /dev/null +++ b/cmd/root_test.go @@ -0,0 +1 @@ +package cmd diff --git a/cmd/util.go b/cmd/util.go new file mode 100644 index 0000000..fa1cde0 --- /dev/null +++ b/cmd/util.go @@ -0,0 +1,13 @@ +package cmd + +import "github.com/hashicorp/go-retryablehttp" + +func getHTTPClient() *retryablehttp.Client { + hc := retryablehttp.NewClient() + hc.RetryWaitMin = 3 + hc.RetryWaitMax = 5 + hc.RetryMax = 3 + hc.Logger = nil + + return hc +} diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 0000000..0fac4c0 --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var ( + version string + semver string +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Print the version number of IPScout", + Long: `Print the version number of IPScout`, + Run: func(cmd *cobra.Command, args []string) { // nolint:revive + fmt.Println("ip scout", version) + }, +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..75b82e0 --- /dev/null +++ b/go.mod @@ -0,0 +1,74 @@ +module github.com/jonhadfield/ipscout + +go 1.22 + +require ( + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de + github.com/briandowns/spinner v1.23.0 + github.com/dgraph-io/badger/v4 v4.2.0 + github.com/dustin/go-humanize v1.0.1 + github.com/fatih/color v1.16.0 + github.com/hashicorp/go-retryablehttp v0.7.5 + github.com/jedib0t/go-pretty/v6 v6.5.8 + github.com/jonhadfield/ip-fetcher v0.0.0-20240503180217-558f6741c36f + github.com/miekg/dns v1.1.59 + github.com/mitchellh/go-homedir v1.1.0 + github.com/spf13/cobra v1.8.0 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.18.2 + github.com/stretchr/testify v1.9.0 + golang.org/x/sync v0.7.0 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 +) + +//replace github.com/jonhadfield/ip-fetcher => ../ip-fetcher + +//replace github.com/likexian/whois-parser => ../whois-parser + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.2.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jszwec/csvutil v1.10.0 // indirect + github.com/klauspost/compress v1.17.8 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/term v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.20.0 // indirect + google.golang.org/protobuf v1.34.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..80f8875 --- /dev/null +++ b/go.sum @@ -0,0 +1,268 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= +github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= +github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= +github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jedib0t/go-pretty/v6 v6.5.8 h1:8BCzJdSvUbaDuRba4YVh+SKMGcAAKdkcF3SVFbrHAtQ= +github.com/jedib0t/go-pretty/v6 v6.5.8/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= +github.com/jonhadfield/ip-fetcher v0.0.0-20240503180217-558f6741c36f h1:HxE19TTjhpHylMfgeoply4qrztj/seduEAO+Q/pQcJA= +github.com/jonhadfield/ip-fetcher v0.0.0-20240503180217-558f6741c36f/go.mod h1:xckxqrzpzUyfNGSRKc+Ugp47f5iwogDYQB4iUUxfXHg= +github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI= +github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= +gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go new file mode 100644 index 0000000..77c5901 --- /dev/null +++ b/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/jonhadfield/ipscout/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/manager/manager.go b/manager/manager.go new file mode 100644 index 0000000..1065d20 --- /dev/null +++ b/manager/manager.go @@ -0,0 +1,207 @@ +package manager + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/dgraph-io/badger/v4" + "github.com/dustin/go-humanize" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/present" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" + "github.com/mitchellh/go-homedir" +) + +const MaxColumnWidth = 60 + +type Client struct { + Config *session.Session +} + +var timeFormat = "2006-01-02 15:04:05 MST" + +func (c *Client) CreateItemsInfoTable(info []CacheItemInfo) (*table.Writer, error) { + tw := table.NewWriter() + tw.AppendHeader(table.Row{"Key", "Expires", "Size", "App Version"}) + + for _, x := range info { + tw.AppendRow(table.Row{x.Key, x.ExpiresAt.Format(timeFormat), humanize.Bytes(uint64(x.EstimatedSize)), present.DashIfEmpty(x.AppVersion)}) + } + + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 1, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 20}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("CACHE ITEMS") + + return &tw, nil +} + +func NewClient(config *session.Session) (Client, error) { + p := Client{ + Config: config, + } + + return p, nil +} + +func (c *Client) List() error { + homeDir, err := homedir.Dir() + if err != nil { + c.Config.Logger.Error("failed to get home directory", "error", err) + + os.Exit(1) + } + + db, err := cache.Create(c.Config.Logger, filepath.Join(homeDir, ".config", "ipscout")) + if err != nil { + c.Config.Logger.Error("failed to create cache", "error", err) + + os.Exit(1) + } + + c.Config.Cache = db + + defer db.Close() + + cacheItemsInfo, err := c.GetCacheItemsInfo() + if err != nil { + return err + } + + tables, err := c.CreateItemsInfoTable(cacheItemsInfo) + if err != nil { + return err + } + + present.Tables(c.Config, []*table.Writer{tables}) + + return nil +} + +func (c *Client) Delete(keys []string) error { + homeDir, err := homedir.Dir() + if err != nil { + c.Config.Logger.Error("failed to get home directory", "error", err) + + os.Exit(1) + } + + db, err := cache.Create(c.Config.Logger, filepath.Join(homeDir, ".config", "ipscout")) + if err != nil { + c.Config.Logger.Error("failed to create cache", "error", err) + + os.Exit(1) + } + + c.Config.Cache = db + + defer db.Close() + + if err = cache.DeleteMultiple(c.Config.Logger, db, keys); err != nil { + return fmt.Errorf("error deleting cache items: %w", err) + } + + return nil +} + +func (c *Client) Get(key string, raw bool) error { + homeDir, err := homedir.Dir() + if err != nil { + c.Config.Logger.Error("failed to get home directory", "error", err) + + os.Exit(1) + } + + db, err := cache.Create(c.Config.Logger, filepath.Join(homeDir, ".config", "ipscout")) + if err != nil { + c.Config.Logger.Error("failed to create cache", "error", err) + + os.Exit(1) + } + + c.Config.Cache = db + + defer db.Close() + + item, err := cache.Read(c.Config.Logger, db, key) + if err != nil { + return fmt.Errorf("error reading cache: %w", err) + } + + if raw { + fmt.Printf("%s\n", item.Value) + + return nil + } + + type PresentationItem struct { + AppVersion string + Key string + Value json.RawMessage + Version string + Created string + } + + var pItem PresentationItem + pItem.Key = item.Key + pItem.AppVersion = item.AppVersion + pItem.Version = item.Version + pItem.Created = item.Created.Format(timeFormat) + pItem.Value = item.Value + + out, err := json.MarshalIndent(&pItem, "", " ") + if err != nil { + return fmt.Errorf("error marshalling item: %w", err) + } + + fmt.Printf("%s\n", out) + + return nil +} + +type CacheItemInfo struct { + AppVersion string + Key string + Value []byte + ExpiresAt time.Time + EstimatedSize int64 +} + +func (c *Client) GetCacheItemsInfo() ([]CacheItemInfo, error) { + var cacheItemsInfo []CacheItemInfo + + db := c.Config.Cache + + err := db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + + prefix := []byte(providers.CacheProviderPrefix) + // prefix := []byte(providers.CacheProviderPrefix) + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + item := it.Item() + k := item.Key() + item.ExpiresAt() + item.EstimatedSize() + + cacheItemsInfo = append(cacheItemsInfo, CacheItemInfo{ + Key: string(k), + EstimatedSize: item.EstimatedSize(), + ExpiresAt: time.Unix(int64(item.ExpiresAt()), 0), + }) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("error reading cache: %w", err) + } + + return cacheItemsInfo, nil +} diff --git a/out b/out new file mode 100644 index 0000000..6de26f5 --- /dev/null +++ b/out @@ -0,0 +1 @@ +/dev/stderr[{"abuseipdb":{"data":{"ipAddress":"172.224.224.60","isPublic":true,"ipVersion":4,"isWhitelisted":null,"abuseConfidenceScore":0,"countryCode":"GB","usageType":"Data Center\/Web Hosting\/Transit","isp":"iCloud Private Relay","domain":"icloud.com","hostnames":["a172-224-224-60.source.akaquill.net"],"isTor":false,"countryName":"United Kingdom of Great Britain and Northern Ireland","totalReports":0,"numDistinctUsers":0,"lastReportedAt":null,"reports":[]}}},{"icloudpr":{"Raw":null,"ip_prefix":"172.224.224.60/31","alpha2code":"GB","region":"GB-WA","city":"Cardiff","postal_code":"","synctoken":"\"662f6d82-b01feb\"","creation_time":"0001-01-01T00:00:00Z"}},{"ptr":{"ptr":[{"Hdr":{"Name":"60.224.224.172.in-addr.arpa.","Rrtype":12,"Class":1,"Ttl":43200,"Rdlength":37},"Ptr":"a172-224-224-60.source.akaquill.net."}],"msg":{"Id":0,"Response":false,"Opcode":0,"Authoritative":false,"Truncated":false,"RecursionDesired":false,"RecursionAvailable":false,"Zero":false,"AuthenticatedData":false,"CheckingDisabled":false,"Rcode":0,"Question":null,"Answer":null,"Ns":null,"Extra":null}}}] diff --git a/present/present.go b/present/present.go new file mode 100644 index 0000000..2068946 --- /dev/null +++ b/present/present.go @@ -0,0 +1,106 @@ +package present + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" + "github.com/jonhadfield/ipscout/providers/criminalip" + "github.com/jonhadfield/ipscout/providers/shodan" + "github.com/jonhadfield/ipscout/session" +) + +type Resulter interface { + CreateTable() *table.Writer +} + +func Tables(c *session.Session, tws []*table.Writer) { + outputTables(c, tws) +} + +func JSON(jms *json.RawMessage) error { + var out bytes.Buffer + + if err := json.Indent(&out, *jms, "", " "); err != nil { + return fmt.Errorf("error indenting JSON: %w", err) + } + + fmt.Println(out.String()) + + return nil +} + +func DashIfEmpty(value interface{}) string { + switch v := value.(type) { + case time.Time: + if v.IsZero() || v == time.Date(0o001, time.January, 1, 0, 0, 0, 0, time.UTC) { + return "-" + } + + return v.Format(time.DateTime) + case string: + trimmed := strings.TrimSpace(v) + if len(trimmed) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(strings.TrimSpace(*v)) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} + +type CombinedData struct { + Shodan shodan.HostSearchResult + CriminalIP criminalip.HostSearchResult +} + +func outputTables(c *session.Session, tws []*table.Writer) { + twOuter := table.NewWriter() + + myOuterStyle := table.StyleColoredDark + myOuterStyle.Title.Align = text.AlignCenter + myOuterStyle.Title.Colors = text.Colors{text.FgRed, text.BgBlack} + + twOuter.SetTitle("IPScout [v" + c.App.SemVer + "]") + twOuter.SetColumnConfigs([]table.ColumnConfig{ + {Number: 1, AutoMerge: false, WidthMin: 60}, + }) + + myOuterStyle.Box = table.StyleBoxRounded + myOuterStyle.Options.SeparateRows = true + myOuterStyle.Options.DrawBorder = true + myOuterStyle.Options.DoNotColorBordersAndSeparators = true + twOuter.SetStyle(myOuterStyle) + + myInnerStyle := table.StyleColoredDark + myInnerStyle.Options = table.OptionsNoBordersAndSeparators + myInnerColourOptions := table.ColorOptionsDark + myInnerColourOptions.Row = text.Colors{text.FgWhite, text.BgBlack} + myInnerColourOptions.IndexColumn = text.Colors{text.FgHiWhite, text.BgBlack} + + myInnerColourOptions.RowAlternate = table.ColorOptionsDefault.Row + // myInnerColourOptions.Row = table.ColorOptionsDefault.Row + myInnerStyle.Color = myInnerColourOptions + + for _, tw := range tws { + t := *tw + t.SetIndexColumn(1) + t.SetStyle(myInnerStyle) + twOuter.AppendRow([]interface{}{t.Render()}) + } + + fmt.Println(twOuter.Render()) +} diff --git a/process/process.go b/process/process.go new file mode 100644 index 0000000..e470cd1 --- /dev/null +++ b/process/process.go @@ -0,0 +1,401 @@ +package process + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/jonhadfield/ipscout/providers/icloudpr" + + "github.com/jonhadfield/ipscout/providers/gcp" + "github.com/jonhadfield/ipscout/providers/linode" + + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/providers/abuseipdb" + "github.com/jonhadfield/ipscout/providers/annotated" + "github.com/jonhadfield/ipscout/providers/aws" + "github.com/jonhadfield/ipscout/providers/azure" + "github.com/jonhadfield/ipscout/providers/digitalocean" + "github.com/jonhadfield/ipscout/providers/ptr" + + "github.com/briandowns/spinner" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/present" + "github.com/jonhadfield/ipscout/providers/criminalip" + "github.com/jonhadfield/ipscout/providers/ipurl" + "github.com/jonhadfield/ipscout/providers/shodan" + "github.com/jonhadfield/ipscout/session" + "github.com/mitchellh/go-homedir" + "golang.org/x/sync/errgroup" +) + +type Provider struct { + Name string + Enabled bool + APIKey string + NewClient func(c session.Session) (providers.ProviderClient, error) +} + +func getProviderClients(sess session.Session) (map[string]providers.ProviderClient, error) { + runners := make(map[string]providers.ProviderClient) + + pros := []Provider{ + {Name: abuseipdb.ProviderName, Enabled: sess.Providers.AbuseIPDB.Enabled, APIKey: sess.Providers.AbuseIPDB.APIKey, NewClient: abuseipdb.NewClient}, + {Name: annotated.ProviderName, Enabled: sess.Providers.Annotated.Enabled, APIKey: "", NewClient: annotated.NewProviderClient}, + {Name: aws.ProviderName, Enabled: sess.Providers.AWS.Enabled, APIKey: "", NewClient: aws.NewProviderClient}, + {Name: azure.ProviderName, Enabled: sess.Providers.Azure.Enabled, APIKey: "", NewClient: azure.NewProviderClient}, + {Name: criminalip.ProviderName, Enabled: sess.Providers.CriminalIP.Enabled, APIKey: sess.Providers.CriminalIP.APIKey, NewClient: criminalip.NewProviderClient}, + {Name: digitalocean.ProviderName, Enabled: sess.Providers.DigitalOcean.Enabled, APIKey: "", NewClient: digitalocean.NewProviderClient}, + {Name: gcp.ProviderName, Enabled: sess.Providers.GCP.Enabled, APIKey: "", NewClient: gcp.NewProviderClient}, + {Name: ipurl.ProviderName, Enabled: sess.Providers.IPURL.Enabled, APIKey: "", NewClient: ipurl.NewProviderClient}, + {Name: icloudpr.ProviderName, Enabled: sess.Providers.ICloudPR.Enabled, APIKey: "", NewClient: icloudpr.NewProviderClient}, + {Name: linode.ProviderName, Enabled: sess.Providers.Linode.Enabled, APIKey: "", NewClient: linode.NewProviderClient}, + {Name: shodan.ProviderName, Enabled: sess.Providers.Shodan.Enabled, APIKey: sess.Providers.Shodan.APIKey, NewClient: shodan.NewProviderClient}, + {Name: ptr.ProviderName, Enabled: sess.Providers.PTR.Enabled, APIKey: "", NewClient: ptr.NewProviderClient}, + } + + for _, provider := range pros { + if provider.Enabled || sess.UseTestData || provider.APIKey != "" { + client, err := provider.NewClient(sess) + if err != nil { + return nil, fmt.Errorf("error creating %s client: %w", provider.Name, err) + } + + if client != nil { + runners[provider.Name] = client + } + } + } + + return runners, nil +} + +func getEnabledProviders(runners map[string]providers.ProviderClient) []string { + keys := make([]string, 0, len(runners)) + for k := range runners { + keys = append(keys, k) + } + + return keys +} + +type Config struct { + session.Session + Shodan shodan.Config + CriminalIP criminalip.Config + IPURL ipurl.Config +} + +type Processor struct { + Session *session.Session +} + +func (p *Processor) Run() { + homeDir, err := homedir.Dir() + if err != nil { + p.Session.Logger.Error("failed to get home directory", "error", err) + + os.Exit(1) + } + + db, err := cache.Create(p.Session.Logger, filepath.Join(homeDir, ".config", "ipscout")) + if err != nil { + p.Session.Logger.Error("failed to create cache", "error", err) + + os.Exit(1) + } + + p.Session.Cache = db + + defer db.Close() + + // get provider clients + providerClients, err := getProviderClients(*p.Session) + if err != nil { + p.Session.Logger.Error("failed to generate provider clients", "error", err) + + // close here as exit prevents defer from running + _ = db.Close() + + os.Exit(1) // nolint:gocritic + } + + enabledProviders := getEnabledProviders(providerClients) + + // initialise providers + initialiseProviders(p.Session.Logger, providerClients, p.Session.HideProgress) + + if strings.EqualFold(p.Session.Config.Global.LogLevel, "debug") { + for provider, dur := range p.Session.Stats.InitialiseDuration { + p.Session.Logger.Debug("initialise timing", "provider", provider, "duration", dur.String()) + } + } + + // find hosts + results := findHosts(providerClients, p.Session.HideProgress) + + if strings.EqualFold(p.Session.Config.Global.LogLevel, "debug") { + for provider, dur := range p.Session.Stats.FindHostDuration { + p.Session.Logger.Debug("find hosts timing", "provider", provider, "duration", dur.String()) + } + + for provider, uc := range p.Session.Stats.FindHostUsedCache { + p.Session.Logger.Debug("find hosts data load", "provider", provider, "cache", uc) + } + } + + results.RLock() + matchingResults := len(results.m) + results.RUnlock() + + p.Session.Logger.Info("host matching results", "providers queried", len(providerClients), "matching results", matchingResults) + + if matchingResults == 0 { + p.Session.Logger.Warn("no results found", "host", p.Session.Host.String(), "providers checked", strings.Join(enabledProviders, ", ")) + + return + } + + // output data + if err = output(p.Session, providerClients, results); err != nil { + p.Session.Logger.Error("failed to output data", "error", err) + + os.Exit(1) + } +} + +func initialiseProviders(l *slog.Logger, runners map[string]providers.ProviderClient, hideProgress bool) { + var err error + + var g errgroup.Group + + s := spinner.New(spinner.CharSets[11], 100*time.Millisecond, spinner.WithWriter(os.Stderr)) + + if !hideProgress { + s.Start() // Start the spinner + // time.Sleep(4 * time.Second) // Run for some time to simulate work + s.Suffix = " initialising providers..." + + defer func() { + stopSpinnerIfActive(s) + }() + } + + for name, runner := range runners { + _, runner := name, runner // https://golang.org/doc/faq#closures_and_goroutines + + g.Go(func() error { + name := name + + gErr := runner.Initialise() + if gErr != nil { + stopSpinnerIfActive(s) + l.Error("failed to initialise", "provider", name, "error", gErr.Error()) + + if !hideProgress { + s.Start() + } + } + + return nil + }) + } + + if err = g.Wait(); err != nil { + stopSpinnerIfActive(s) + + return + } + // allow time to output spinner + time.Sleep(50 * time.Millisecond) +} + +func stopSpinnerIfActive(s *spinner.Spinner) { + if s != nil && s.Active() { + s.Stop() + } +} + +type findHostsResults struct { + sync.RWMutex + m map[string][]byte +} + +type generateTablesResults struct { + sync.RWMutex + m []*table.Writer +} + +func findHosts(runners map[string]providers.ProviderClient, hideProgress bool) *findHostsResults { + var results findHostsResults + + results.Lock() + results.m = make(map[string][]byte) + results.Unlock() + + var w sync.WaitGroup + + if !hideProgress { + s := spinner.New(spinner.CharSets[11], 100*time.Millisecond, spinner.WithWriter(os.Stderr)) + s.Start() // Start the spinner + // time.Sleep(4 * time.Second) // Run for some time to simulate work + s.Suffix = " searching providers..." + + defer s.Stop() + } + + for name, runner := range runners { + w.Add(1) + + go func() { + defer w.Done() + + result, err := runner.FindHost() + if err != nil { + runner.GetConfig().Logger.Debug(err.Error()) + + return + } + + if result != nil { + results.Lock() + results.m[name] = result + results.Unlock() + } + }() + } + + w.Wait() + // allow time to output spinner + time.Sleep(50 * time.Millisecond) + + return &results +} + +// func generateOutput(conf *session.Session, runners map[string]providers.ProviderClient, results *findHostsResults) error { +func output(sess *session.Session, runners map[string]providers.ProviderClient, results *findHostsResults) error { + switch sess.Config.Global.Output { + case "table": + tables := generateTables(sess, runners, results) + + if strings.EqualFold(sess.Config.Global.LogLevel, "debug") { + for provider, dur := range sess.Stats.CreateTableDuration { + sess.Logger.Debug("create tables timing", "provider", provider, "duration", dur.String()) + } + } + + present.Tables(sess, tables) + case "json": + jo, err := generateJSON(results) + if err != nil { + return err + } + + if err = present.JSON(&jo); err != nil { + return fmt.Errorf("error outputting JSON: %w", err) + } + default: + return fmt.Errorf("unsupported output format: %s", sess.Config.Global.Output) + } + + return nil +} + +func generateTables(conf *session.Session, runners map[string]providers.ProviderClient, results *findHostsResults) []*table.Writer { + var tables generateTablesResults + + var w sync.WaitGroup + + if !conf.HideProgress { + s := spinner.New(spinner.CharSets[11], 100*time.Millisecond, spinner.WithWriterFile(conf.Target)) + s.Start() // Start the spinner + + s.Suffix = " generating output..." + + defer s.Stop() + } + + for name, runner := range runners { + name, runner := name, runner // https://golang.org/doc/faq#closures_and_goroutines + + w.Add(1) + + go func() { + defer w.Done() + results.RLock() + if results.m[name] == nil { + return + } + + createTableData := results.m[name] + results.RUnlock() + + tbl, err := runner.CreateTable(createTableData) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + + if tbl != nil { + tables.RWMutex.Lock() + tables.m = append(tables.m, tbl) + tables.RWMutex.Unlock() + } + }() + } + + w.Wait() + // allow time to output spinner + time.Sleep(50 * time.Millisecond) + + return tables.m +} + +func generateJSON(results *findHostsResults) (json.RawMessage, error) { + var counter int64 + + var out json.RawMessage + + for name := range results.m { + results.RLock() + + if results.m[name] == nil { + return nil, fmt.Errorf("no data found for %s", name) + } + + if counter == 0 { + out = json.RawMessage([]byte("[")) + } + + cj := json.RawMessage(results.m[name]) + out = append(out, json.RawMessage([]byte("{\""+name+"\":"))...) + out = append(out, cj...) + out = append(out, json.RawMessage([]byte("}"))...) + + if counter == int64(len(results.m)-1) { + out = append(out, json.RawMessage([]byte("]"))...) + } else { + out = append(out, json.RawMessage([]byte(","))...) + } + + counter++ + + results.RUnlock() + } + + return out, nil +} + +func New(config *session.Session) (Processor, error) { + p := Processor{ + Session: config, + } + + return p, nil +} diff --git a/process/process_test.go b/process/process_test.go new file mode 100644 index 0000000..f2ef9d2 --- /dev/null +++ b/process/process_test.go @@ -0,0 +1 @@ +package process diff --git a/providers/abuseipdb/abuseipdb.go b/providers/abuseipdb/abuseipdb.go new file mode 100644 index 0000000..0cf296b --- /dev/null +++ b/providers/abuseipdb/abuseipdb.go @@ -0,0 +1,366 @@ +package abuseipdb + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/netip" + "net/url" + "os" + "strings" + "time" + + "github.com/fatih/color" + "github.com/hashicorp/go-retryablehttp" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "abuseipdb" + APIURL = "https://api.abuseipdb.com" + HostIPPath = "/api/v2/check" + MaxColumnWidth = 120 + IndentPipeHyphens = " |-----" + portLastModifiedFormat = "2006-01-02T15:04:05+07:00" + ResultTTL = time.Duration(12 * time.Hour) +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +func NewClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating abuseipdb client") + + tc := Client{ + c, + } + + return &tc, nil +} + +func (c *Client) GetData() (result *HostSearchResult, err error) { + result, err = loadResultsFile("abuseipdb/testdata/abuseipdb_google_dns_resp.json") + if err != nil { + return nil, err + } + + return result, nil +} + +type Provider interface { + LoadData() ([]byte, error) + CreateTable([]byte) (*table.Writer, error) +} + +func (c *Client) Enabled() bool { + return c.Session.Providers.AbuseIPDB.Enabled +} + +func (c *Client) GetConfig() *session.Session { + return &c.Session +} + +type Client struct { + session.Session +} + +func (c *Client) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising abuseipdb client") + + if c.Providers.AbuseIPDB.APIKey == "" && !c.UseTestData { + return fmt.Errorf("abuseipdb provider api key not set") + } + + return nil +} + +func (c *Client) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := fetchData(c.Session) + if err != nil { + return nil, err + } + + c.Logger.Debug("abuseipdb host match data", "size", len(result.Raw)) + + return result.Raw, nil +} + +func (c *Client) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var result *HostSearchResult + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("error unmarshalling abuseipdb data: %w", err) + } + + if result == nil { + return nil, nil + } + + tw := table.NewWriter() + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 1, AutoMerge: true}, + }) + + tw.AppendRow(table.Row{"Last Reported", providers.DashIfEmpty(result.Data.LastReportedAt)}) + tw.AppendRow(table.Row{"Abuse Confidence Score", providers.DashIfEmpty(result.Data.AbuseConfidenceScore)}) + tw.AppendRow(table.Row{"Public", result.Data.IsPublic}) + tw.AppendRow(table.Row{"Domain", providers.DashIfEmpty(result.Data.Domain)}) + tw.AppendRow(table.Row{"Hostnames", providers.DashIfEmpty(strings.Join(result.Data.Hostnames, ", "))}) + tw.AppendRow(table.Row{"TOR", result.Data.IsTor}) + tw.AppendRow(table.Row{"Country", providers.DashIfEmpty(result.Data.CountryName)}) + tw.AppendRow(table.Row{"Usage Type", providers.DashIfEmpty(result.Data.UsageType)}) + tw.AppendRow(table.Row{"ISP", providers.DashIfEmpty(result.Data.Isp)}) + tw.AppendRow(table.Row{"Reports", fmt.Sprintf("%d (%d days %d users)", + result.Data.TotalReports, c.Providers.AbuseIPDB.MaxAge, result.Data.NumDistinctUsers)}) + + for x, dr := range result.Data.Reports { + tw.AppendRow(table.Row{"", color.CyanString("%s", dr.ReportedAt.Format(time.DateTime))}) + tw.AppendRow(table.Row{"", fmt.Sprintf("%s Comment: %s", IndentPipeHyphens, dr.Comment)}) + + if x == c.Config.Global.MaxReports { + break + } + } + + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: true, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + // tw.SetStyle(table.StyleColoredDark) + // tw.Style().Options.DrawBorder = true + tw.SetTitle("AbuseIPDB | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("AbuseIPDB | Host: %s", result.Data.IPAddress) + } + + c.Logger.Debug("abuseipdb table created", "host", c.Host.String()) + + return &tw, nil +} + +func loadAPIResponse(ctx context.Context, c session.Session, apiKey string) (res *HostSearchResult, err error) { + urlPath, err := url.JoinPath(APIURL, HostIPPath) + if err != nil { + return nil, fmt.Errorf("failed to create abuseipdb api url path: %w", err) + } + + sURL, err := url.Parse(urlPath) + if err != nil { + panic(err) + } + + sURL.RawQuery = fmt.Sprintf("ipAddress=%s&verbose=false&maxAgeInDays=1", c.Host.String()) + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodGet, sURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Add("Key", apiKey) + req.Header.Add("Accept", "application/json") + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to make request: %w", err) + } + + if resp.StatusCode == http.StatusNotFound { + return nil, providers.ErrNoMatchFound + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("abuseipdb api request failed: %s", resp.Status) + } + + // read response body + rBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading abuseipdb response: %w", err) + } + + defer resp.Body.Close() + + if rBody == nil { + return nil, providers.ErrNoDataFound + } + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + if err = os.WriteFile(fmt.Sprintf("%s/backups/abuseipdb_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), rBody, 0o600); err != nil { + panic(err) + } + + c.Logger.Debug("backed up abuseipdb response", "host", c.Host.String()) + } + + res, err = unmarshalResponse(rBody) + if err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + res.Raw = rBody + if res.Raw == nil { + return nil, fmt.Errorf("abuseipdb: %w", providers.ErrNoMatchFound) + } + + return res, nil +} + +func unmarshalResponse(data []byte) (*HostSearchResult, error) { + var res HostSearchResult + + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling abuseipdb response: %w", err) + } + + res.Raw = data + + return &res, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening abuseipdb file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding abuseipdb file: %w", err) + } + + return res, nil +} + +func (ssr *HostSearchResult) CreateTable() *table.Writer { + tw := table.NewWriter() + + return &tw +} + +func fetchData(c session.Session) (*HostSearchResult, error) { + var result *HostSearchResult + + var err error + + if c.UseTestData { + result, err = loadResultsFile("providers/abuseipdb/testdata/abuseipdb_google_dns_resp.json") + if err != nil { + return nil, fmt.Errorf("error loading abuseipdb test data: %w", err) + } + + return result, nil + } + + // load data from cache + cacheKey := fmt.Sprintf("abuseipdb_%s_report.json", strings.ReplaceAll(c.Host.String(), ".", "_")) + + var item *cache.Item + + if item, err = cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + if item.Value != nil && len(item.Value) > 0 { + result, err = unmarshalResponse(item.Value) + if err != nil { + return nil, fmt.Errorf("error unmarshalling cached abuseipdb response: %w", err) + } + + c.Logger.Info("abuseipdb response found in cache", "host", c.Host.String()) + + result.Raw = item.Value + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return result, nil + } + } + + result, err = loadAPIResponse(context.Background(), c, c.Providers.AbuseIPDB.APIKey) + if err != nil { + return nil, fmt.Errorf("loading abuseipdb api response: %w", err) + } + + if err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: cacheKey, + Value: result.Raw, + Created: time.Now(), + }, ResultTTL); err != nil { + return nil, fmt.Errorf("error caching abuseipdb response: %w", err) + } + + return result, nil +} + +type HostSearchResult struct { + Raw []byte `json:"raw"` + Data struct { + IPAddress string `json:"ipAddress,omitempty"` + IsPublic bool `json:"isPublic,omitempty"` + IPVersion int `json:"ipVersion,omitempty"` + IsWhitelisted bool `json:"isWhitelisted,omitempty"` + AbuseConfidenceScore int `json:"abuseConfidenceScore,omitempty"` + CountryCode string `json:"countryCode,omitempty"` + CountryName string `json:"countryName,omitempty"` + UsageType string `json:"usageType,omitempty"` + Isp string `json:"isp,omitempty"` + Domain string `json:"domain,omitempty"` + Hostnames []string `json:"hostnames,omitempty"` + IsTor bool `json:"isTor,omitempty"` + TotalReports int `json:"totalReports,omitempty"` + NumDistinctUsers int `json:"numDistinctUsers,omitempty"` + LastReportedAt time.Time `json:"lastReportedAt,omitempty"` + Reports []struct { + ReportedAt time.Time `json:"reportedAt,omitempty"` + Comment string `json:"comment,omitempty"` + Categories []int `json:"categories,omitempty"` + ReporterID int `json:"reporterId,omitempty"` + ReporterCountryCode string `json:"reporterCountryCode,omitempty"` + ReporterCountryName string `json:"reporterCountryName,omitempty"` + } `json:"reports,omitempty"` + } `json:"data,omitempty"` +} diff --git a/providers/annotated/annotated.go b/providers/annotated/annotated.go new file mode 100644 index 0000000..f0340cc --- /dev/null +++ b/providers/annotated/annotated.go @@ -0,0 +1,495 @@ +package annotated + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/netip" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" + "gopkg.in/yaml.v3" + + "github.com/araddon/dateparse" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "annotated" + CacheTTL = 5 * time.Minute + MaxColumnWidth = 120 + ipFileSuffixesToIgnore = "sh,conf" +) + +type Annotated struct { + Client *retryablehttp.Client + Root string + Paths []string +} +type ProviderClient struct { + session.Session +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.Annotated.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating annotated client") + + if c.Logger == nil { + return nil, errors.New("logger not set") + } + + if c.Stats == nil { + return nil, errors.New("stats not set") + } + + if c.Cache == nil { + return nil, errors.New("cache not set") + } + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +func LoadAnnotatedIPPrefixesFromPaths(paths []string, prefixesWithAnnotations PrefixesWithAnnotations) error { + for _, path := range paths { + if err := LoadFilePrefixesWithAnnotationsFromPath(path, prefixesWithAnnotations); err != nil { + return err + } + } + + return nil +} + +type YamlPrefixAnnotationsRecords struct { + Prefixes []string `yaml:"prefixes"` + Annotations []yamlAnnotation `yaml:"annotations"` +} + +func ReadAnnotatedPrefixesFromFile(l *slog.Logger, path string, prefixesWithAnnotations map[netip.Prefix][]annotation) error { + file, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading file: %w", err) + } + + var pwars []YamlPrefixAnnotationsRecords + + err = yaml.Unmarshal(file, &pwars) + if err != nil { + return fmt.Errorf("error unmarshalling yaml: %w", err) + } + + for _, pwar := range pwars { + // parse and repack annotations + annotationSource := path + + annotations := parseAndRepackYAMLAnnotations(nil, annotationSource, pwar.Annotations) + + for _, p := range pwar.Prefixes { + var parsedPrefix netip.Prefix + + parsedPrefix, err = netip.ParsePrefix(p) + if err != nil { + l.Debug("failed to parse", "prefix", parsedPrefix) + + continue + } + + prefixesWithAnnotations[parsedPrefix] = append(prefixesWithAnnotations[parsedPrefix], annotations...) + } + } + + if err != nil { + return fmt.Errorf("error parsing prefixes: %w", err) + } + + return nil +} + +func parseAndRepackYAMLAnnotations(l *slog.Logger, source string, yas []yamlAnnotation) (pyas []annotation) { + for _, ya := range yas { + pDate, err := dateparse.ParseAny(ya.Date, dateparse.PreferMonthFirst(false)) + if err != nil { + l.Debug("failed to parse date,so zeroing", "date", pDate) + } + + pyas = append(pyas, annotation{ + Date: pDate, + Author: ya.Author, + Notes: ya.Notes, + Source: source, + }) + } + + return +} + +func (c *ProviderClient) Initialise() error { + if c.Providers.Annotated.Paths == nil { + return errors.New("no paths provided for annotated provider") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising annotated client") + + // check for combined data in cache + uh := generateURLsHash(c.Providers.Annotated.Paths) + + ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName+"_"+uh) + if err != nil { + return fmt.Errorf("error checking cache for annotated provider data: %w", err) + } + + if ok { + c.Logger.Info("annotated provider data found in cache") + + return nil + } + + // load data from source and store in cache + prefixesWithAnnotations := make(map[netip.Prefix][]annotation) + + err = LoadAnnotatedIPPrefixesFromPaths(c.Providers.Annotated.Paths, prefixesWithAnnotations) + if err != nil { + return fmt.Errorf("loading annotated files: %w", err) + } + + mPWAs, err := json.Marshal(prefixesWithAnnotations) + if err != nil { + return fmt.Errorf("error marshalling annotated prefixes: %w", err) + } + + if err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName + "_" + uh, + Value: mPWAs, + Version: "-", + Created: time.Now(), + }, CacheTTL); err != nil { + return fmt.Errorf("error caching annotated prefixes: %w", err) + } + + return nil +} + +func generateURLsHash(urls []string) string { + sort.Strings(urls) + + s := strings.Join(urls, "") + h := sha256.New() + h.Write([]byte(s)) + + return hex.EncodeToString(h.Sum(nil))[:providers.CacheKeySHALen] +} + +type HostSearchResult map[netip.Prefix][]annotation + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var err error + + var result HostSearchResult + + if err = json.Unmarshal(data, &result); err != nil { + switch { + case errors.Is(err, providers.ErrNoDataFound): + return nil, fmt.Errorf("data not loaded: %w", err) + case errors.Is(err, providers.ErrFailedToFetchData): + return nil, fmt.Errorf("error fetching annotated api response: %w", err) + case errors.Is(err, providers.ErrNoMatchFound): + // reset the error as no longer useful for table creation + return nil, nil + default: + return nil, fmt.Errorf("error loading annotated api response: %w", err) + } + } + + tw := table.NewWriter() + + var rows []table.Row + + for prefix, annotations := range result { + tw.AppendRow(table.Row{"Prefix", dashIfEmpty(prefix.String())}) + + for _, anno := range annotations { + tw.AppendRow(table.Row{"Date", anno.Date}) + tw.AppendRow(table.Row{"Author", anno.Author}) + + if len(anno.Notes) == 0 { + tw.AppendRow(table.Row{"Notes", "-"}) + } else { + for x := range anno.Notes { + if x == 0 { + tw.AppendRow(table.Row{"Notes", anno.Notes[x]}) + continue + } + + tw.AppendRow(table.Row{"", anno.Notes[x]}) + } + } + + tw.AppendRow(table.Row{"Source", anno.Source}) + } + } + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("ANNOTATED | Host: %s", c.Host.String()) + + return &tw, nil +} + +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var err error + + doc, err := c.loadProviderDataFromCache() + if err != nil { + return nil, err + } + + match, err := matchIPToDoc(c.Host, doc) + if err != nil { + return nil, err + } + + c.Logger.Info("annotated match found", "host", c.Host.String()) + + var raw []byte + + raw, err = json.Marshal(match) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + // TODO: remove before release + // if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + // if err = os.WriteFile(fmt.Sprintf("%s/backups/annotated_%s_report.json", session.GetConfigRoot("", session.AppName), + // strings.ReplaceAll(c.Host.String(), ".", "_")), raw, 0o600); err != nil { + // panic(err) + // } + // c.Logger.Info("backed up annotated response", "host", c.Host.String()) + //} + + return raw, nil +} + +func matchIPToDoc(ip netip.Addr, doc map[netip.Prefix][]annotation) (*HostSearchResult, error) { + var result HostSearchResult + + for prefix, annotations := range doc { + if prefix.Contains(ip) { + if result == nil { + result = make(HostSearchResult) + } + + result[prefix] = annotations + } + } + + if result == nil { + return nil, providers.ErrNoMatchFound + } + + return &result, nil +} + +func unmarshalResponse(data []byte) (HostSearchResult, error) { + var res HostSearchResult + + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling annotated response: %w", err) + } + // res.Raw = data + return res, nil +} + +func (c *ProviderClient) loadProviderDataFromCache() (map[netip.Prefix][]annotation, error) { + // load data from cache + uh := generateURLsHash(c.Providers.Annotated.Paths) + + cacheKey := providers.CacheProviderPrefix + ProviderName + "_" + uh + if item, err := cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + if item.Value != nil && len(item.Value) > 0 { + var result map[netip.Prefix][]annotation + + result, err = unmarshalResponse(item.Value) + if err != nil { + return nil, fmt.Errorf("error unmarshalling cached annotated response: %w", err) + } + + c.Logger.Info("annotated response found in cache", "host", c.Host.String()) + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return result, nil + } + } + + return nil, nil +} + +type Repository struct { + URL string `toml:"url"` + GitHubUser string `toml:"github_user"` + GitHubToken string `toml:"github_token"` + Paths []string `toml:"paths"` + Patterns []string `toml:"patterns"` +} + +func getValidFilePathsFromDir(l *slog.Logger, dir string) (paths []os.DirEntry) { + files, err := os.ReadDir(dir) + if err != nil { + l.Warn("failed to read", "dir", dir, "error", err.Error()) + } + + suffixesToIgnore := strings.Split(ipFileSuffixesToIgnore, ",") + + for _, file := range files { + if !file.IsDir() { + if slices.Contains(suffixesToIgnore, filepath.Ext(file.Name())) { + continue + } + + paths = append(paths, file) + } + } + + return +} + +func LoadFilePrefixesWithAnnotationsFromPath(path string, prefixesWithAnnotations map[netip.Prefix][]annotation) error { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return err // nolint: wrapcheck + } + + path, err = filepath.Abs(path) + if err != nil { + return fmt.Errorf("error getting absolute path: %w", err) + } + + var fileCount int64 + + var fileNames []string + + pathIsDir := info.IsDir() + if pathIsDir { + // if directory, then retrieve all dirEntries within + dirEntries := getValidFilePathsFromDir(nil, path) + for _, file := range dirEntries { + // only read up to one level deep + if !file.IsDir() { + fileNames = append(fileNames, file.Name()) + } + } + } else { + fileNames = []string{info.Name()} + } + + for _, fileName := range fileNames { + // set the entry to read to the path + fPath := path + // prefix with path if it was a directory + if pathIsDir { + fPath = filepath.Join(path, fileName) + } + + // Get annotations from entry + err = ReadAnnotatedPrefixesFromFile(nil, fPath, prefixesWithAnnotations) + if err != nil { + return err + } + + fileCount++ + } + + return err +} + +type PrefixesWithAnnotations map[netip.Prefix][]annotation + +type VersionedAnnotatedDoc struct { + LastFetchedFromSource time.Time + LastFetchededFromDB time.Time + Doc PrefixesWithAnnotations +} + +type annotation struct { + Date time.Time `yaml:"date"` + Author string `yaml:"author"` + Notes []string `yaml:"notes"` + Source string `yaml:"source"` +} + +type yamlAnnotation struct { + Date string `yaml:"date"` + Author string `yaml:"author"` + Notes []string `yaml:"notes"` + Source string `yaml:"source"` +} + +func dashIfEmpty(value interface{}) string { + switch v := value.(type) { + case string: + if len(v) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(*v) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} diff --git a/providers/annotated/annotated_test.go b/providers/annotated/annotated_test.go new file mode 100644 index 0000000..582af6e --- /dev/null +++ b/providers/annotated/annotated_test.go @@ -0,0 +1,105 @@ +package annotated + +import ( + "fmt" + "log/slog" + "net/netip" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" + "github.com/stretchr/testify/require" +) + +func TestReadAnnotatedPrefixesFromFile(t *testing.T) { + prefixesWithAnnotations := make(map[netip.Prefix][]annotation) + require.NoError(t, ReadAnnotatedPrefixesFromFile(nil, filepath.Join("testdata", "small", "small.yml"), prefixesWithAnnotations)) + require.Len(t, prefixesWithAnnotations, 3) + require.Equal(t, time.Date(2024, time.April, 19, 18, 58, 0, 0, time.UTC), prefixesWithAnnotations[netip.MustParsePrefix("8.8.8.8/32")][0].Date) + require.Equal(t, time.Date(2024, time.April, 19, 19, 0, 0, 0, time.UTC), prefixesWithAnnotations[netip.MustParsePrefix("9.9.9.0/24")][0].Date) +} + +func TestLoadFilePrefixesWithAnnotationsFromPath(t *testing.T) { + prefixesWithAnnotations := make(map[netip.Prefix][]annotation) + require.NoError(t, LoadAnnotatedIPPrefixesFromPaths([]string{filepath.Join("testdata", "small")}, prefixesWithAnnotations)) + require.Len(t, prefixesWithAnnotations, 3) + require.Equal(t, time.Date(2024, time.April, 19, 18, 58, 0, 0, time.UTC), prefixesWithAnnotations[netip.MustParsePrefix("8.8.8.8/32")][0].Date) + require.Equal(t, time.Date(2024, time.April, 19, 19, 0, 0, 0, time.UTC), prefixesWithAnnotations[netip.MustParsePrefix("9.9.9.0/24")][0].Date) +} + +func TestInitialise(t *testing.T) { + c, err := initialiseSetup(t.TempDir()) + require.NoError(t, err) + require.NoError(t, err) + + config := c.GetConfig() + + uh := generateURLsHash(config.Providers.Annotated.Paths) + + ok, err := cache.CheckExists(c.GetConfig().Logger, config.Cache, providers.CacheProviderPrefix+ProviderName+"_"+uh) + require.NoError(t, err) + require.True(t, ok) + + res, err := c.FindHost() + require.NoError(t, err) + require.NotNil(t, res) + require.NotEqual(t, "null", string(res)) +} + +func initialiseSetup(homeDir string) (providers.ProviderClient, error) { + lg := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + db, err := cache.Create(lg, filepath.Join(homeDir, ".config", "ipscout")) + if err != nil { + return nil, fmt.Errorf("error creating cache: %w", err) + } + + c, err := NewProviderClient(session.Session{ + Logger: slog.New(slog.NewTextHandler(os.Stdout, nil)), + Stats: session.CreateStats(), + Target: nil, + Cache: db, + }) + if err != nil { + return nil, err + } + + config := c.GetConfig() + if err = config.Validate(); err != nil { + return nil, fmt.Errorf("error validating session: %w", err) + } + + // set paths + config.Providers.Annotated.Paths = []string{filepath.Join("testdata", "small")} + + config.Host = netip.MustParseAddr("9.9.9.9") + + if err = c.Initialise(); err != nil { + return nil, fmt.Errorf("error initialising client: %w", err) + } + + return c, nil +} + +func TestLoadProviderDataFromCache(t *testing.T) { + c, err := initialiseSetup(t.TempDir()) + require.NoError(t, err) + + config := c.GetConfig() + + uh := generateURLsHash(config.Providers.Annotated.Paths) + + ok, err := cache.CheckExists(config.Logger, config.Cache, providers.CacheProviderPrefix+ProviderName+"_"+uh) + require.NoError(t, err) + require.True(t, ok) + + ac := c.(*ProviderClient) + + pData, err := ac.loadProviderDataFromCache() + require.NoError(t, err) + require.NotNil(t, pData) +} diff --git a/providers/annotated/testdata/small/small.yml b/providers/annotated/testdata/small/small.yml new file mode 100644 index 0000000..4bcc4bd --- /dev/null +++ b/providers/annotated/testdata/small/small.yml @@ -0,0 +1,21 @@ +--- +- prefixes: ["8.8.8.8/32"] + annotations: + - date: 2024/04/19 18:58 + author: jon hadfield + notes: + - My First Annotation + - My Second Annotation +- prefixes: ["9.9.9.0/24"] + annotations: + - date: 2024/04/19 19:00 + author: jon hadfield + notes: + - My Third Annotation + - My Fourth Annotation +- prefixes: ["179.43.180.108/32"] + annotations: + - date: 2024/04/20 18:00 + author: jon hadfield + notes: + - My fifth Annotation \ No newline at end of file diff --git a/providers/aws/aws.go b/providers/aws/aws.go new file mode 100644 index 0000000..8884552 --- /dev/null +++ b/providers/aws/aws.go @@ -0,0 +1,395 @@ +package aws + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ip-fetcher/providers/aws" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "aws" + DocTTL = time.Duration(24 * time.Hour) + MaxColumnWidth = 120 +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +type ProviderClient struct { + session.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating aws client") + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.AWS.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func unmarshalProviderData(rBody []byte) (*aws.Doc, error) { + var res *aws.Doc + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling aws provider doc: %w", err) + } + + return res, nil +} + +func (c *ProviderClient) loadProviderData() error { + awsClient := aws.New() + awsClient.Client = c.HTTPClient + + if c.Providers.AWS.URL != "" { + awsClient.DownloadURL = c.Providers.AWS.URL + c.Logger.Debug("overriding aws source", "url", aws.DownloadURL) + } + + doc, etag, err := awsClient.Fetch() + if err != nil { + return fmt.Errorf("error fetching aws provider data: %w", err) + } + + data, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("error marshalling aws provider doc: %w", err) + } + + err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName, + Value: data, + Version: etag, + Created: time.Now(), + }, DocTTL) + if err != nil { + return fmt.Errorf("error caching aws provider data: %w", err) + } + + return nil +} + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising aws client") + + ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName) + if err != nil { + return fmt.Errorf("error checking cache for aws provider data: %w", err) + } + + if ok { + c.Logger.Info("aws provider data found in cache") + + return nil + } + + // load data from source and store in cache + err = c.loadProviderData() + if err != nil { + return err + } + + return nil +} + +func loadTestData() ([]byte, error) { + tdf, err := loadResultsFile("providers/aws/testdata/aws_18_164_52_75_report.json") + if err != nil { + return nil, err + } + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + return out, nil +} + +func (c *ProviderClient) loadProviderDataFromCache() (*aws.Doc, error) { + cacheKey := providers.CacheProviderPrefix + ProviderName + + var doc *aws.Doc + + if item, err := cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + var uErr error + + doc, uErr = unmarshalProviderData(item.Value) + if uErr != nil { + defer func() { + _ = cache.Delete(c.Logger, c.Cache, cacheKey) + }() + + return nil, fmt.Errorf("error unmarshalling cached aws provider doc: %w", err) + } + } else if err != nil { + return nil, fmt.Errorf("error reading aws provider cache: %w", err) + } + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return doc, nil +} + +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var out []byte + + var err error + + // load test results data + if c.UseTestData { + var loadErr error + + out, loadErr = loadTestData() + if loadErr != nil { + return nil, loadErr + } + + c.Logger.Info("aws match returned from test data", "host", c.Host.String()) + + return out, nil + } + + doc, err := c.loadProviderDataFromCache() + if err != nil { + return nil, err + } + + match, err := matchIPToDoc(c.Host, doc) + if err != nil { + return nil, err + } + + c.Logger.Info("aws match found", "host", c.Host.String()) + + match.SyncToken = doc.SyncToken + + match.CreateDate, err = time.Parse("2006-01-02-15-04-05", doc.CreateDate) + if err != nil { + return nil, fmt.Errorf("error parsing create date: %w", err) + } + + var raw []byte + + raw, err = json.Marshal(match) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + if err = os.WriteFile(fmt.Sprintf("%s/backups/aws_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), raw, 0o600); err != nil { + panic(err) + } + + c.Logger.Info("backed up aws response", "host", c.Host.String()) + } + + return raw, nil +} + +func matchIPToDoc(host netip.Addr, doc *aws.Doc) (*HostSearchResult, error) { + var match *HostSearchResult + + if host.Is4() { + return matchIPv4ToDoc(host, doc) + } + + if host.Is6() { + return matchIPv6ToDoc(host, doc) + } + + return match, nil +} + +func matchIPv6ToDoc(host netip.Addr, doc *aws.Doc) (*HostSearchResult, error) { + var match *HostSearchResult + + for _, prefix := range doc.IPv6Prefixes { + if prefix.IPv6Prefix.Contains(host) { + match = &HostSearchResult{ + Prefix: aws.Prefix{ + IPPrefix: prefix.IPv6Prefix, + Region: prefix.Region, + Service: prefix.Service, + }, + } + + return match, nil + } + } + + return nil, fmt.Errorf("aws: %w", providers.ErrNoMatchFound) +} + +func matchIPv4ToDoc(host netip.Addr, doc *aws.Doc) (*HostSearchResult, error) { + var match *HostSearchResult + + for _, prefix := range doc.Prefixes { + if prefix.IPPrefix.Contains(host) { + match = &HostSearchResult{ + Prefix: aws.Prefix{ + IPPrefix: prefix.IPPrefix, + Region: prefix.Region, + Service: prefix.Service, + }, + } + + return match, nil + } + } + + return nil, fmt.Errorf("aws: %w", providers.ErrNoMatchFound) +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var result HostSearchResult + + if err := json.Unmarshal(data, &result); err != nil { + switch { + case errors.Is(err, providers.ErrNoDataFound): + return nil, fmt.Errorf("data not loaded: %w", err) + case errors.Is(err, providers.ErrFailedToFetchData): + return nil, fmt.Errorf("error fetching aws data: %w", err) + case errors.Is(err, providers.ErrNoMatchFound): + // reset the error as no longer useful for table creation + return nil, nil + default: + return nil, fmt.Errorf("error loading aws api response: %w", err) + } + } + + tw := table.NewWriter() + + var rows []table.Row + + tw.AppendRow(table.Row{"Prefix", dashIfEmpty(result.Prefix.IPPrefix.String())}) + tw.AppendRow(table.Row{"Service", dashIfEmpty(result.Prefix.Service)}) + tw.AppendRow(table.Row{"Region", dashIfEmpty(result.Prefix.Region)}) + + if !result.CreateDate.IsZero() { + tw.AppendRow(table.Row{"Source Update", dashIfEmpty(result.CreateDate.String())}) + } + + if result.SyncToken != "" { + tw.AppendRow(table.Row{"Sync Token", dashIfEmpty(result.SyncToken)}) + } + + if result.ETag != "" { + tw.AppendRow(table.Row{"Version", dashIfEmpty(result.ETag)}) + } + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("AWS | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("AWS | Host: 18.164.100.99") + } + + return &tw, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding json: %w", err) + } + + return res, nil +} + +type HostSearchResult struct { + Raw []byte + aws.Prefix `json:"prefix"` + aws.IPv6Prefix `json:"ipv6Prefix"` + ETag string `json:"etag"` + SyncToken string `json:"syncToken"` + CreateDate time.Time `json:"createDate"` +} + +func dashIfEmpty(value interface{}) string { + switch v := value.(type) { + case string: + if len(v) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(*v) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} diff --git a/providers/aws/testdata/aws_18_164_52_75_report.json b/providers/aws/testdata/aws_18_164_52_75_report.json new file mode 100644 index 0000000..50d328c --- /dev/null +++ b/providers/aws/testdata/aws_18_164_52_75_report.json @@ -0,0 +1,16 @@ +{ + "Raw": null, + "prefix": { + "ip_prefix": "18.164.0.0/15", + "region": "GLOBAL", + "service": "AMAZON" + }, + "ipv6Prefix": { + "ipv6_prefix": "", + "region": "", + "service": "" + }, + "etag": "", + "syncToken": "", + "createDate": "0001-01-01T00:00:00Z" +} diff --git a/providers/azure/azure.go b/providers/azure/azure.go new file mode 100644 index 0000000..f201a45 --- /dev/null +++ b/providers/azure/azure.go @@ -0,0 +1,359 @@ +package azure + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ip-fetcher/providers/azure" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "azure" + DocTTL = time.Duration(24 * time.Hour) +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +func unmarshalProviderData(rBody []byte) (*azure.Doc, error) { + var res *azure.Doc + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling azure provider doc: %w", err) + } + + return res, nil +} + +type ProviderClient struct { + session.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating azure client") + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.Azure.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +const ( + MaxColumnWidth = 120 +) + +func (c *ProviderClient) loadProviderDataFromSource() error { + azureClient := azure.New() + azureClient.Client = c.HTTPClient + + if c.Providers.Azure.URL != "" { + azureClient.DownloadURL = c.Providers.Azure.URL + c.Logger.Debug("overriding azure source", "url", azureClient.DownloadURL) + } + + doc, etag, err := azureClient.Fetch() + if err != nil { + return fmt.Errorf("%s %w", err.Error(), providers.ErrFailedToFetchData) + } + + c.Logger.Debug("fetched azure data from source", "size", len(doc.Values), "etag", etag) + + data, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("error marshalling azure provider doc: %w", err) + } + + c.Logger.Debug("writing azure provider data to cache", "size", len(data), "etag", etag) + + if err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName, + Value: data, + Version: etag, + Created: time.Now(), + }, DocTTL); err != nil { + return fmt.Errorf("error writing azure provider data to cache: %w", err) + } + + return nil +} + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising azure client") + + ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName) + if err != nil { + return fmt.Errorf("error checking cache for azure provider data: %w", err) + } + + if ok { + c.Logger.Info("azure provider data found in cache") + + return nil + } + + err = c.loadProviderDataFromSource() + if err != nil { + return err + } + + return nil +} + +func loadTestData(c *ProviderClient) ([]byte, error) { + tdf, err := loadResultsFile("providers/azure/testdata/azure_40_126_12_192_report.json") + if err != nil { + return nil, err + } + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + c.Logger.Info("azure match returned from test data", "host", c.Host.String()) + + return out, nil +} + +func (c *ProviderClient) loadProviderDataFromCache() (*azure.Doc, error) { + c.Logger.Info("loading azure provider data from cache") + + cacheKey := providers.CacheProviderPrefix + ProviderName + + var doc *azure.Doc + + if item, err := cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + var uErr error + + doc, uErr = unmarshalProviderData(item.Value) + if uErr != nil { + defer func() { + _ = cache.Delete(c.Logger, c.Cache, cacheKey) + }() + + return nil, fmt.Errorf("error unmarshalling cached azure provider doc: %w", uErr) + } + } else { + return nil, fmt.Errorf("error reading azure provider data from cache: %w", err) + } + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return doc, nil +} + +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var out []byte + + var err error + + // load test results data + if c.UseTestData { + var loadErr error + + out, loadErr = loadTestData(c) + if loadErr != nil { + return nil, loadErr + } + + c.Logger.Info("azure match returned from test data", "host", c.Host.String()) + + return out, nil + } + + doc, err := c.loadProviderDataFromCache() + if err != nil { + return nil, err + } + + match, err := matchIPToDoc(c.Host, doc) + if err != nil { + return nil, err + } + + c.Logger.Info("azure match found", "host", c.Host.String()) + + var raw []byte + + raw, err = json.Marshal(match) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + if err = os.WriteFile(fmt.Sprintf("%s/backups/azure_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), raw, 0o600); err != nil { + panic(err) + } + + c.Logger.Info("backed up azure response", "host", c.Host.String()) + } + + return raw, nil +} + +func matchIPToDoc(host netip.Addr, doc *azure.Doc) (*HostSearchResult, error) { + var match *HostSearchResult + + for _, value := range doc.Values { + props := value.Properties + + for _, prefix := range props.AddressPrefixes { + p, err := netip.ParsePrefix(prefix) + if err != nil { + return nil, fmt.Errorf("error parsing prefix: %w", err) + } + + if p.Contains(host) { + match = &HostSearchResult{ + Raw: nil, + Prefix: p, + ChangeNumber: props.ChangeNumber, + Cloud: doc.Cloud, + Name: value.Name, + ID: value.ID, + Properties: props, + } + + return match, nil + } + } + } + + return nil, fmt.Errorf("azure: %w", providers.ErrNoMatchFound) +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var err error + + var result HostSearchResult + + if err = json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("error unmarshalling azure data: %w", err) + } + + tw := table.NewWriter() + + var rows []table.Row + + tw.AppendRow(table.Row{"Name", dashIfEmpty(result.Name)}) + tw.AppendRow(table.Row{"ID", dashIfEmpty(result.ID)}) + tw.AppendRow(table.Row{"Region", dashIfEmpty(result.Properties.Region)}) + tw.AppendRow(table.Row{"Prefix", dashIfEmpty(result.Prefix)}) + tw.AppendRow(table.Row{"Platform", dashIfEmpty(result.Properties.Platform)}) + tw.AppendRow(table.Row{"Cloud", dashIfEmpty(result.Cloud)}) + tw.AppendRow(table.Row{"System Service", dashIfEmpty(result.Properties.SystemService)}) + tw.AppendRow(table.Row{"Network Features", dashIfEmpty(strings.Join(result.Properties.NetworkFeatures, ","))}) + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("AZURE | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("AZURE | Host: 40.126.12.192") + } + + return &tw, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding file: %w", err) + } + + return res, nil +} + +type HostSearchResult struct { + Raw []byte + Prefix netip.Prefix + ChangeNumber int `json:"changeNumber"` + Cloud string `json:"cloud"` + Name string `json:"name"` + ID string `json:"id"` + Properties azure.Properties `json:"properties"` +} + +func dashIfEmpty(value interface{}) string { + switch v := value.(type) { + case string: + if len(v) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(*v) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} diff --git a/providers/azure/testdata/azure_40_126_12_192_report.json b/providers/azure/testdata/azure_40_126_12_192_report.json new file mode 100644 index 0000000..3a5ef56 --- /dev/null +++ b/providers/azure/testdata/azure_40_126_12_192_report.json @@ -0,0 +1 @@ +{"Raw":null,"Prefix":"40.126.0.0/18","changeNumber":15,"cloud":"Public","name":"AzureActiveDirectory","id":"AzureActiveDirectory","properties":{"changeNumber":15,"region":"","regionId":0,"platform":"Azure","systemService":"AzureAD","addressPrefixes":["4.150.253.96/28","13.64.151.161/32","13.66.141.64/27","13.67.9.224/27","13.69.66.160/27","13.69.229.96/27","13.70.73.32/27","13.71.172.160/27","13.71.195.224/27","13.71.201.64/26","13.73.240.32/27","13.74.104.0/26","13.74.249.156/32","13.75.38.32/27","13.75.105.168/32","13.77.52.160/27","13.78.108.192/27","13.78.172.246/32","13.79.37.247/32","13.86.219.0/27","13.87.16.0/26","13.87.57.160/27","13.87.123.160/27","13.89.174.0/27","20.20.32.0/19","20.36.107.192/27","20.36.115.64/27","20.37.75.96/27","20.40.228.64/28","20.43.120.32/27","20.44.3.160/27","20.44.16.32/27","20.46.10.64/27","20.51.9.80/28","20.51.14.72/31","20.51.16.128/27","20.61.98.160/27","20.61.99.128/28","20.62.58.80/28","20.62.129.0/27","20.62.129.240/28","20.62.134.74/31","20.65.132.96/28","20.66.2.32/27","20.66.3.16/28","20.72.21.64/26","20.88.66.0/27","20.111.78.128/28","20.187.197.32/27","20.187.197.240/28","20.190.128.0/18","20.194.73.0/28","20.195.56.102/32","20.195.57.118/32","20.195.64.192/27","20.195.64.240/28","20.231.128.0/19","23.101.0.70/32","23.101.6.190/32","40.68.160.142/32","40.69.107.160/27","40.71.13.0/27","40.74.101.64/27","40.74.146.192/27","40.78.195.160/27","40.78.203.64/27","40.79.131.128/27","40.79.179.128/27","40.83.144.56/32","40.126.0.0/18","51.140.148.192/27","51.140.208.0/26","51.140.211.192/27","52.138.65.157/32","52.138.68.41/32","52.146.132.96/27","52.146.133.80/28","52.146.137.66/31","52.150.157.0/27","52.157.20.148/32","52.157.20.186/32","52.157.20.205/32","52.159.175.31/32","52.159.175.117/32","52.159.175.121/32","52.161.13.71/32","52.161.13.95/32","52.161.110.169/32","52.162.110.96/27","52.169.125.119/32","52.169.218.0/32","52.174.189.149/32","52.175.18.134/32","52.178.27.112/32","52.179.122.218/32","52.179.126.223/32","52.180.177.87/32","52.180.179.108/32","52.180.181.61/32","52.180.183.8/32","52.187.19.1/32","52.187.113.48/32","52.187.117.83/32","52.187.120.237/32","52.225.184.198/32","52.225.188.89/32","52.226.169.40/32","52.226.169.45/32","52.226.169.53/32","52.231.19.128/27","52.231.147.192/27","52.249.207.8/32","52.249.207.23/32","52.249.207.27/32","65.52.251.96/27","98.66.133.128/28","104.40.84.19/32","104.40.87.209/32","104.40.156.18/32","104.40.168.0/26","104.41.159.212/32","104.45.138.161/32","104.46.178.128/27","104.211.147.160/27","172.172.255.96/28","191.233.204.160/27","2603:1006:2000::/48","2603:1007:200::/48","2603:1016:1400::/48","2603:1017::/48","2603:1026:3000::/48","2603:1027:1::/48","2603:1030:107:2::/120","2603:1030:107:2::100/121","2603:1036:3000::/48","2603:1037:1::/48","2603:1046:2000::/48","2603:1047:1::/48","2603:1056:2000::/48","2603:1057:2::/48"],"networkFeatures":["API","NSG","UDR","FW","VSE"]}} \ No newline at end of file diff --git a/providers/criminalip/criminalip.go b/providers/criminalip/criminalip.go new file mode 100644 index 0000000..a646a93 --- /dev/null +++ b/providers/criminalip/criminalip.go @@ -0,0 +1,792 @@ +package criminalip + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/netip" + "net/url" + "os" + "regexp" + "strings" + "time" + + "github.com/fatih/color" + "github.com/hashicorp/go-retryablehttp" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "criminalip" + APIURL = "https://api.criminalip.io" + HostIPPath = "/v1/asset/ip/report" + IndentPipeHyphens = " |-----" + ResultTTL = time.Duration(24 * time.Hour) +) + +type Client struct { + session.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating criminalip client") + + tc := &Client{ + Session: c, + } + + return tc, nil +} + +func (c *Client) GetConfig() *session.Session { + return &c.Session +} + +func (c *Client) Enabled() bool { + return c.Session.Providers.CriminalIP.Enabled +} + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +func loadAPIResponse(ctx context.Context, conf *session.Session, apiKey string) (res *HostSearchResult, err error) { + urlPath, err := url.JoinPath(APIURL, HostIPPath) + if err != nil { + return nil, fmt.Errorf("error joining criminal ip api url: %w", err) + } + + sURL, err := url.Parse(urlPath) + if err != nil { + panic(err) + } + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + q := sURL.Query() + q.Add("ip", conf.Host.String()) + sURL.RawQuery = q.Encode() + + req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodGet, sURL.String(), nil) + if err != nil { + panic(err) + } + + conf.HTTPClient.HTTPClient.Timeout = 30 * time.Second + + req.Header.Add("x-api-key", apiKey) + + resp, err := conf.HTTPClient.Do(req) + if err != nil { + panic(err) + } + + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("criminal ip: %w", providers.ErrNoMatchFound) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("criminal api request failed: %s", resp.Status) + } + + // read response body + rBody, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) + } + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + if err = os.WriteFile(fmt.Sprintf("%s/backups/criminalip_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(conf.Host.String(), ".", "_")), rBody, 0o600); err != nil { + panic(err) + } + } + + defer resp.Body.Close() + + res, err = unmarshalResponse(rBody) + if err != nil { + return nil, err + } + + // check if response contains an error, despite a 200 status code + if res.Status == http.StatusForbidden { + return nil, fmt.Errorf("criminal ip api error: %s: %w", res.Message, providers.ErrForbiddenByProvider) + } + + res.Raw = rBody + + return res, nil +} + +func unmarshalResponse(rBody []byte) (*HostSearchResult, error) { + var res *HostSearchResult + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling criminal ip response: %w", err) + } + + return res, nil +} + +func loadTestData(c *Client) ([]byte, error) { + tdf, err := loadResultsFile("providers/criminalip/testdata/criminalip_9_9_9_9_report.json") + if err != nil { + return nil, err + } + + c.Logger.Info("criminalip match returned from test data", "host", "9.9.9.9") + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + return out, nil +} + +func fetchData(client session.Session) (*HostSearchResult, error) { + cacheKey := fmt.Sprintf("criminalip_%s_report.json", strings.ReplaceAll(client.Host.String(), ".", "_")) + if item, err := cache.Read(client.Logger, client.Cache, cacheKey); err == nil { + if item != nil { + result, uErr := unmarshalResponse(item.Value) + if uErr != nil { + return nil, uErr + } + + client.Logger.Info("criminal ip response found in cache", "host", client.Host.String()) + + result.Raw = item.Value + + client.Stats.Mu.Lock() + client.Stats.FindHostUsedCache[ProviderName] = true + client.Stats.Mu.Unlock() + + return result, nil + } + } + + result, err := loadAPIResponse(context.Background(), &client, client.Providers.CriminalIP.APIKey) + if err != nil { + return nil, fmt.Errorf("error loading criminal ip api response: %w", err) + } + + if err = cache.UpsertWithTTL(client.Logger, client.Cache, cache.Item{ + AppVersion: client.App.Version, + Key: cacheKey, + Value: result.Raw, + Created: time.Now(), + }, ResultTTL); err != nil { + return nil, fmt.Errorf("error caching criminal ip response: %w", err) + } + + return result, nil +} + +const ( + MaxColumnWidth = 120 +) + +func tidyBanner(banner string) string { + // remove empty lines using regex match + var lines []string + + r := regexp.MustCompile(`^(\s*$|$)`) + for x, line := range strings.Split(banner, "\n") { + if r.MatchString(line) { + continue + } + + if x > 0 { + line = strings.TrimSpace(line) + line = fmt.Sprintf("%s %s", strings.Repeat(" ", len(IndentPipeHyphens)+1), line) + } + + lines = append(lines, line) + } + + return strings.Join(lines, "\n") +} + +func getDomains(domain HostSearchResultDomain) []string { + var domains []string + + for _, d := range domain.Data { + domains = append(domains, d.Domain) + } + + return domains +} + +func (c *Client) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising criminalip client") + + return nil +} + +func (c *Client) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + if c.UseTestData { + return loadTestData(c) + } + + if c.Host.Is6() { + return nil, fmt.Errorf("ipv6 not supported by criminalip: %w", providers.ErrNoMatchFound) + } + + result, err := fetchData(c.Session) + if err != nil { + return nil, fmt.Errorf("error loading criminalip api response: %w", err) + } + + return result.Raw, nil +} + +type GeneratePortDataForTableInput struct { +} + +type GeneratePortDataForTableOutput struct { + entries []WrappedPortDataEntry + skips int + matches int +} + +func (c *Client) GenPortDataForTable(in []PortDataEntry) (GeneratePortDataForTableOutput, error) { + var err error + + var out GeneratePortDataForTableOutput + + out.entries = make([]WrappedPortDataEntry, len(in)) + + for _, entry := range in { + var ageMatch, netMatch bool + ageMatch, netMatch, err = providers.PortMatchFilter(providers.PortMatchFilterInput{ + IncomingPort: fmt.Sprintf("%d/%s", entry.OpenPortNo, entry.Socket), + MatchPorts: c.Config.Global.Ports, + ConfirmedDate: entry.ConfirmedTime, + ConfirmedDateFormat: time.DateTime, + MaxAge: c.Config.Global.MaxAge, + }) + + if err != nil { + return GeneratePortDataForTableOutput{}, fmt.Errorf("error checking port match filter: %w", err) + } + + wrappedEntry := WrappedPortDataEntry{ + AgeMatch: ageMatch, + NetworkMatch: netMatch, + PortDataEntry: entry, + } + + out.entries = append(out.entries, wrappedEntry) + + if ageMatch && netMatch { + out.matches++ + } else { + out.skips++ + } + } + + return out, nil +} + +func (c *Client) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := unmarshalResponse(data) + if err != nil { + return nil, fmt.Errorf("error unmarshalling criminalip api response: %w", err) + } + + if result == nil { + return nil, nil + } + + tw := table.NewWriter() + // tw.SetStyle(myInnerStyle) + var rows []table.Row + + if result.Whois.Count > 0 { + for _, whois := range result.Whois.Data { + tw.AppendRow(table.Row{"WHOIS", providers.DashIfEmpty(whois.ConfirmedTime)}) + tw.AppendRow(table.Row{" - Org", providers.DashIfEmpty(whois.OrgName)}) + tw.AppendRow(table.Row{" - Country", providers.DashIfEmpty(strings.ToUpper(whois.OrgCountryCode))}) + tw.AppendRow(table.Row{" - Region", providers.DashIfEmpty(whois.Region)}) + tw.AppendRow(table.Row{" - City", providers.DashIfEmpty(whois.City)}) + } + } + + if domains := getDomains(result.Domain); domains != nil { + tw.AppendRow(table.Row{"Domains", strings.Join(getDomains(result.Domain), ", ")}) + } + + tw.AppendRow(table.Row{"Score Inbound", result.Score.Inbound}) + tw.AppendRow(table.Row{"Score Outbound", result.Score.Outbound}) + + portDataForTable, err := c.GenPortDataForTable(result.Port.Data) + if err != nil { + return nil, fmt.Errorf("error generating port data for table: %w", err) + } + + if portDataForTable.skips > 0 { + tw.AppendRow(table.Row{"Ports", fmt.Sprintf("%d (%d filtered)", len(result.Port.Data), portDataForTable.skips)}) + } else { + tw.AppendRow(table.Row{"Ports", len(result.Port.Data)}) + } + + var portsDisplayed int + + for x, port := range portDataForTable.entries { + if !port.AgeMatch || !port.NetworkMatch { + continue + } + + tw.AppendRow(table.Row{"", color.CyanString("%d/%s", port.OpenPortNo, port.Socket)}) + tw.AppendRow(table.Row{"", fmt.Sprintf("%s Protocol: %s", IndentPipeHyphens, providers.DashIfEmpty(port.Protocol))}) + tw.AppendRow(table.Row{"", fmt.Sprintf("%s Confirmed Time: %s", IndentPipeHyphens, port.ConfirmedTime)}) + + // vary output based on protocol + switch strings.ToLower(port.Protocol) { + case "https": + tw.AppendRow(table.Row{"", fmt.Sprintf("%s SDN Common Name: %s", IndentPipeHyphens, port.SdnCommonName)}) + tw.AppendRow(table.Row{"", fmt.Sprintf("%s DNS Names: %s", IndentPipeHyphens, providers.PreProcessValueOutput(&c.Session, port.DNSNames))}) + case "dns": + tw.AppendRow(table.Row{"", fmt.Sprintf("%s App Name (Version): %s (%s)", IndentPipeHyphens, port.AppName, port.AppVersion)}) + tw.AppendRow(table.Row{"", fmt.Sprintf("%s Banner: %s", + IndentPipeHyphens, tidyBanner(providers.PreProcessValueOutput(&c.Session, port.Banner)))}) + default: + tw.AppendRow(table.Row{"", fmt.Sprintf("%s App Name (Version): %s (%s)", IndentPipeHyphens, port.AppName, port.AppVersion)}) + tw.AppendRow(table.Row{"", fmt.Sprintf("%s Banner: %s", + IndentPipeHyphens, tidyBanner(providers.PreProcessValueOutput(&c.Session, port.Banner)))}) + } + + // always include if detected as vulnerability + tw.AppendRow(table.Row{"", fmt.Sprintf("%s Is Vulnerability: %t", IndentPipeHyphens, port.IsVulnerability)}) + + if x+1 < len(result.Port.Data) { + // add a blank row between ports + tw.AppendRow(table.Row{"", ""}) + } + + portsDisplayed++ + + if portsDisplayed == c.Config.Global.MaxReports { + tw.AppendRow(table.Row{"", color.YellowString("--- Max reports reached ---")}) + + break + } + } + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: true, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("CRIMINAL IP | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("CRIMINAL IP | Host: %s", result.IP) + } + + return &tw, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding criminalip data: %w", err) + } + + return res, nil +} + +type HostSearchResultData struct { + Hash int `json:"hash"` + Opts struct{} `json:"opts,omitempty"` + Timestamp string `json:"timestamp"` + Isp string `json:"isp"` + Data string `json:"data"` + CriminalIP struct { + Region string `json:"region"` + Module string `json:"module"` + Ptr bool `json:"ptr"` + Options struct{} `json:"options"` + ID string `json:"id"` + Crawler string `json:"crawler"` + } `json:"_criminalip,omitempty"` + Port int `json:"port"` + Hostnames []string `json:"hostnames"` + Location struct { + City string `json:"city"` + RegionCode string `json:"region_code"` + AreaCode any `json:"area_code"` + Longitude float64 `json:"longitude"` + CountryName string `json:"country_name"` + CountryCode string `json:"country_code"` + Latitude float64 `json:"latitude"` + } `json:"location"` + DNS struct { + ResolverHostname any `json:"resolver_hostname"` + Recursive bool `json:"recursive"` + ResolverID any `json:"resolver_id"` + Software any `json:"software"` + } `json:"dns,omitempty"` + HTTP struct { + Status int `json:"status"` + RobotsHash string `json:"robots_hash"` + Redirects []struct { + Host string `json:"host"` + Data string `json:"data"` + Location string `json:"location"` + } + SecurityTxt string `json:"security_txt"` + Title string `json:"title"` + SitemapHash string `json:"sitemap_hash"` + HTMLHash int `json:"html_hash"` + Robots string `json:"robots"` + Favicon struct { + Hash int `json:"hash"` + Data string `json:"data"` + Location string `json:"location"` + } `json:"favicon"` + HeadersHash int `json:"headers_hash"` + Host string `json:"host"` + HTML string `json:"html"` + Location string `json:"location"` + Components struct{} `json:"components"` + Server string `json:"server"` + Sitemap string `json:"sitemap"` + SecurityTxtHash string `json:"securitytxt_hash"` + } `json:"http,omitempty"` + IP string `json:"ip"` + Domains []string `json:"domains"` + Org string `json:"org"` + Os any `json:"os"` + Asn string `json:"asn"` + Transport string `json:"transport"` + IPStr string `json:"ip_str"` + Ssl struct { + ChainSha256 []string `json:"chain_sha256"` + Jarm string `json:"jarm"` + Chain []string `json:"chain"` + Dhparams any `json:"dhparams"` + Versions []string `json:"versions"` + AcceptableCas []any `json:"acceptable_cas"` + Tlsext []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"tlsext"` + Ja3S string `json:"ja3s"` + Cert struct { + SigAlg string `json:"sig_alg"` + Issued string `json:"issued"` + Expires string `json:"expires"` + Expired bool `json:"expired"` + Version int `json:"version"` + Extensions []struct { + Critical bool `json:"critical,omitempty"` + Data string `json:"data"` + Name string `json:"name"` + } `json:"extensions"` + Fingerprint struct { + Sha256 string `json:"sha256"` + Sha1 string `json:"sha1"` + } `json:"fingerprint"` + Serial json.RawMessage `json:"serial"` + Subject struct { + Cn string `json:"CN"` + } `json:"subject"` + Pubkey struct { + Type string `json:"type"` + Bits int `json:"bits"` + } `json:"pubkey"` + Issuer struct { + C string `json:"C"` + Cn string `json:"CN"` + O string `json:"O"` + } `json:"issuer"` + } `json:"cert"` + Cipher struct { + Version string `json:"version"` + Bits int `json:"bits"` + Name string `json:"name"` + } `json:"cipher"` + Trust struct { + Revoked bool `json:"revoked"` + Browser any `json:"browser"` + } `json:"trust"` + HandshakeStates []string `json:"handshake_states"` + Alpn []any `json:"alpn"` + Ocsp struct{} `json:"ocsp"` + } `json:"ssl,omitempty"` +} + +type HostSearchResultDomain struct { + Count int `json:"count"` + Data []struct { + Domain string `json:"domain"` + IPType string `json:"ip_type"` + Registrar string `json:"registrar"` + CreateDate string `json:"create_date"` + ConfirmedTime string `json:"confirmed_time"` + Email string `json:"email"` + } `json:"data"` +} + +type WrappedPortDataEntry struct { + AgeMatch bool + NetworkMatch bool + PortDataEntry +} + +type PortDataEntry struct { + AppName string `json:"app_name"` + ConfirmedTime string `json:"confirmed_time"` + Banner string `json:"banner"` + AppVersion string `json:"app_version"` + OpenPortNo int `json:"open_port_no"` + PortStatus string `json:"port_status"` + Protocol string `json:"protocol"` + Socket string `json:"socket"` + Tags []string `json:"tags"` + DNSNames string `json:"dns_names"` + SdnCommonName string `json:"sdn_common_name"` + JarmHash string `json:"jarm_hash"` + SslInfoRaw string `json:"ssl_info_raw"` + Technologies []struct { + TechName string `json:"tech_name"` + TechVersion string `json:"tech_version"` + TechLogoURL string `json:"tech_logo_url"` + } `json:"technologies"` + IsVulnerability bool `json:"is_vulnerability"` +} + +type HostSearchResult struct { + Raw []byte + IP string `json:"ip"` + Issues struct { + IsVpn bool `json:"is_vpn"` + IsCloud bool `json:"is_cloud"` + IsTor bool `json:"is_tor"` + IsProxy bool `json:"is_proxy"` + IsHosting bool `json:"is_hosting"` + IsMobile bool `json:"is_mobile"` + IsDarkweb bool `json:"is_darkweb"` + IsScanner bool `json:"is_scanner"` + IsSnort bool `json:"is_snort"` + IsAnonymousVpn bool `json:"is_anonymous_vpn"` + } `json:"issues"` + Score struct { + Inbound string `json:"inbound"` + Outbound string `json:"outbound"` + } `json:"score"` + UserSearchCount int `json:"user_search_count"` + ProtectedIP struct { + Count int `json:"count"` + Data []struct { + IPAddress string `json:"ip_address"` + ConfirmedTime string `json:"confirmed_time"` + } `json:"data"` + } `json:"protected_ip"` + Domain HostSearchResultDomain `json:"domain"` + Whois struct { + Count int `json:"count"` + Data []struct { + AsName string `json:"as_name"` + AsNo int `json:"as_no"` + City string `json:"city"` + Region string `json:"region"` + OrgName string `json:"org_name"` + PostalCode string `json:"postal_code"` + Longitude float64 `json:"longitude"` + Latitude float64 `json:"latitude"` + OrgCountryCode string `json:"org_country_code"` + ConfirmedTime string `json:"confirmed_time"` + } `json:"data"` + } `json:"whois"` + Hostname struct { + Count int `json:"count"` + Data []struct { + DomainNameRep string `json:"domain_name_rep"` + DomainNameFull string `json:"domain_name_full"` + ConfirmedTime string `json:"confirmed_time"` + } `json:"data"` + } `json:"hostname"` + IDs struct { + Count int `json:"count"` + Data []struct { + Classification string `json:"classification"` + URL string `json:"url"` + Message string `json:"message"` + ConfirmedTime string `json:"confirmed_time"` + SourceSystem string `json:"source_system"` + } `json:"data"` + } `json:"ids"` + Vpn struct { + Count int `json:"count"` + Data []struct { + VpnName string `json:"vpn_name"` + VpnURL string `json:"vpn_url"` + VpnSourceURL string `json:"vpn_source_url"` + SocketType string `json:"socket_type"` + ConfirmedTime string `json:"confirmed_time"` + } `json:"data"` + } `json:"vpn"` + AnonymousVpn struct { + Count int `json:"count"` + Data []struct { + VpnName string `json:"vpn_name"` + VpnURL string `json:"vpn_url"` + VpnSourceURL string `json:"vpn_source_url"` + SocketType string `json:"socket_type"` + ConfirmedTime string `json:"confirmed_time"` + } `json:"data"` + } `json:"anonymous_vpn"` + Webcam struct { + Count int `json:"count"` + Data []struct { + ImagePath string `json:"image_path"` + CamURL string `json:"cam_url"` + Country string `json:"country"` + City string `json:"city"` + OpenPortNo int `json:"open_port_no"` + Manufacturer string `json:"manufacturer"` + ConfirmedTime string `json:"confirmed_time"` + } `json:"data"` + } `json:"webcam"` + Honeypot struct { + Count int `json:"count"` + Data []struct { + IPAddress string `json:"ip_address"` + LogDate string `json:"log_date"` + DstPort int `json:"dst_port"` + Message string `json:"message"` + UserAgent string `json:"user_agent"` + ProtocolType string `json:"protocol_type"` + ConfirmedTime string `json:"confirmed_time"` + } `json:"data"` + } `json:"honeypot"` + IPCategory struct { + Count int `json:"count"` + Data []struct { + DetectSource string `json:"detect_source"` + Type string `json:"type"` + DetectInfo struct{} `json:"detect_info,omitempty"` + ConfirmedTime string `json:"confirmed_time"` + DetectInfo0 struct { // nolint:govet + Md5 string `json:"md5"` + Domain string `json:"domain"` + } `json:"detect_info,omitempty"` + } `json:"data"` + } `json:"ip_category"` + Port struct { + Count int `json:"count"` + Data []PortDataEntry `json:"data"` + } `json:"port"` + Vulnerability struct { + Count int `json:"count"` + Data []struct { + CveID string `json:"cve_id"` + CveDescription string `json:"cve_description"` + Cvssv2Vector string `json:"cvssv2_vector"` + Cvssv2Score float64 `json:"cvssv2_score"` + Cvssv3Vector string `json:"cvssv3_vector"` + Cvssv3Score float64 `json:"cvssv3_score"` + ListCwe []struct { + CveID string `json:"cve_id"` + CweID int `json:"cwe_id"` + CweName string `json:"cwe_name"` + CweDescription string `json:"cwe_description"` + } `json:"list_cwe"` + ListEdb []struct { + CveID string `json:"cve_id"` + EdbID int `json:"edb_id"` + Type string `json:"type"` + Platform string `json:"platform"` + VerifyCode int `json:"verify_code"` + Title string `json:"title"` + ConfirmedTime string `json:"confirmed_time"` + } `json:"list_edb"` + AppName string `json:"app_name"` + AppVersion string `json:"app_version"` + OpenPortNoList struct { + TCP []int `json:"TCP"` + UDP []any `json:"UDP"` + } `json:"open_port_no_list"` + HaveMorePorts bool `json:"have_more_ports"` + OpenPortNo []struct { + Port int `json:"port"` + Socket string `json:"socket"` + } `json:"open_port_no"` + ListChild []struct { + AppName string `json:"app_name"` + AppVersion string `json:"app_version"` + Vendor string `json:"vendor"` + Type string `json:"type"` + IsVuln string `json:"is_vuln"` + TargetHw string `json:"target_hw"` + TargetSw string `json:"target_sw"` + Update string `json:"update"` + Edition string `json:"edition"` + } `json:"list_child"` + Vendor string `json:"vendor"` + Type string `json:"type"` + IsVuln string `json:"is_vuln"` + TargetHw string `json:"target_hw"` + TargetSw string `json:"target_sw"` + Update string `json:"update"` + Edition string `json:"edition"` + } `json:"data"` + } `json:"vulnerability"` + Mobile struct { + Count int `json:"count"` + Data []struct { + Broadband string `json:"broadband"` + Organization string `json:"organization"` + } `json:"data"` + } `json:"mobile"` + Message string `json:"message"` + Status int `json:"status"` +} diff --git a/providers/criminalip/testdata/criminalip_1_1_1_1_report.json b/providers/criminalip/testdata/criminalip_1_1_1_1_report.json new file mode 100644 index 0000000..985f431 --- /dev/null +++ b/providers/criminalip/testdata/criminalip_1_1_1_1_report.json @@ -0,0 +1,264 @@ +{ + "ip": "1.1.1.1", + "issues": { + "is_vpn": false, + "is_cloud": false, + "is_tor": false, + "is_proxy": false, + "is_hosting": true, + "is_mobile": false, + "is_darkweb": false, + "is_scanner": false, + "is_snort": true, + "is_anonymous_vpn": true + }, + "score": { + "inbound": "Critical", + "outbound": "Moderate" + }, + "user_search_count": 2, + "protected_ip": { + "count": 1, + "data": [ + { + "ip_address": "1.1.1.1", + "confirmed_time": "2022-12-07 03:09:46" + } + ] + }, + "domain": { + "count": 1057, + "data": [ + { + "domain": "xyhaidy.shop", + "ip_type": "Unknown", + "registrar": "Namecheap, Inc.", + "create_date": "2023-06-15 00:00:00", + "confirmed_time": "2023-06-18 19:19:36", + "email": "abuse@namecheap.com" + } + ] + }, + "whois": { + "count": 1, + "data": [ + { + "as_name": "MICROSOFT-CORP-MSN-AS-BLOCK", + "as_no": 8075, + "city": "San Jose", + "region": "California", + "org_name": "MICROSOFT-CORP-MSN-AS-BLOCK", + "postal_code": "", + "longitude": -121.8916, + "latitude": 37.3388, + "org_country_code": "us", + "confirmed_time": "2022-04-04 00:00:00" + } + ] + }, + "hostname": { + "count": 1, + "data": [ + { + "domain_name_rep": "cmk.ru", + "domain_name_full": "ip-195-182-143-57.clients.cmk.ru", + "confirmed_time": "2021-09-03 19:27:44" + } + ] + }, + "ids": { + "count": 1, + "data": [ + { + "classification": "botcc", + "url": "doc.emergingthreats.net/bin/view/Main/BotCC", + "message": "ET CNC Feodo Tracker Reported CnC Server UDP group 3", + "confirmed_time": "2022-01-28 00:12:16", + "source_system": "./snort-2.9.0 9949" + } + ] + }, + "vpn": { + "count": 1, + "data": [ + { + "vpn_name": "vpngate", + "vpn_url": "vpn369337084.opengw.net", + "vpn_source_url": "https://www.vpngate.net", + "socket_type": "tcp", + "confirmed_time": "2022-04-18 14:02:25" + } + ] + }, + "anonymous_vpn": { + "count": 1, + "data": [ + { + "vpn_name": "piavpn", + "vpn_url": "179.61.228.2", + "vpn_source_url": "https://www.privateinternetaccess.com", + "socket_type": "tcp", + "confirmed_time": "2022-12-10 00:16:59" + } + ] + }, + "webcam": { + "count": 1, + "data": [ + { + "image_path": "https://s3.us-west-1.amazonaws.com/cip-web-screenshot-new/cctv/151.192.67.27_8081_cctv.jpg", + "cam_url": "http://151.192.67.27:8081/webcapture.jpg?command=snap&channel=1?COUNTER", + "country": "Singapore", + "city": "Singapore", + "open_port_no": 8081, + "manufacturer": "Hi3516", + "confirmed_time": "2022-03-02 09:53:15" + } + ] + }, + "honeypot": { + "count": 1, + "data": [ + { + "ip_address": "64.62.197.76", + "log_date": "2023-06-15", + "dst_port": 22, + "message": "[15/Jun/2023:00:00:10] SSH-2.0-Go", + "user_agent": "-", + "protocol_type": "tcp", + "confirmed_time": "2023-06-15" + } + ] + }, + "ip_category": { + "count": 2, + "data": [ + { + "detect_source": "", + "type": "cloud service", + "detect_info": {}, + "confirmed_time": "2021-05-07 14:56:27" + }, + { + "detect_source": "", + "type": "MISP", + "detect_info": { + "md5": "460fb928925cb1fe4ae49dd208d43647", + "domain": "ab88be2e175710350.bitcoin.com" + }, + "confirmed_time": "2020-09-10 03:40:06" + } + ] + }, + "port": { + "count": 39, + "data": [ + { + "app_name": "ms-wbt-server", + "confirmed_time": "2022-03-10 13:53:06", + "banner": "HTTP header: HTTP/1.1 403 Forbidden\nServer: nginx/1.14.0 (Ubuntu)\nDate: Thu, 10 Mar 2022 09:45:57 GMT\nContent-Type: text/html\nContent-Length: 178\nConnection: keep-alive\n html: \n\n403 Forbidden\n\n

403 Forbidden

\n
nginx/1.14.0 (Ubuntu)
\n\n", + "app_version": "Unknown", + "open_port_no": 3389, + "port_status": "open", + "protocol": "RDP", + "socket": "tcp", + "tags": [ + "Data Leak" + ], + "dns_names": "*.cloudflare-dns.com,one.one.one.one", + "sdn_common_name": "cloudflare-dns.com", + "jarm_hash": "27d3ed3ed0003ed1dc42d43d00041d6183ff1bfae51ebd88d70384363d525c", + "ssl_info_raw": "TLS Certificate\nVersion: 3\nSerial Number: 4997145087721625482890198484998041183\n", + "technologies": [ + { + "tech_name": "jQuery", + "tech_version": "2.2.4", + "tech_logo_url": "https://cip-live-image.s3.us-west-1.amazonaws.com/tech/jQuery.svg" + } + ], + "is_vulnerability": false + } + ] + }, + "vulnerability": { + "count": 9, + "data": [ + { + "cve_id": "CVE-2021-23017", + "cve_description": "A security issue in nginx resolver was identified, which might allow an attacker who is able to forge UDP packets from the DNS server to cause 1-byte memory overwrite, resulting in worker process crash or potential other impact.", + "cvssv2_vector": "NETWORK", + "cvssv2_score": 6.8, + "cvssv3_vector": "NETWORK", + "cvssv3_score": 9.4, + "list_cwe": [ + { + "cve_id": "CVE-2021-23017", + "cwe_id": 193, + "cwe_name": "Off-by-one Error", + "cwe_description": "A product calculates or uses an incorrect maximum or minimum value that is 1 more, or 1 less, than the correct value." + } + ], + "list_edb": [ + { + "cve_id": "CVE-2021-44790", + "edb_id": 51193, + "type": "WEBAPPS", + "platform": "MULTIPLE", + "verify_code": 0, + "title": "Apache 2.4.x - Buffer Overflow", + "confirmed_time": "2023-04-01" + } + ], + "app_name": "nginx", + "app_version": "1.14.0", + "open_port_no_list": { + "TCP": [ + 514 + ], + "UDP": [] + }, + "have_more_ports": false, + "open_port_no": [ + { + "port": 514, + "socket": "tcp" + } + ], + "list_child": [ + { + "app_name": "nginx", + "app_version": "1.14.0", + "vendor": "nginx", + "type": "a", + "is_vuln": "True", + "target_hw": "iphone_os", + "target_sw": "ipad", + "update": "sp6a", + "edition": "workstation" + } + ], + "vendor": "nginx", + "type": "a", + "is_vuln": "True", + "target_hw": "iphone_os", + "target_sw": "ipad", + "update": "sp6a", + "edition": "workstation" + } + ] + }, + "mobile": { + "count": 2, + "data": [ + { + "broadband": "3G", + "organization": "SKT" + }, + { + "broadband": "LTE", + "organization": "SKT" + } + ] + }, + "status": 200 +} diff --git a/providers/criminalip/testdata/criminalip_9_9_9_9_report.json b/providers/criminalip/testdata/criminalip_9_9_9_9_report.json new file mode 100644 index 0000000..d40e943 --- /dev/null +++ b/providers/criminalip/testdata/criminalip_9_9_9_9_report.json @@ -0,0 +1,606 @@ +{ + "ip": "9.9.9.9", + "issues": { + "is_vpn": false, + "is_cloud": false, + "is_tor": false, + "is_proxy": false, + "is_hosting": false, + "is_mobile": false, + "is_darkweb": false, + "is_scanner": false, + "is_snort": false + }, + "score": { + "inbound": "Safe", + "outbound": "Safe" + }, + "user_search_count": 22, + "domain": { + "count": 72, + "data": [ + { + "domain": "wesvpn.com", + "ip_type": "Unknown", + "registrar": "NameCheap, Inc.", + "create_date": "2024-02-05 00:00:00", + "confirmed_time": "2024-02-07 16:27:06", + "email": "abuse@namecheap.com" + }, + { + "domain": "worthington.bank", + "ip_type": "Unknown", + "registrar": "MarkMonitor, Inc.", + "create_date": "2023-10-18 00:00:00", + "confirmed_time": "2023-10-26 17:05:52", + "email": "ccopsbilling@markmonitor.com" + }, + { + "domain": "khk.lol", + "ip_type": "Unknown", + "registrar": "Dynadot LLC", + "create_date": "2023-09-30 00:00:00", + "confirmed_time": "2023-10-02 16:59:04", + "email": "abuse@dynadot.com" + }, + { + "domain": "ipamaas.com", + "ip_type": "Unknown", + "registrar": "DREAMHOST", + "create_date": "2023-09-26 00:00:00", + "confirmed_time": "2023-09-28 15:13:05", + "email": "" + }, + { + "domain": "sneakpeek.ovh", + "ip_type": "Unknown", + "registrar": "OVH", + "create_date": "2023-09-07 00:00:00", + "confirmed_time": "2023-09-10 15:45:01", + "email": "" + }, + { + "domain": "cdn4ts.xyz", + "ip_type": "Unknown", + "registrar": "Namecheap", + "create_date": "2023-08-10 00:00:00", + "confirmed_time": "2023-08-12 15:30:59", + "email": "abuse@namecheap.com" + }, + { + "domain": "vvisa.xyz", + "ip_type": "Unknown", + "registrar": "Dynadot LLC", + "create_date": "2023-07-23 00:00:00", + "confirmed_time": "2023-07-24 16:37:18", + "email": "abuse@dynadot.com" + }, + { + "domain": "chenxiangpeng.top", + "ip_type": "Unknown", + "registrar": "Alibaba Cloud Computing Ltd. d/b/a HiChina (www.net.cn)", + "create_date": "2023-02-12 00:00:00", + "confirmed_time": "2023-02-16 07:56:23", + "email": "" + }, + { + "domain": "garliclab.xyz", + "ip_type": "Unknown", + "registrar": "Porkbun, LLC", + "create_date": "2023-02-07 00:00:00", + "confirmed_time": "2023-02-10 16:19:59", + "email": "abuse@porkbun.com" + }, + { + "domain": "get-your-team.info", + "ip_type": "Unknown", + "registrar": "NameSilo, LLC", + "create_date": "2023-02-01 00:00:00", + "confirmed_time": "2023-02-04 17:25:55", + "email": "" + }, + { + "domain": "tk112233.top", + "ip_type": "Unknown", + "registrar": "NameSilo,LLC", + "create_date": "2023-01-19 00:00:00", + "confirmed_time": "2023-01-22 20:27:10", + "email": "" + }, + { + "domain": "gawel.ovh", + "ip_type": "Unknown", + "registrar": "OVH", + "create_date": "2023-01-13 00:00:00", + "confirmed_time": "2023-01-16 19:33:26", + "email": "" + }, + { + "domain": "winfanqie.com", + "ip_type": "top_rank", + "registrar": "", + "create_date": "", + "confirmed_time": "2022-12-31 23:49:06", + "email": "" + }, + { + "domain": "xn--cnq01i.tech", + "ip_type": "Unknown", + "registrar": "Alibaba Cloud Computing Ltd. d/b/a HiChina (www.net.cn)", + "create_date": "2022-11-27 00:00:00", + "confirmed_time": "2022-11-29 16:26:45", + "email": "domainabuse@service.aliyun.com" + }, + { + "domain": "pegasbank.com", + "ip_type": "Unknown", + "registrar": "MarkMonitor, Inc.", + "create_date": "2022-10-31 00:00:00", + "confirmed_time": "2022-11-02 17:32:01", + "email": "" + }, + { + "domain": "ngdns.top", + "ip_type": "Unknown", + "registrar": "Alibaba Cloud Computing Ltd. d/b/a HiChina (www.net.cn)", + "create_date": "2022-10-19 00:00:00", + "confirmed_time": "2022-10-21 15:06:00", + "email": "" + }, + { + "domain": "shoresecuritysolutions.com", + "ip_type": "Unknown", + "registrar": "Dynu Systems, Inc.", + "create_date": "2022-08-19 00:00:00", + "confirmed_time": "2022-08-25 03:15:49", + "email": "" + }, + { + "domain": "super-plast.it", + "ip_type": "top_rank", + "registrar": "", + "create_date": "", + "confirmed_time": "2022-07-23 00:28:30", + "email": "" + }, + { + "domain": "yourapi.xyz", + "ip_type": "Unknown", + "registrar": "Dynadot LLC", + "create_date": "2022-06-20 00:00:00", + "confirmed_time": "2022-06-24 01:58:33", + "email": "abuse@dynadot.com" + }, + { + "domain": "tbw6699.com", + "ip_type": "Unknown", + "registrar": "DYNADOT, LLC", + "create_date": "2022-04-30 00:00:00", + "confirmed_time": "2022-05-05 04:43:30", + "email": "abuse@dynadot.com" + } + ] + }, + "whois": { + "count": 1, + "data": [ + { + "as_name": "QUAD9-AS-1", + "as_no": 19281, + "city": "Berkeley", + "region": "California", + "org_name": "Quad9", + "postal_code": "94709", + "latitude": 37.8767, + "longitude": -122.2676, + "org_country_code": "us", + "confirmed_time": "2024-03-24 00:00:00" + } + ] + }, + "hostname": { + "count": 1, + "data": [ + { + "domain_name_rep": "quad9.net", + "domain_name_full": "dns9.quad9.net", + "confirmed_time": "2024-03-06 06:00:33" + } + ] + }, + "ids": { + "count": 0, + "data": [] + }, + "vpn": { + "count": 0, + "data": [] + }, + "webcam": { + "count": 0, + "data": [] + }, + "honeypot": { + "count": 0, + "data": [] + }, + "ip_category": { + "count": 3, + "data": [ + { + "detect_source": "C-TAS(kisa-alliance)", + "type": "attack (High)", + "detect_info": {}, + "confirmed_time": "2023-10-12 21:39:26" + }, + { + "detect_source": "", + "type": "MISP", + "detect_info": {}, + "confirmed_time": "2022-12-19 12:03:18" + }, + { + "detect_source": "", + "type": "MISP", + "detect_info": { + "domain": "test1.bw.com" + }, + "confirmed_time": "2021-03-16 23:15:40" + } + ] + }, + "port": { + "count": 46, + "data": [ + { + "app_name": "PowerDNS dnsdist", + "banner": "HTTP/1.1\nStatus: 404 File Not Found\nDate: Thu, 21 Mar 2024 15:35:42 GMT\nContent Length: 9\nContent Type: text/plain; charset=utf-8\nServer: h2o/dnsdist\n\nnot found\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "app_version": "Unknown", + "open_port_no": 443, + "port_status": "open", + "protocol": "HTTPS", + "socket": "tcp", + "tags": [], + "dns_names": "149.112.112.14,149.112.112.15,9.9.9.9,149.112.112.9,149.112.112.11,149.112.112.13,149.112.112.112,*.quad9.net,quad9.net,9.9.9.12,9.9.9.14,9.9.9.10,149.112.112.12,9.9.9.11,9.9.9.13,9.9.9.15,149.112.112.10", + "sdn_common_name": "*.quad9.net", + "jarm_hash": "40d40d40d00040d00042d42d0000005a3e96c1dfa4bdb24b8b3c04cae18cc3", + "ssl_info_raw": "\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "technologies": [], + "is_vulnerability": false, + "confirmed_time": "2024-03-22 09:32:02" + }, + { + "app_name": "Q9-P", + "banner": "xef@\tQ9-P-7.6\n\n Resolver ID: res711.ams.rrdns.pch.net\n bind.version: Q9-P-7.6\n Recursion: enabled", + "app_version": "7.6", + "open_port_no": 53, + "port_status": "open", + "protocol": "DNS", + "socket": "udp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": false, + "confirmed_time": "2024-03-22 08:26:55" + }, + { + "app_name": "Q9-P", + "banner": "\n Resolver ID: res220.ams.rrdns.pch.net\n bind.version: Q9-P-7.6\n Recursion: enabled", + "app_version": "7.6", + "open_port_no": 53, + "port_status": "open", + "protocol": "DNS", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": false, + "confirmed_time": "2024-03-19 13:54:24" + }, + { + "app_name": "httpd", + "banner": "HTTP/1.0\nStatus: 400 Bad Request\nClient sent an HTTP request to an HTTPS server.", + "app_version": "Unknown", + "open_port_no": 5053, + "port_status": "open", + "protocol": "", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": false, + "confirmed_time": "2024-03-14 07:32:08" + }, + { + "app_name": "PowerDNS dnsdist", + "banner": "HTTP/1.1\nStatus: 404 File Not Found\nDate: Tue, 27 Feb 2024 12:43:14 GMT\nContent Length: 9\nContent Type: text/plain; charset=utf-8\nServer: h2o/dnsdist\n\nnot found\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "app_version": "Unknown", + "open_port_no": 443, + "port_status": "open", + "protocol": "HTTPS", + "socket": "tcp", + "tags": [], + "dns_names": "149.112.112.14,149.112.112.15,9.9.9.9,149.112.112.9,149.112.112.11,149.112.112.13,149.112.112.112,*.quad9.net,quad9.net,9.9.9.12,9.9.9.14,9.9.9.10,149.112.112.12,9.9.9.11,9.9.9.13,9.9.9.15,149.112.112.10", + "sdn_common_name": "*.quad9.net", + "jarm_hash": "40d40d40d00040d00042d42d0000005a3e96c1dfa4bdb24b8b3c04cae18cc3", + "ssl_info_raw": "\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2024-02-29 07:26:06" + }, + { + "app_name": "Q9-P", + "banner": "xef8I\tQ9-P-7.6\n\n Resolver ID: res120.ams.rrdns.pch.net\n bind.version: Q9-P-7.6\n Recursion: enabled", + "app_version": "7.6", + "open_port_no": 53, + "port_status": "open", + "protocol": "DNS", + "socket": "udp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2024-02-26 21:48:08" + }, + { + "app_name": "httpd", + "banner": "HTTP/1.0\nStatus: 400 Bad Request\nClient sent an HTTP request to an HTTPS server.", + "app_version": "Unknown", + "open_port_no": 5053, + "port_status": "open", + "protocol": "", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2024-02-16 22:37:09" + }, + { + "app_name": "Q9-P", + "banner": "\n Resolver ID: res121.ams.rrdns.pch.net\n bind.version: Q9-P-7.6\n Recursion: enabled", + "app_version": "7.6", + "open_port_no": 53, + "port_status": "open", + "protocol": "DNS", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2024-02-16 12:23:29" + }, + { + "app_name": "Q9-P", + "banner": "\n Resolver ID: res221.ams.rrdns.pch.net\n bind.version: Q9-P-7.6\n Resolver name: res711.ams.rrdns.pch.net\n Recursion: enabled", + "app_version": "7.6", + "open_port_no": 53, + "port_status": "open", + "protocol": "DNS", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2024-01-31 07:27:55" + }, + { + "app_name": "PowerDNS dnsdist", + "banner": "HTTP/1.1\nStatus: 404 File Not Found\nDate: Sun, 28 Jan 2024 23:25:19 GMT\nContent Length: 9\nContent Type: text/plain; charset=utf-8\nServer: h2o/dnsdist\n\nnot found\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "app_version": "Unknown", + "open_port_no": 443, + "port_status": "open", + "protocol": "HTTPS", + "socket": "tcp", + "tags": [], + "dns_names": "149.112.112.14,149.112.112.15,9.9.9.9,149.112.112.9,149.112.112.11,149.112.112.13,149.112.112.112,*.quad9.net,quad9.net,9.9.9.12,9.9.9.14,9.9.9.10,149.112.112.12,9.9.9.11,9.9.9.13,9.9.9.15,149.112.112.10", + "sdn_common_name": "*.quad9.net", + "jarm_hash": "40d40d40d00040d00042d42d0000005a3e96c1dfa4bdb24b8b3c04cae18cc3", + "ssl_info_raw": "\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2024-01-29 01:03:11" + }, + { + "app_name": "Q9-P", + "banner": "xef!\tQ9-P-7.6\n\n Resolver ID: res221.ams.rrdns.pch.net\n bind.version: Q9-P-7.6\n Recursion: enabled", + "app_version": "7.6", + "open_port_no": 53, + "port_status": "open", + "protocol": "DNS", + "socket": "udp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2024-01-25 12:59:33" + }, + { + "app_name": "httpd", + "banner": "HTTP/1.0\nStatus: 400 Bad Request\nClient sent an HTTP request to an HTTPS server.", + "app_version": "Unknown", + "open_port_no": 5053, + "port_status": "open", + "protocol": "", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2024-01-21 08:58:10" + }, + { + "app_name": "Q9-P", + "banner": "\n Resolver ID: res310.ams.rrdns.pch.net\n bind.version: Q9-P-7.6\n Resolver name: res721.ams.rrdns.pch.net\n Recursion: enabled", + "app_version": "7.6", + "open_port_no": 53, + "port_status": "open", + "protocol": "DNS", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2023-12-31 04:54:30" + }, + { + "app_name": "PowerDNS dnsdist", + "banner": "HTTP/1.1\nStatus: 404 File Not Found\nDate: Sat, 30 Dec 2023 06:31:57 GMT\nContent Length: 9\nContent Type: text/plain; charset=utf-8\nServer: h2o/dnsdist\n\nnot found\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "app_version": "Unknown", + "open_port_no": 443, + "port_status": "open", + "protocol": "HTTPS", + "socket": "tcp", + "tags": [], + "dns_names": "149.112.112.14,149.112.112.15,9.9.9.9,149.112.112.9,149.112.112.11,149.112.112.13,149.112.112.112,*.quad9.net,quad9.net,9.9.9.12,9.9.9.14,9.9.9.10,149.112.112.12,9.9.9.11,9.9.9.13,9.9.9.15,149.112.112.10", + "sdn_common_name": "*.quad9.net", + "jarm_hash": "40d40d40d00040d00042d42d0000005a3e96c1dfa4bdb24b8b3c04cae18cc3", + "ssl_info_raw": "\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2023-12-31 03:15:04" + }, + { + "app_name": "httpd", + "banner": "HTTP/1.0\nStatus: 400 Bad Request\nClient sent an HTTP request to an HTTPS server.", + "app_version": "Unknown", + "open_port_no": 5053, + "port_status": "open", + "protocol": "", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2023-12-13 18:57:44" + }, + { + "app_name": "PowerDNS dnsdist", + "banner": "HTTP/1.1\nStatus: 404 File Not Found\nDate: Tue, 28 Nov 2023 23:17:08 GMT\nContent Length: 9\nContent Type: text/plain; charset=utf-8\nServer: h2o/dnsdist\n\nnot found\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "app_version": "Unknown", + "open_port_no": 443, + "port_status": "open", + "protocol": "HTTPS", + "socket": "tcp", + "tags": [], + "dns_names": "149.112.112.14,149.112.112.15,9.9.9.9,149.112.112.9,149.112.112.11,149.112.112.13,149.112.112.112,*.quad9.net,quad9.net,9.9.9.12,9.9.9.14,9.9.9.10,149.112.112.12,9.9.9.11,9.9.9.13,9.9.9.15,149.112.112.10", + "sdn_common_name": "*.quad9.net", + "jarm_hash": "40d40d40d00040d00042d42d0000005a3e96c1dfa4bdb24b8b3c04cae18cc3", + "ssl_info_raw": "\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2023-11-29 03:14:08" + }, + { + "app_name": "Q9-P", + "banner": "\n Resolver ID: res110.ams.rrdns.pch.net\n bind.version: Q9-P-7.6\n Recursion: enabled", + "app_version": "7.6", + "open_port_no": 53, + "port_status": "open", + "protocol": "DNS", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2023-11-26 21:30:07" + }, + { + "app_name": "httpd", + "banner": "HTTP/1.0\nStatus: 400 Bad Request\nClient sent an HTTP request to an HTTPS server.", + "app_version": "Unknown", + "open_port_no": 5053, + "port_status": "open", + "protocol": "", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2023-11-21 06:28:15" + }, + { + "app_name": "PowerDNS dnsdist", + "banner": "HTTP/1.1\nStatus: 404 File Not Found\nDate: Mon, 30 Oct 2023 06:44:11 GMT\nContent Length: 9\nContent Type: text/plain; charset=utf-8\nServer: h2o/dnsdist\n\nnot found\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "app_version": "Unknown", + "open_port_no": 443, + "port_status": "open", + "protocol": "HTTPS", + "socket": "tcp", + "tags": [], + "dns_names": "149.112.112.14,149.112.112.15,9.9.9.9,149.112.112.9,149.112.112.11,149.112.112.13,149.112.112.112,*.quad9.net,quad9.net,9.9.9.12,9.9.9.14,9.9.9.10,149.112.112.12,9.9.9.11,9.9.9.13,9.9.9.15,149.112.112.10", + "sdn_common_name": "*.quad9.net", + "jarm_hash": "40d40d40d00040d00042d42d0000005a3e96c1dfa4bdb24b8b3c04cae18cc3", + "ssl_info_raw": "\n\nTLS Certificate\nVersion: 3\nSerial Number: 17337767665402821956911373044371295383\nSignature Algorithm: \n\tName: ECDSAWitHSHA384\n\tOid: 1.2.840.10045.4.3.3\nIssuer: \n\tCommon Name: DigiCert TLS Hybrid ECC SHA384 2020 CA1\n\tCountry: US\n\tOrganization: DigiCert Inc\nIssuer Dn: C=US, O=DigiCert Inc, CN=DigiCert TLS Hybrid ECC SHA384 2020 CA1\nValidity: \n\tStart: 2023-07-31T00:00:00Z\n\tEnd: 2024-08-06T23:59:59Z\n\tLength: 32227199\nSubject: \n\tCommon Name: *.quad9.net\n\tCountry: US\n\tLocality: Berkeley\n\tProvince: California\n\tOrganization: Quad9\nSubject Dn: C=US, ST=California, L=Berkeley, O=Quad9, CN=*.quad9.net\nSubject Key Info: \n\tKey Algorithm: \n\t\tName: ECDSA\n\tEcdsa Public Key: \n\t\tB: WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=\n\t\tCurve: P-256\n\t\tGx: axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=\n\t\tGy: T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=\n\t\tLength: 256\n\t\tN: /////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=\n\t\tP: /////wAAAAEAAAAAAAAAAAAAAAD///////////////8=\n\t\tPub: BH2L1x0DhQ0YJbM0HCmhJ9SsASVIiqDx6gK52FEsCGqsclbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\t\tX: fYvXHQOFDRglszQcKaEn1KwBJUiKoPHqArnYUSwIaqw=\n\t\tY: clbs+j2moJ9JCVWOrP65cxdcAvt4zCSRlG9DI4kOHWY=\n\tFingerprint Sha256: fd296cbe20646f4e58ffc5e2285f7e099b200adaea3d09396e1e3ba34477fc28\nExtensions: \n\tKey Usage: \n\t\tDigital Signature: True\n\t\tValue: 1\n\tBasic Constraints: \n\tSubject Alt Name: \n\t\tDns Names: *.quad9.net, quad9.net\n\t\tIp Addresses: 9.9.9.9, 9.9.9.10, 9.9.9.11, 9.9.9.12, 9.9.9.13, 9.9.9.14, 9.9.9.15, 149.112.112.9, 149.112.112.10, 149.112.112.11, 149.112.112.12, 149.112.112.13, 149.112.112.14, 149.112.112.15, 149.112.112.112, 2620:fe::9, 2620:fe::10, 2620:fe::11, 2620:fe::12, 2620:fe::13, 2620:fe::14, 2620:fe::15, 2620:fe::fe, 2620:fe::fe:9, 2620:fe::fe:10, 2620:fe::fe:11, 2620:fe::fe:12, 2620:fe::fe:13, 2620:fe::fe:14, 2620:fe::fe:15\n\tCrl Distribution Points: http://crl3.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl, http://crl4.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crl\n\tAuthority Key Id: 0abc0829178ca5396d7a0ece33c72eb3edfbc37a\n\tSubject Key Id: 7fa912a5d7c68b4802c73d2a456e401e4060f497\n\tExtended Key Usage: \n\t\tClient Auth: True\n\t\tServer Auth: True\n\tCertificate Policies: [{'id': '2.23.140.1.2.2', 'cps': ['http://www.digicert.com/CPS']}]\n\tAuthority Info Access: \n\t\tOcsp Urls: http://ocsp.digicert.com\n\t\tIssuer Urls: http://cacerts.digicert.com/DigiCertTLSHybridECCSHA3842020CA1-1.crt\n\tSigned Certificate Timestamps: [{'version': 0, 'log_id': '7s3QZNXbGs7FXLedtM0TojKHRny87N7DUUhZRnEftZs=', 'timestamp': 1690835926, 'signature': 'BAMARzBFAiEAgFrZoE0Z2bKo8We43Yg6tBBH8QQeJDI+6GmT7/lEYoUCIFCcQV5Yh8N4nP9uPtDYpRY/1nxHFdZDUQEVIV0usZ9P'}, {'version': 0, 'log_id': 'SLDja9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHM=', 'timestamp': 1690835926, 'signature': 'BAMASDBGAiEApUGxpLDzRBpEbDvd9FjnaWAwYRVlQ+OJ8PZLPbsmsCQCIQD8qRPUjxQw5DoKzsPivvoLs+POpc3Y2gH1c3cjurMPNA=='}, {'version': 0, 'log_id': '2ra/az+1tiKfm8K7XGvocJFxbLtRhIU0vaQ9MEjX+6s=', 'timestamp': 1690835926, 'signature': 'BAMARjBEAiBMmvofeflmsV3JoyFVid5GiJaPHkH9fDWkS93eP9fgEQIgfkTwCbSFNKnF47riYP4MJow7haBO+pFwRW5WAEC1AQQ='}]\nSignature: \n\tSignature Algorithm: \n\t\tName: ECDSAWitHSHA384\n\t\tOid: 1.2.840.10045.4.3.3\n\tValue: MGUCMDjrEa5jaoOtQkE1R/+iOltWSm/uXnyWaYGS1YQoq6Wsu5corjcgtUJBeo30wIfpxgIxAKEUjaguKWP9RlCd8oBI0jpFCpRZ6V0sWCMSc/cieN89OX2hf7i/pq6QFfRl4tnKbQ==\n\tValid: True\nFingerprint Md5: eb243d399443e997b07c8eef88b27084\n", + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2023-10-30 09:59:10" + }, + { + "app_name": "httpd", + "banner": "HTTP/1.0\nStatus: 400 Bad Request\nClient sent an HTTP request to an HTTPS server.", + "app_version": "Unknown", + "open_port_no": 5053, + "port_status": "open", + "protocol": "", + "socket": "tcp", + "tags": [], + "dns_names": null, + "sdn_common_name": null, + "jarm_hash": null, + "ssl_info_raw": null, + "technologies": [], + "is_vulnerability": null, + "confirmed_time": "2023-10-28 14:55:50" + } + ] + }, + "vulnerability": { + "count": 0, + "data": [] + }, + "mobile": { + "count": 0, + "data": [] + }, + "status": 200 +} diff --git a/providers/digitalocean/digitalocean.go b/providers/digitalocean/digitalocean.go new file mode 100644 index 0000000..fb8f745 --- /dev/null +++ b/providers/digitalocean/digitalocean.go @@ -0,0 +1,346 @@ +package digitalocean + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ip-fetcher/providers/digitalocean" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "digitalocean" + DocTTL = time.Duration(24 * time.Hour) +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +type ProviderClient struct { + session.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating digitalocean client") + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.DigitalOcean.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func unmarshalResponse(rBody []byte) (*HostSearchResult, error) { + var res *HostSearchResult + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + res.Raw = rBody + + return res, nil +} + +func unmarshalProviderData(data []byte) (*digitalocean.Doc, error) { + var res *digitalocean.Doc + + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling digitalocean data: %w", err) + } + + return res, nil +} + +func (c *ProviderClient) loadProviderData() error { + digitaloceanClient := digitalocean.New() + digitaloceanClient.Client = c.HTTPClient + + if c.Providers.DigitalOcean.URL != "" { + digitaloceanClient.DownloadURL = c.Providers.DigitalOcean.URL + c.Logger.Debug("overriding digitalocean source", "url", digitaloceanClient.DownloadURL) + } + + doc, err := digitaloceanClient.Fetch() + if err != nil { + return fmt.Errorf("error fetching digitalocean data: %w", err) + } + + data, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("error marshalling digitalocean provider doc: %w", err) + } + + err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName, + Value: data, + Version: doc.ETag, + Created: time.Now(), + }, DocTTL) + if err != nil { + return fmt.Errorf("error upserting digitalocean data: %w", err) + } + + return nil +} + +const ( + MaxColumnWidth = 120 +) + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising digitalocean client") + + // load provider data into cache if not already present and fresh + ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName) + if err != nil { + return fmt.Errorf("checking digitalocean cache: %w", err) + } + + if ok { + c.Logger.Info("digitalocean provider data found in cache") + + return nil + } + + c.Logger.Info("loading digitalocean provider data from source") + + err = c.loadProviderData() + if err != nil { + return fmt.Errorf("loading digitalocean api response: %w", err) + } + + return nil +} + +func (c *ProviderClient) loadProviderDataFromCache() (*digitalocean.Doc, error) { + c.Logger.Info("loading digitalocean provider data from cache") + + cacheKey := providers.CacheProviderPrefix + ProviderName + + var doc *digitalocean.Doc + + if item, err := cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + var uErr error + + doc, uErr = unmarshalProviderData(item.Value) + if uErr != nil { + defer func() { + _ = cache.Delete(c.Logger, c.Cache, cacheKey) + }() + + return nil, fmt.Errorf("error unmarshalling cached digitalocean provider doc: %w", uErr) + } + } else { + return nil, fmt.Errorf("error reading digitalocean cache: %w", err) + } + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return doc, nil +} + +func loadTestData(c *ProviderClient) ([]byte, error) { + tdf, err := loadResultsFile("providers/digitalocean/testdata/digitalocean_165_232_46_239_report.json") + if err != nil { + return nil, err + } + + c.Logger.Info("digitalocean match returned from test data", "host", "9.9.9.9") + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + return out, nil +} + +// FindHost searches for the host in the digitalocean data +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var result *HostSearchResult + + var err error + + // return cached report if test data is enabled + if c.UseTestData { + return loadTestData(c) + } + + doc, err := c.loadProviderDataFromCache() + if err != nil { + return nil, fmt.Errorf("loading digitalocean host data from cache: %w", err) + } + + // search in the data for the host + for _, record := range doc.Records { + if record.Network.Contains(c.Host) { + result = &HostSearchResult{ + Record: record, + ETag: doc.ETag, + LastModified: doc.LastModified, + } + + c.Logger.Debug("returning digitalocean host match data") + + break + } + } + + if result == nil { + return nil, fmt.Errorf("digitalocean: %w", providers.ErrNoMatchFound) + } + + var raw []byte + + raw, err = json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + result.Raw = raw + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + c.Logger.Debug("backing up digitalocean host report") + + if err = os.WriteFile(fmt.Sprintf("%s/backups/digitalocean_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), raw, 0o600); err != nil { + panic(err) + } + } + + return result.Raw, nil +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := unmarshalResponse(data) + if err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + tw := table.NewWriter() + + var rows []table.Row + + tw.AppendRow(table.Row{"Prefix", dashIfEmpty(result.Record.NetworkText)}) + tw.AppendRow(table.Row{"Country Code", dashIfEmpty(result.Record.CountryCode)}) + tw.AppendRow(table.Row{"City Name", dashIfEmpty(result.Record.CityName)}) + tw.AppendRow(table.Row{"City Code", dashIfEmpty(result.Record.CityCode)}) + tw.AppendRow(table.Row{"Zip Code", dashIfEmpty(result.Record.ZipCode)}) + + if !result.LastModified.IsZero() { + tw.AppendRow(table.Row{"Source Update", dashIfEmpty(result.LastModified.String())}) + } + + if result.ETag != "" { + tw.AppendRow(table.Row{"Version", dashIfEmpty(result.ETag)}) + } + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("DIGITAL OCEAN | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("DIGITAL OCEAN | Host: %s", "165.232.46.239") + } + + return &tw, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding file: %w", err) + } + + return res, nil +} + +type HostSearchResult struct { + Raw []byte + Record digitalocean.Record `json:"prefix"` + ETag string `json:"etag"` + LastModified time.Time `json:"last_modified"` +} + +func dashIfEmpty(value interface{}) string { + switch v := value.(type) { + case string: + if len(v) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(*v) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} diff --git a/providers/digitalocean/testdata/digitalocean_165_232_46_239_report.json b/providers/digitalocean/testdata/digitalocean_165_232_46_239_report.json new file mode 100644 index 0000000..b3790b7 --- /dev/null +++ b/providers/digitalocean/testdata/digitalocean_165_232_46_239_report.json @@ -0,0 +1,13 @@ +{ + "Raw": null, + "prefix": { + "Network": "165.232.32.0/20", + "NetworkText": "165.232.32.0/20", + "CountryCode": "GB", + "CityCode": "GB-SLG", + "CityName": "London", + "ZipCode": "SL1 4AX" + }, + "etag": "\"662bc170-e030\"", + "last_modified": "2024-04-26T16:00:00+01:00" +} diff --git a/providers/gcp/gcp.go b/providers/gcp/gcp.go new file mode 100644 index 0000000..49ac5e7 --- /dev/null +++ b/providers/gcp/gcp.go @@ -0,0 +1,370 @@ +package gcp + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ip-fetcher/providers/gcp" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "gcp" + DocTTL = time.Duration(24 * time.Hour) +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +type ProviderClient struct { + session.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating gcp client") + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.GCP.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func unmarshalResponse(rBody []byte) (*HostSearchResult, error) { + var res *HostSearchResult + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + res.Raw = rBody + + return res, nil +} + +func unmarshalProviderData(data []byte) (*gcp.Doc, error) { + var res *gcp.Doc + + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling gcp data: %w", err) + } + + return res, nil +} + +func (c *ProviderClient) loadProviderData() error { + gcpClient := gcp.New() + gcpClient.Client = c.HTTPClient + + if c.Providers.GCP.URL != "" { + gcpClient.DownloadURL = c.Providers.GCP.URL + c.Logger.Debug("overriding gcp source", "url", gcpClient.DownloadURL) + } + + doc, err := gcpClient.Fetch() + if err != nil { + return fmt.Errorf("error fetching gcp data: %w", err) + } + + data, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("error marshalling gcp provider doc: %w", err) + } + + err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName, + Value: data, + Version: doc.SyncToken, + Created: time.Now(), + }, DocTTL) + if err != nil { + return fmt.Errorf("error upserting gcp data: %w", err) + } + + return nil +} + +const ( + MaxColumnWidth = 120 +) + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising gcp client") + + // load provider data into cache if not already present and fresh + ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName) + if err != nil { + return fmt.Errorf("checking gcp cache: %w", err) + } + + if ok { + c.Logger.Info("gcp provider data found in cache") + + return nil + } + + c.Logger.Info("loading gcp provider data from source") + + err = c.loadProviderData() + if err != nil { + return fmt.Errorf("loading gcp api response: %w", err) + } + + return nil +} + +func (c *ProviderClient) loadProviderDataFromCache() (*gcp.Doc, error) { + c.Logger.Info("loading gcp provider data from cache") + + cacheKey := providers.CacheProviderPrefix + ProviderName + + var doc *gcp.Doc + + if item, err := cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + var uErr error + + doc, uErr = unmarshalProviderData(item.Value) + if uErr != nil { + defer func() { + _ = cache.Delete(c.Logger, c.Cache, cacheKey) + }() + + return nil, fmt.Errorf("error unmarshalling cached gcp provider doc: %w", uErr) + } + } else { + return nil, fmt.Errorf("error reading gcp cache: %w", err) + } + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return doc, nil +} + +func loadTestData(c *ProviderClient) ([]byte, error) { + tdf, err := loadResultsFile("providers/gcp/testdata/gcp_34_128_62_0_report.json") + if err != nil { + return nil, err + } + + c.Logger.Info("gcp match returned from test data", "host", "9.9.9.9") + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + return out, nil +} + +// FindHost searches for the host in the gcp data +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var result *HostSearchResult + + var err error + + // return cached report if test data is enabled + if c.UseTestData { + return loadTestData(c) + } + + doc, err := c.loadProviderDataFromCache() + if err != nil { + return nil, fmt.Errorf("loading gcp host data from cache: %w", err) + } + + // search in the data for the host + switch { + case c.Host.Is4(): + for _, record := range doc.IPv4Prefixes { + if record.IPv4Prefix.Contains(c.Host) { + result = &HostSearchResult{ + Prefix: record.IPv4Prefix, + SyncToken: doc.SyncToken, + Scope: record.Scope, + Service: record.Service, + CreationTime: doc.CreationTime, + } + + c.Logger.Debug("returning gcp host match data") + + break + } + } + case c.Host.Is6(): + for _, record := range doc.IPv6Prefixes { + if record.IPv6Prefix.Contains(c.Host) { + result = &HostSearchResult{ + Prefix: record.IPv6Prefix, + SyncToken: doc.SyncToken, + Scope: record.Scope, + Service: record.Service, + CreationTime: doc.CreationTime, + } + + c.Logger.Debug("returning gcp host match data") + + break + } + } + default: + return nil, fmt.Errorf("invalid host: %s", c.Host.String()) + } + + if result == nil { + c.Logger.Debug("no gcp host match found") + return nil, providers.ErrNoDataFound + } + + var raw []byte + + raw, err = json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + result.Raw = raw + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + c.Logger.Debug("backing up gcp host report") + + if err = os.WriteFile(fmt.Sprintf("%s/backups/gcp_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), raw, 0o600); err != nil { + panic(err) + } + } + + return result.Raw, nil +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := unmarshalResponse(data) + if err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + tw := table.NewWriter() + + var rows []table.Row + + tw.AppendRow(table.Row{"Prefix", dashIfEmpty(result.Prefix.String())}) + tw.AppendRow(table.Row{"Scope", dashIfEmpty(result.Scope)}) + tw.AppendRow(table.Row{"Service", dashIfEmpty(result.Service)}) + + if !result.CreationTime.IsZero() { + tw.AppendRow(table.Row{"Creation Time", dashIfEmpty(result.CreationTime.String())}) + } + + if result.SyncToken != "" { + tw.AppendRow(table.Row{"SyncToken", dashIfEmpty(result.SyncToken)}) + } + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("GCP | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("GCP | Host: %s", "34.128.62.2") + } + + return &tw, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding file: %w", err) + } + + return res, nil +} + +type HostSearchResult struct { + Raw []byte + Prefix netip.Prefix `json:"prefix"` + Service string `json:"service"` + Scope string `json:"scope"` + SyncToken string `json:"synctoken"` + CreationTime time.Time `json:"creation_time"` +} + +func dashIfEmpty(value interface{}) string { + switch v := value.(type) { + case string: + if len(v) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(*v) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} diff --git a/providers/gcp/testdata/gcp_34_128_62_0_report.json b/providers/gcp/testdata/gcp_34_128_62_0_report.json new file mode 100644 index 0000000..000549f --- /dev/null +++ b/providers/gcp/testdata/gcp_34_128_62_0_report.json @@ -0,0 +1,8 @@ +{ + "Raw": "eyJSYXciOm51bGwsInByZWZpeCI6IjM0LjEyOC42Mi4wLzIzIiwic2VydmljZSI6Ikdvb2dsZSBDbG91ZCIsInNjb3BlIjoidXMtd2VzdDgiLCJzeW5jdG9rZW4iOiIxNzE0NTcyMjk5OTU1IiwiY3JlYXRpb25fdGltZSI6IjIwMjQtMDUtMDFUMDc6MDQ6NTkuOTU1OTQ1WiJ9", + "prefix": "34.128.62.0/23", + "service": "Google Cloud", + "scope": "us-west8", + "synctoken": "1714572299955", + "creation_time": "2024-05-01T07:04:59.955945Z" +} diff --git a/providers/icloudpr/icloudpr.go b/providers/icloudpr/icloudpr.go new file mode 100644 index 0000000..7d0862d --- /dev/null +++ b/providers/icloudpr/icloudpr.go @@ -0,0 +1,351 @@ +package icloudpr + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ip-fetcher/providers/icloudpr" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "icloudpr" + DocTTL = 24 * time.Hour +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +type ProviderClient struct { + session.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating icloudpr client") + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.ICloudPR.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func unmarshalResponse(rBody []byte) (*HostSearchResult, error) { + var res *HostSearchResult + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + res.Raw = rBody + + return res, nil +} + +func unmarshalProviderData(data []byte) (*icloudpr.Doc, error) { + var res *icloudpr.Doc + + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling icloudpr data: %w", err) + } + + return res, nil +} + +func (c *ProviderClient) loadProviderData() error { + icloudprClient := icloudpr.New() + icloudprClient.Client = c.HTTPClient + + if c.Providers.ICloudPR.URL != "" { + icloudprClient.DownloadURL = c.Providers.ICloudPR.URL + c.Logger.Debug("overriding icloudpr source", "url", icloudprClient.DownloadURL) + } + + doc, err := icloudprClient.Fetch() + if err != nil { + return fmt.Errorf("error fetching icloudpr data: %w", err) + } + + data, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("error marshalling icloudpr provider doc: %w", err) + } + + err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName, + Value: data, + Version: doc.ETag, + Created: time.Now(), + }, DocTTL) + if err != nil { + return fmt.Errorf("error upserting icloudpr data: %w", err) + } + + return nil +} + +const ( + MaxColumnWidth = 120 +) + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising icloudpr client") + + // load provider data into cache if not already present and fresh + ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName) + if err != nil { + return fmt.Errorf("checking icloudpr cache: %w", err) + } + + if ok { + c.Logger.Info("icloudpr provider data found in cache") + + return nil + } + + c.Logger.Info("loading icloudpr provider data from source") + + err = c.loadProviderData() + if err != nil { + return fmt.Errorf("loading icloudpr api response: %w", err) + } + + return nil +} + +func (c *ProviderClient) loadProviderDataFromCache() (*icloudpr.Doc, error) { + c.Logger.Info("loading icloudpr provider data from cache") + + cacheKey := providers.CacheProviderPrefix + ProviderName + + var doc *icloudpr.Doc + + if item, err := cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + var uErr error + + doc, uErr = unmarshalProviderData(item.Value) + if uErr != nil { + defer func() { + _ = cache.Delete(c.Logger, c.Cache, cacheKey) + }() + + return nil, fmt.Errorf("error unmarshalling cached icloudpr provider doc: %w", uErr) + } + } else { + return nil, fmt.Errorf("error reading icloudpr cache: %w", err) + } + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return doc, nil +} + +func loadTestData(c *ProviderClient) ([]byte, error) { + tdf, err := loadResultsFile("providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json") + if err != nil { + return nil, err + } + + c.Logger.Info("icloudpr match returned from test data", "host", "172.224.224.60") + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + return out, nil +} + +// FindHost searches for the host in the icloudpr data +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var result *HostSearchResult + + var err error + + // return cached report if test data is enabled + if c.UseTestData { + return loadTestData(c) + } + + doc, err := c.loadProviderDataFromCache() + if err != nil { + return nil, fmt.Errorf("loading icloudpr host data from cache: %w", err) + } + + // search in the data for the host + for _, record := range doc.Records { + if record.Prefix.Contains(c.Host) { + result = &HostSearchResult{ + Prefix: record.Prefix, + Alpha2Code: record.Alpha2Code, + Region: record.Region, + City: record.City, + PostalCode: record.PostalCode, + SyncToken: doc.ETag, + CreationTime: time.Time{}, + } + + c.Logger.Debug("returning icloudpr host match data") + + break + } + } + + if result == nil { + c.Logger.Debug("no icloudpr host match found") + return nil, providers.ErrNoMatchFound + } + + var raw []byte + + raw, err = json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + result.Raw = raw + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + c.Logger.Debug("backing up icloudpr host report") + + if err = os.WriteFile(fmt.Sprintf("%s/backups/icloudpr_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), raw, 0o600); err != nil { + panic(err) + } + } + + return result.Raw, nil +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := unmarshalResponse(data) + if err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + tw := table.NewWriter() + + var rows []table.Row + + tw.AppendRow(table.Row{"Prefix", dashIfEmpty(result.Prefix.String())}) + tw.AppendRow(table.Row{"Alpha2Code", dashIfEmpty(result.Alpha2Code)}) + tw.AppendRow(table.Row{"Region", dashIfEmpty(result.Region)}) + tw.AppendRow(table.Row{"City", dashIfEmpty(result.City)}) + // tw.AppendRow(table.Row{"Postal Code", dashIfEmpty(result.PostalCode)}) + + if !result.CreationTime.IsZero() { + tw.AppendRow(table.Row{"Creation Time", dashIfEmpty(result.CreationTime.String())}) + } + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("ICLOUD PRIVATE RELAY | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("ICLOUD PRIVATE RELAY | Host: %s", "172.224.224.60") + } + + return &tw, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding file: %w", err) + } + + return res, nil +} + +type HostSearchResult struct { + Raw []byte + Prefix netip.Prefix `json:"ip_prefix"` + Alpha2Code string `json:"alpha2code"` + Region string `json:"region"` + City string `json:"city"` + PostalCode string `json:"postal_code"` + SyncToken string `json:"synctoken"` + CreationTime time.Time `json:"creation_time"` +} + +func dashIfEmpty(value interface{}) string { + switch v := value.(type) { + case string: + if len(v) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(*v) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} diff --git a/providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json b/providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json new file mode 100644 index 0000000..5fda0f0 --- /dev/null +++ b/providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json @@ -0,0 +1,10 @@ +{ + "Raw": "eyJSYXciOm51bGwsImlwX3ByZWZpeCI6IjE3Mi4yMjQuMjI0LjYwLzMxIiwiYWxwaGEyY29kZSI6IkdCIiwicmVnaW9uIjoiR0ItV0EiLCJjaXR5IjoiQ2FyZGlmZiIsInBvc3RhbF9jb2RlIjoiIiwic3luY3Rva2VuIjoiXCI2NjJmNmQ4Mi1iMDFmZWJcIiIsImNyZWF0aW9uX3RpbWUiOiIwMDAxLTAxLTAxVDAwOjAwOjAwWiJ9", + "ip_prefix": "172.224.224.60/31", + "alpha2code": "GB", + "region": "GB-WA", + "city": "Cardiff", + "postal_code": "", + "synctoken": "\"662f6d82-b01feb\"", + "creation_time": "0001-01-01T00:00:00Z" +} diff --git a/providers/ipurl/ipurl.go b/providers/ipurl/ipurl.go new file mode 100644 index 0000000..235688c --- /dev/null +++ b/providers/ipurl/ipurl.go @@ -0,0 +1,390 @@ +package ipurl + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "sync" + "time" + + "github.com/fatih/color" + "github.com/jedib0t/go-pretty/v6/table" + ipfetcherURL "github.com/jonhadfield/ip-fetcher/providers/url" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "ipurl" + CacheTTL = 3 * time.Hour +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + URLs []string +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.IPURL.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func loadTestData() ([]byte, error) { + tdf, err := loadResultsFile("providers/ipurl/testdata/ipurl_5_105_62_0_report.json") + if err != nil { + return nil, err + } + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + return out, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding json: %w", err) + } + + return res, nil +} + +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + if c.UseTestData { + return loadTestData() + } + + pwp := make(map[netip.Prefix][]string) + + err := c.loadProviderDataFromCache(pwp) + if err != nil { + return nil, err + } + + var matches map[netip.Prefix][]string + + for prefix, urls := range pwp { + if prefix.Contains(c.Host) { + c.Logger.Info("ipurl match found", "host", c.Host.String(), "urls", urls) + + if matches == nil { + matches = make(map[netip.Prefix][]string) + } + + matches[prefix] = urls + } + } + + if matches == nil { + return nil, fmt.Errorf("ip urls: %w", providers.ErrNoMatchFound) + } + + var raw []byte + + raw, err = json.Marshal(matches) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + return raw, nil +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating ipurl client") + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +type ProviderClient struct { + session.Session +} + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising ipurl client") + + err := c.refreshURLCache() + if err != nil { + return err + } + + return nil +} + +// refreshURLCache checks the cache for each of the urls in the session and loads them into cache if not found +func (c *ProviderClient) refreshURLCache() error { + // refresh list + var refreshList []string + + for _, u := range c.Providers.IPURL.URLs { + if c.Config.Global.DisableCache { + c.Logger.Debug("cache disabled, refreshing all ipurl urls") + + refreshList = append(refreshList, u) + + continue + } + + if ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName+"_"+generateURLHash(u)); err != nil { + return fmt.Errorf("error checking cache for ipurl provider data: %w", err) + } else if !ok { + // add to refresh list + refreshList = append(refreshList, u) + } + } + + c.Logger.Debug("refreshing ipurl cache", + "urls", len(c.Providers.IPURL.URLs), + "fresh", len(c.Providers.IPURL.URLs)-len(refreshList), + "not in cache", len(refreshList)) + + if len(refreshList) == 0 { + c.Stats.Mu.Lock() + c.Stats.InitialiseUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return nil + } + + if err := c.loadProviderURLsFromSource(refreshList); err != nil { + return err + } + + return nil +} + +// generateURLHash concatenates the provider name and the url string and returns a hash +func generateURLHash(us string) string { + h := sha256.New() + h.Write([]byte(us)) + r := hex.EncodeToString(h.Sum(nil)) + + return r[:providers.CacheKeySHALen] +} + +type StoredPrefixes struct { + Hash string + Prefixes map[netip.Prefix][]string +} + +type StoredURLPrefixes struct { + URL string + Prefixes []netip.Prefix +} + +func (c *ProviderClient) loadProviderURLsFromSource(providerUrls []string) error { + ic := ipfetcherURL.New(ipfetcherURL.WithHttpClient(c.HTTPClient)) + ic.HttpClient = c.HTTPClient + + var wg sync.WaitGroup + + // create a hash from the slice of urls + // this will identify the data we cache based of this input + for _, iu := range providerUrls { + wg.Add(1) + + go func() { + defer wg.Done() + + _, err := c.loadProviderURLFromSource(iu) + if err != nil { + c.Logger.Error("error loading provider", "url", iu, "error", err) + } + }() + } + + wg.Wait() + + return nil +} + +// loadProviderDataFromSource fetches the data from the source and caches it for individual urls +func (c *ProviderClient) loadProviderURLFromSource(pURL string) ([]netip.Prefix, error) { + hf := ipfetcherURL.HttpFile{ + Client: c.HTTPClient, + Url: pURL, + } + + if c.Config.Global.LogLevel == "debug" { + hf.Debug = true + } + + hfPrefixes, err := hf.FetchPrefixes() + if err != nil { + return nil, fmt.Errorf("error fetching ipurl data: %w", err) + } + + // cache the prefixes for this url + var mHfPrefixes []byte + + if mHfPrefixes, err = json.Marshal(hfPrefixes); err != nil { + return nil, fmt.Errorf("error marshalling ipurl provider doc: %w", err) + } + + uh := generateURLHash(pURL) + + if err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName + "_" + uh, + Value: mHfPrefixes, + Version: "-", + Created: time.Now(), + }, CacheTTL); err != nil { + return nil, fmt.Errorf("error upserting ipurl provider data: %w", err) + } + + return hfPrefixes, nil +} + +func (c *ProviderClient) loadProviderURLDataFromCache(pURL string) ([]netip.Prefix, error) { + cacheKey := providers.CacheProviderPrefix + ProviderName + "_" + generateURLHash(pURL) + + var item *cache.Item + + var err error + + if item, err = cache.Read(c.Logger, c.Cache, cacheKey); err != nil { + return nil, fmt.Errorf("error reading ipurl provider cache: %w", err) + } + + prefixes, uErr := unmarshalProviderData(item.Value) + if uErr != nil { + defer func() { + // remove any data that can't be unmarshalled + _ = cache.Delete(c.Logger, c.Cache, cacheKey) + }() + + return nil, fmt.Errorf("error unmarshalling cached ipurl provider doc: %w", err) + } + + return prefixes, nil +} + +func (c *ProviderClient) loadProviderDataFromCache(pwp map[netip.Prefix][]string) error { + for _, u := range c.Providers.IPURL.URLs { + prefixes, err := c.loadProviderURLDataFromCache(u) + if err != nil { + return err + } + + for _, prefix := range prefixes { + pwp[prefix] = append(pwp[prefix], u) + } + } + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return nil +} + +func unmarshalProviderData(rBody []byte) ([]netip.Prefix, error) { + var prefixes []netip.Prefix + + if err := json.Unmarshal(rBody, &prefixes); err != nil { + return nil, fmt.Errorf("error unmarshalling ipurl api response: %w", err) + } + + return prefixes, nil +} + +type HostSearchResult map[netip.Prefix][]string + +func unmarshalResponse(rBody []byte) (HostSearchResult, error) { + var res HostSearchResult + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling ipurl api response: %w", err) + } + + return res, nil +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := unmarshalResponse(data) + if err != nil { + return nil, fmt.Errorf("error unmarshalling stored ipurl data: %w", err) + } + + if result == nil { + return nil, nil + } + + tw := table.NewWriter() + tw.AppendRow(table.Row{color.HiWhiteString("Prefixes")}) + + for prefix, urls := range result { + tw.AppendRow(table.Row{"", color.CyanString(prefix.String())}) + + for _, url := range urls { + tw.AppendRow(table.Row{"", fmt.Sprintf("%s %s", IndentPipeHyphens, url)}) + } + } + + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: true, WidthMax: MaxColumnWidth, WidthMin: 10}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("IP URL | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("IP URL | Host: 5.105.62.60") + } + + return &tw, nil +} + +const MaxColumnWidth = 120 + +const IndentPipeHyphens = " |-----" diff --git a/providers/ipurl/testdata/ipurl_5_105_62_0_report.json b/providers/ipurl/testdata/ipurl_5_105_62_0_report.json new file mode 100644 index 0000000..0d177eb --- /dev/null +++ b/providers/ipurl/testdata/ipurl_5_105_62_0_report.json @@ -0,0 +1 @@ +{"5.105.62.0/24":["https://iplists.firehol.org/files/firehol_level1.netset"]} \ No newline at end of file diff --git a/providers/linode/linode.go b/providers/linode/linode.go new file mode 100644 index 0000000..2c3c027 --- /dev/null +++ b/providers/linode/linode.go @@ -0,0 +1,350 @@ +package linode + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ip-fetcher/providers/linode" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "linode" + DocTTL = time.Duration(24 * time.Hour) +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +type ProviderClient struct { + session.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating linode client") + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.Linode.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func unmarshalResponse(rBody []byte) (*HostSearchResult, error) { + var res *HostSearchResult + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + res.Raw = rBody + + return res, nil +} + +func unmarshalProviderData(data []byte) (*linode.Doc, error) { + var res *linode.Doc + + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling linode data: %w", err) + } + + return res, nil +} + +func (c *ProviderClient) loadProviderData() error { + linodeClient := linode.New() + linodeClient.Client = c.HTTPClient + + if c.Providers.Linode.URL != "" { + linodeClient.DownloadURL = c.Providers.Linode.URL + c.Logger.Debug("overriding linode source", "url", linodeClient.DownloadURL) + } + + doc, err := linodeClient.Fetch() + if err != nil { + return fmt.Errorf("error fetching linode data: %w", err) + } + + data, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("error marshalling linode provider doc: %w", err) + } + + err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName, + Value: data, + Version: doc.ETag, + Created: time.Now(), + }, DocTTL) + if err != nil { + return fmt.Errorf("error upserting linode data: %w", err) + } + + return nil +} + +const ( + MaxColumnWidth = 120 +) + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising linode client") + + // load provider data into cache if not already present and fresh + ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName) + if err != nil { + return fmt.Errorf("checking linode cache: %w", err) + } + + if ok { + c.Logger.Info("linode provider data found in cache") + + return nil + } + + c.Logger.Info("loading linode provider data from source") + + err = c.loadProviderData() + if err != nil { + return fmt.Errorf("loading linode api response: %w", err) + } + + return nil +} + +func (c *ProviderClient) loadProviderDataFromCache() (*linode.Doc, error) { + c.Logger.Info("loading linode provider data from cache") + + cacheKey := providers.CacheProviderPrefix + ProviderName + + var doc *linode.Doc + + if item, err := cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + var uErr error + + doc, uErr = unmarshalProviderData(item.Value) + if uErr != nil { + defer func() { + _ = cache.Delete(c.Logger, c.Cache, cacheKey) + }() + + return nil, fmt.Errorf("error unmarshalling cached linode provider doc: %w", uErr) + } + } else { + return nil, fmt.Errorf("error reading linode cache: %w", err) + } + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return doc, nil +} + +func loadTestData(c *ProviderClient) ([]byte, error) { + tdf, err := loadResultsFile("providers/linode/testdata/linode_69_164_198_1_report.json") + if err != nil { + return nil, err + } + + c.Logger.Info("linode match returned from test data", "host", "69.164.198.1") + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + return out, nil +} + +// FindHost searches for the host in the linode data +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var result *HostSearchResult + + var err error + + // return cached report if test data is enabled + if c.UseTestData { + return loadTestData(c) + } + + doc, err := c.loadProviderDataFromCache() + if err != nil { + return nil, fmt.Errorf("loading linode host data from cache: %w", err) + } + + // search in the data for the host + for _, record := range doc.Records { + if record.Prefix.Contains(c.Host) { + result = &HostSearchResult{ + Prefix: record.Prefix, + Alpha2Code: record.Alpha2Code, + Region: record.Region, + City: record.City, + PostalCode: record.PostalCode, + SyncToken: doc.ETag, + CreationTime: time.Time{}, + } + + c.Logger.Debug("returning linode host match data") + + break + } + } + + if result == nil { + c.Logger.Debug("no linode host match found") + return nil, providers.ErrNoMatchFound + } + + var raw []byte + + raw, err = json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + result.Raw = raw + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + c.Logger.Debug("backing up linode host report") + + if err = os.WriteFile(fmt.Sprintf("%s/backups/linode_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), raw, 0o600); err != nil { + panic(err) + } + } + + return result.Raw, nil +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := unmarshalResponse(data) + if err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + tw := table.NewWriter() + + var rows []table.Row + + tw.AppendRow(table.Row{"Prefix", dashIfEmpty(result.Prefix.String())}) + tw.AppendRow(table.Row{"Alpha2Code", dashIfEmpty(result.Alpha2Code)}) + tw.AppendRow(table.Row{"Region", dashIfEmpty(result.Region)}) + tw.AppendRow(table.Row{"City", dashIfEmpty(result.City)}) + + if !result.CreationTime.IsZero() { + tw.AppendRow(table.Row{"Creation Time", dashIfEmpty(result.CreationTime.String())}) + } + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("LINODE | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("LINODE | Host: %s", "69.164.198.1") + } + + return &tw, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding file: %w", err) + } + + return res, nil +} + +type HostSearchResult struct { + Raw []byte + Prefix netip.Prefix `json:"ip_prefix"` + Alpha2Code string `json:"alpha2code"` + Region string `json:"region"` + City string `json:"city"` + PostalCode string `json:"postal_code"` + SyncToken string `json:"synctoken"` + CreationTime time.Time `json:"creation_time"` +} + +func dashIfEmpty(value interface{}) string { + switch v := value.(type) { + case string: + if len(v) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(*v) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} diff --git a/providers/linode/testdata/linode_69_164_198_1_report.json b/providers/linode/testdata/linode_69_164_198_1_report.json new file mode 100644 index 0000000..a2ab3f4 --- /dev/null +++ b/providers/linode/testdata/linode_69_164_198_1_report.json @@ -0,0 +1,10 @@ +{ + "Raw": "eyJSYXciOm51bGwsImlwX3ByZWZpeCI6IjY5LjE2NC4xOTguMC8yNCIsImFscGhhMmNvZGUiOiJVUyIsInJlZ2lvbiI6IlVTLVRYIiwiY2l0eSI6IlJpY2hhcmRzb24iLCJwb3N0YWxfY29kZSI6IiIsInN5bmN0b2tlbiI6IlwiNjYzMzY0MjQtMWY5NTFcIiIsImNyZWF0aW9uX3RpbWUiOiIwMDAxLTAxLTAxVDAwOjAwOjAwWiJ9", + "ip_prefix": "69.164.198.0/24", + "alpha2code": "US", + "region": "US-TX", + "city": "Richardson", + "postal_code": "", + "synctoken": "\"66336424-1f951\"", + "creation_time": "0001-01-01T00:00:00Z" +} \ No newline at end of file diff --git a/providers/providers.go b/providers/providers.go new file mode 100644 index 0000000..2e9be3f --- /dev/null +++ b/providers/providers.go @@ -0,0 +1,276 @@ +package providers + +import ( + "errors" + "fmt" + "log/slog" + "strconv" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + + "github.com/jonhadfield/ipscout/session" +) + +var ( + ErrFailedToFetchData = errors.New("failed to fetch data") + ErrNoDataFound = errors.New("no data found") + ErrNoMatchFound = errors.New("no match found") + ErrForbiddenByProvider = errors.New("forbidden by provider") + CacheProviderPrefix = "provider_" + CacheKeySHALen = 16 +) + +func AgeToHours(age string) (int64, error) { + if age == "" { + return 0, nil + } + + age = strings.ToLower(age) + multiplier := map[string]int{ + "h": 1, + "d": 24, + "w": 24 * 7, + "m": 24 * 30, + "y": 24 * 365, + } + + var ageNum int64 + + for k, v := range multiplier { + if strings.HasSuffix(age, k) { + age = strings.TrimSuffix(age, k) + + var err error + + ageNum, err = strconv.ParseInt(age, 10, 64) + if err != nil { + return 0, fmt.Errorf("error parsing age: %w", err) + } + + ageNum *= int64(v) + + break + } + } + + return ageNum, nil +} + +// PortNetworkMatch returns true if the incomingPort matches any of the matchPorts +func PortNetworkMatch(incomingPort string, matchPorts []string) bool { + if len(matchPorts) == 0 { + // if len(matchPorts) == 0 || matchPorts[0] == "[]" { + return true + } + + for _, p := range matchPorts { + splitMatch := splitPortTransport(p) + splitIncomingPort := splitPortTransport(incomingPort) + + // most specific first + switch { + case splitIncomingPort.port != "" && splitIncomingPort.transport != "" && splitMatch.port == splitIncomingPort.port && splitMatch.transport == splitIncomingPort.transport: + // if both parts of match incomingPort are set and both match then return true + return true + case splitIncomingPort.port != "" && splitIncomingPort.transport != "" && splitMatch.transport == "" && splitMatch.port == splitIncomingPort.port && splitMatch.transport != splitIncomingPort.transport: + // if both parts of match incomingPort are set and only port to match is set matches then return true + return true + case splitIncomingPort.port != "" && splitIncomingPort.transport != "" && splitMatch.port == "" && splitMatch.transport == splitIncomingPort.transport: + // if both parts of match incomingPort are set and only transport matches then return true + return true + case splitIncomingPort.transport == "" && splitIncomingPort.port != "" && splitMatch.port == splitIncomingPort.port: + // if only incomingPort is set, then only incomingPort needs to match + return true + case splitIncomingPort.port == "" && splitIncomingPort.transport != "" && splitMatch.transport == splitIncomingPort.transport: + // if only transport is set, then only transport needs to match + return true + } + } + + return false +} + +// portAgeCheck returns true if the port is within the max age +func portAgeCheck(portConfirmedTime string, timeFormat string, maxAge string) (bool, error) { + switch { + case portConfirmedTime == "": + return false, fmt.Errorf("no port confirmed time provided") + case timeFormat == "": + return false, fmt.Errorf("no time format provided") + } + + // if no age filter provided, then return true + if maxAge == "" { + return true, nil + } + + var confirmedTime time.Time + + var err error + + maxAgeHours, err := AgeToHours(maxAge) + if err != nil { + return false, fmt.Errorf("error parsing max-age: %w", err) + } + + confirmedTime, err = time.Parse(timeFormat, portConfirmedTime) + if err != nil { + return false, fmt.Errorf("error parsing confirmed time: %w", err) + } + + if confirmedTime.After(time.Now().Add(-time.Duration(maxAgeHours) * time.Hour)) { + return true, nil + } + + return false, nil +} + +type PortMatchFilterInput struct { + Provider string + IncomingPort string + Logger *slog.Logger + MatchPorts []string + ConfirmedDate string + ConfirmedDateFormat string + MaxAge string +} + +// PortMatchFilter returns true by default, and false if either age or netmatch is specified +// and doesn't match +func PortMatchFilter(in PortMatchFilterInput) (ageMatch, netMatch bool, err error) { + switch in.IncomingPort { + case "": + netMatch = true + default: + netMatch = PortNetworkMatch(in.IncomingPort, in.MatchPorts) + } + + switch { + case in.ConfirmedDate == "" && in.ConfirmedDateFormat == "": + ageMatch = true + case in.ConfirmedDate == "" || in.ConfirmedDateFormat == "": + return false, false, fmt.Errorf("both confirmed date and format must be specified") + default: + ageMatch, err = portAgeCheck(in.ConfirmedDate, in.ConfirmedDateFormat, in.MaxAge) + if err != nil { + return false, false, fmt.Errorf("error checking port age: %w", err) + } + } + + return ageMatch, netMatch, nil +} + +func PreProcessValueOutput(sess *session.Session, in string) string { + out := strings.TrimSpace(in) + + // abbreviate output value if it exceeds max value chars + if sess.Config.Global.MaxValueChars > 0 { + if len(out) > int(sess.Config.Global.MaxValueChars) { + out = out[:sess.Config.Global.MaxValueChars] + "..." + } + } + + return out +} + +type PortTransport struct { + port string + transport string +} + +func splitPortTransport(portTransport string) (pt PortTransport) { + parts := strings.Split(portTransport, "/") + + switch len(parts) { + case 1: + if isPort(parts[0]) { + pt.port = parts[0] + } else if isTransport(parts[0]) { + pt.transport = parts[0] + } + case 2: + if isPort(parts[0]) && isTransport(parts[1]) { + pt.port = parts[0] + pt.transport = parts[1] + } + } + + return pt +} + +var validTransports = []string{"tcp", "udp", "icmp"} + +func isPort(in any) bool { + switch v := in.(type) { + case string: + var cint int + + var err error + + if cint, err = strconv.Atoi(v); err != nil { + return false + } + + return isPort(cint) + case int: + if v > 0 && v < 65535 { + return true + } + case int32: + if v > 0 && v < 65535 { + return true + } + } + + return false +} + +func isTransport(in any) bool { + if s, ok := in.(string); ok { + for _, t := range validTransports { + if strings.EqualFold(s, t) { + return true + } + } + } + + return false +} + +func DashIfEmpty(value interface{}) string { + switch v := value.(type) { + case time.Time: + if v.IsZero() || v == time.Date(0o001, time.January, 1, 0, 0, 0, 0, time.UTC) { + return "-" + } + + return v.Format(time.DateTime) + case string: + trimmed := strings.TrimSpace(v) + if len(trimmed) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(strings.TrimSpace(*v)) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} + +type ProviderClient interface { + Enabled() bool + GetConfig() *session.Session + Initialise() error + FindHost() ([]byte, error) + CreateTable([]byte) (*table.Writer, error) +} diff --git a/providers/providers_test.go b/providers/providers_test.go new file mode 100644 index 0000000..9a2bd6c --- /dev/null +++ b/providers/providers_test.go @@ -0,0 +1,232 @@ +package providers + +import ( + "testing" + "time" + + "github.com/jonhadfield/ipscout/session" + + "github.com/stretchr/testify/require" +) + +// portAgeCheck returns true if the port is within the max age +// func portAgeCheck(portConfirmedTime string, timeFormat string, maxAge string) (bool, error) { +// // if no age filter provided, then return false +// if maxAge == "" { +// return false, nil +// } +// +// var confirmedTime time.Time +// var err error +// +// maxAgeHours, err := AgeToHours(maxAge) +// if err != nil { +// return false, fmt.Errorf("error parsing max-age: %w", err) +// } +// +// confirmedTime, err = time.Parse(timeFormat, portConfirmedTime) +// if err != nil { +// return false, err +// } +// +// if confirmedTime.Before(time.Now().Add(-time.Duration(maxAgeHours) * time.Hour)) { +// return true, err +// } +// +// return false, nil +// } + +func TestPortAgeCheckWithNoValues(t *testing.T) { + res, err := portAgeCheck("", "", "") + require.Error(t, err) + require.False(t, res) +} + +func TestPortAgeCheckNoMaxAge(t *testing.T) { + res, err := portAgeCheck("2024-04-04 00:00:00", time.DateTime, "") + require.NoError(t, err) + require.True(t, res) +} + +func TestPortAgeCheckOlderThanMaxAge(t *testing.T) { + res, err := portAgeCheck("2024-04-01 00:00:00", time.DateTime, "1d") + require.NoError(t, err) + require.False(t, res) +} + +func TestPortMatchFilterWithNoValues(t *testing.T) { + ageMatch, netMatch, err := PortMatchFilter(PortMatchFilterInput{ + IncomingPort: "", + MatchPorts: nil, + ConfirmedDate: "", + ConfirmedDateFormat: "", + MaxAge: "", + }) + require.NoError(t, err) + require.True(t, ageMatch) + require.True(t, netMatch) +} + +func TestPortMatchFilterWithNetworkMatch(t *testing.T) { + age, res, err := PortMatchFilter(PortMatchFilterInput{ + IncomingPort: "80", + MatchPorts: []string{"90/udp", "80"}, + ConfirmedDate: "2006-01-02 15:04:05", + ConfirmedDateFormat: time.DateTime, + MaxAge: "8h", + }) + require.NoError(t, err) + require.True(t, res) + require.False(t, age) +} + +func TestPortMatchFilterWithNegativeNetworkMatch(t *testing.T) { + age, res, err := PortMatchFilter(PortMatchFilterInput{ + IncomingPort: "80", + MatchPorts: []string{"800"}, + ConfirmedDate: "2024-01-02 15:04:05", + ConfirmedDateFormat: time.DateTime, + MaxAge: "10000w", + }) + require.NoError(t, err) + require.False(t, res) + require.True(t, age) +} + +func TestPortMatchFilterWithDateAndNoMaxAge(t *testing.T) { + _, res, err := PortMatchFilter(PortMatchFilterInput{ + IncomingPort: "80", + MatchPorts: []string{"800"}, + ConfirmedDate: "2006-01-02 15:04:05", + ConfirmedDateFormat: time.DateTime, + MaxAge: "", + }) + // PortMatch will only attempt to match age if provided, + // so this does not constitute an error despite providing confirmed date and format + require.NoError(t, err) + // returns false as no port match and no age match attempted + require.False(t, res) +} + +func TestPortMatchFilterWithNegativeNetworkMatchPositiveDateMatch(t *testing.T) { + _, res, err := PortMatchFilter(PortMatchFilterInput{ + IncomingPort: "80", + MatchPorts: []string{"800"}, + ConfirmedDate: "2006-01-02 15:04:05", + ConfirmedDateFormat: time.DateTime, + MaxAge: "", + }) + require.NoError(t, err) + require.False(t, res) +} + +func TestDashIfEmptyWithString(t *testing.T) { + empty := "" + notEmpty := "test" + + require.Equal(t, "-", DashIfEmpty(empty)) + require.Equal(t, "-", DashIfEmpty(&empty)) + require.Equal(t, "test", DashIfEmpty(notEmpty)) + require.Equal(t, "test", DashIfEmpty(¬Empty)) +} + +func TestDashIfEmptyWithInt(t *testing.T) { + empty := 0 + notEmpty := 1 + + require.Equal(t, "0", DashIfEmpty(empty)) + require.Equal(t, "1", DashIfEmpty(notEmpty)) +} + +func TestDashIfEmptyWithTime(t *testing.T) { + require.Equal(t, "-", DashIfEmpty(time.Time{})) + require.Equal(t, "2024-04-19 19:00:00", DashIfEmpty(time.Date(2024, time.April, 19, 19, 0, 0, 0, time.UTC))) +} + +func TestPreProcessValueOutput(t *testing.T) { + require.Equal(t, "test", PreProcessValueOutput(&session.Session{Config: session.Config{Global: session.GlobalConfig{MaxValueChars: 0}}}, "test")) + require.Equal(t, "test", PreProcessValueOutput(&session.Session{Config: session.Config{Global: session.GlobalConfig{MaxValueChars: 4}}}, "test")) + require.Equal(t, "test...", PreProcessValueOutput(&session.Session{Config: session.Config{Global: session.GlobalConfig{MaxValueChars: 4}}}, "testing")) +} + +func TestPortMatchFilterWithPortMatchOnly(t *testing.T) { + ageMatch, netMatch, err := PortMatchFilter(PortMatchFilterInput{ + IncomingPort: "80/tcp", + MatchPorts: []string{"tcp"}, + ConfirmedDate: "", + ConfirmedDateFormat: "", + MaxAge: "", + }) + require.NoError(t, err) + require.True(t, ageMatch) + require.True(t, netMatch) +} + +func TestPortNetworkMatchWithoutPortsSpecified(t *testing.T) { + var ports []string + + require.True(t, PortNetworkMatch("80", ports)) + require.True(t, PortNetworkMatch("80/udp", ports)) +} + +func TestPortNetworkMatch(t *testing.T) { + ports := []string{"80", "tcp", "80/tcp"} + + require.True(t, PortNetworkMatch("80", []string{})) + require.True(t, PortNetworkMatch("80", ports)) + require.False(t, PortNetworkMatch("800", ports)) + require.True(t, PortNetworkMatch("tcp", ports)) + require.False(t, PortNetworkMatch("udp", ports)) + require.True(t, PortNetworkMatch("80/tcp", ports)) + require.True(t, PortNetworkMatch("80/udp", ports)) +} + +func TestPortNetworkMatchNonWideTransport(t *testing.T) { + ports := []string{"80", "80/tcp"} + + require.False(t, PortNetworkMatch("50/tcp", ports)) + require.True(t, PortNetworkMatch("80", []string{})) + require.True(t, PortNetworkMatch("80", ports)) + require.False(t, PortNetworkMatch("800", ports)) + require.True(t, PortNetworkMatch("tcp", ports)) + require.False(t, PortNetworkMatch("udp", ports)) + require.True(t, PortNetworkMatch("80/tcp", ports)) + require.True(t, PortNetworkMatch("80/udp", ports)) +} + +func TestPortNetworkMatchNonWidePort(t *testing.T) { + ports := []string{"tcp", "80/tcp"} + require.True(t, PortNetworkMatch("50/tcp", ports)) + require.True(t, PortNetworkMatch("80", []string{})) + require.True(t, PortNetworkMatch("80", ports)) + require.False(t, PortNetworkMatch("800", ports)) + require.True(t, PortNetworkMatch("tcp", ports)) + require.False(t, PortNetworkMatch("udp", ports)) + require.True(t, PortNetworkMatch("80/tcp", ports)) + require.False(t, PortNetworkMatch("80/udp", ports)) +} + +func TestSplitPortTransport(t *testing.T) { + pt := splitPortTransport("80") + require.Equal(t, "80", pt.port) + require.Empty(t, pt.transport) + + pt = splitPortTransport("tcp") + require.Empty(t, pt.port) + require.Equal(t, "tcp", pt.transport) + + pt = splitPortTransport("80/tcp") + require.Equal(t, "80", pt.port) + require.Equal(t, "tcp", pt.transport) + + pt = splitPortTransport("80/udp") + require.Equal(t, "80", pt.port) + require.Equal(t, "udp", pt.transport) +} + +func TestIsPort(t *testing.T) { + require.True(t, isPort("80")) + require.True(t, isPort("800")) + require.False(t, isPort("80000")) + require.False(t, isPort("tcp")) +} diff --git a/providers/ptr/ptr.go b/providers/ptr/ptr.go new file mode 100644 index 0000000..fe87375 --- /dev/null +++ b/providers/ptr/ptr.go @@ -0,0 +1,338 @@ +package ptr + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "log/slog" + "net/netip" + "os" + "strings" + "time" + + "github.com/jonhadfield/ipscout/providers" + + "github.com/miekg/dns" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "ptr" + MaxColumnWidth = 120 + IndentPipeHyphens = " |-----" + portLastModifiedFormat = "2006-01-02T15:04:05+07:00" + ResultTTL = time.Duration(30 * time.Minute) + DefaultNameserver = "9.9.9.9" +) + +type Client struct { + session.Session +} + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating ptr client") + + tc := Client{ + c, + } + + return &tc, nil +} + +type Provider interface { + LoadData() ([]byte, error) + CreateTable([]byte) (*table.Writer, error) +} + +func (c *Client) Enabled() bool { + return c.Session.Providers.PTR.Enabled +} + +func (c *Client) GetConfig() *session.Session { + return &c.Session +} + +func (c *Client) Initialise() error { + if c.Session.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Session.Stats.Mu.Lock() + c.Session.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Session.Stats.Mu.Unlock() + }() + + c.Session.Logger.Debug("initialising ptr client") + + return nil +} + +func (c *Client) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Session.Stats.Mu.Lock() + c.Session.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Session.Stats.Mu.Unlock() + }() + + result, err := fetchData(c.Session) + if err != nil { + return nil, err + } + + c.Session.Logger.Debug("ptr host match data", "size", len(result.Raw)) + + return result.Raw, nil +} + +func (c *Client) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Session.Stats.Mu.Lock() + c.Session.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Session.Stats.Mu.Unlock() + }() + + var findHostData *Data + if err := json.Unmarshal(data, &findHostData); err != nil { + return nil, fmt.Errorf("error unmarshalling ptr data: %w", err) + } + + if findHostData == nil { + return nil, errors.New("no ptr data") + } + + tw := table.NewWriter() + + for x, ptr := range findHostData.RR { + tw.AppendRow(table.Row{fmt.Sprintf("RR[%d]", x+1), ptr}) + } + + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 1, AutoMerge: true}, + }) + + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: true, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + // tw.SetStyle(table.StyleColoredDark) + // tw.Style().Options.DrawBorder = true + tw.SetTitle("PTR | Host: %s", c.Session.Host.String()) + c.Session.Logger.Debug("ptr table created", "host", c.Session.Host.String()) + + return &tw, nil +} + +func loadResponse(c session.Session, nameserver string) (res *HostSearchResult, err error) { + res = &HostSearchResult{} + + target := c.Host.String() + + if nameserver == "" { + nameserver = DefaultNameserver + } + + arpa, err := dns.ReverseAddr(target) + if err != nil { + log.Fatal(err) + } + + dc := dns.Client{} + m := dns.Msg{} + m.SetQuestion(arpa, dns.TypePTR) + + r, _, err := dc.Exchange(&m, nameserver+":53") + if err != nil { + return nil, fmt.Errorf("error querying nameserver: %w", err) + } + + if len(r.Answer) == 0 { + return nil, providers.ErrNoDataFound + } + + for _, ans := range r.Answer { + if ans != nil { + res.Data.RR = append(res.Data.RR, ans.(*dns.PTR)) + } + } + + rd, err := json.Marshal(res.Data) + if err != nil { + return nil, fmt.Errorf("error marshalling ptr data: %w", err) + } + + res.Raw = rd + + return res, nil +} + +func unmarshalResponse(data []byte) (*HostSearchResult, error) { + var res HostSearchResult + + uData := Data{} + if err := json.Unmarshal(data, &uData); err != nil { + return nil, fmt.Errorf("error unmarshalling ptr data: %w", err) + } + + res.Raw = data + res.Data = uData + + return &res, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening ptr file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding ptr file: %w", err) + } + + return res, nil +} + +func (ssr *HostSearchResult) CreateTable() *table.Writer { + tw := table.NewWriter() + + return &tw +} + +func loadTestData(l *slog.Logger) (*HostSearchResult, error) { + tdf, err := loadResultsFile("providers/ptr/testdata/ptr_8_8_8_8_report.json") + if err != nil { + return nil, err + } + + l.Info("ptr match returned from test data", "host", "8.8.8.8") + + return tdf, nil +} + +func fetchData(c session.Session) (*HostSearchResult, error) { + var result *HostSearchResult + + var err error + + if c.UseTestData { + result, err = loadTestData(c.Logger) + if err != nil { + return nil, fmt.Errorf("error loading ptr test data: %w", err) + } + + return result, nil + } + + // load data from cache + cacheKey := fmt.Sprintf("ptr_%s_report.json", strings.ReplaceAll(c.Host.String(), ".", "_")) + + var item *cache.Item + if item, err = cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + if item.Value != nil && len(item.Value) > 0 { + result, err = unmarshalResponse(item.Value) + if err != nil { + return nil, fmt.Errorf("error unmarshalling cached ptr response: %w", err) + } + + c.Logger.Info("ptr response found in cache", "host", c.Host.String()) + + result.Raw = item.Value + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return result, nil + } + } + + result, err = loadResponse(c, "") + if err != nil { + return nil, fmt.Errorf("loading ptr api response: %w", err) + } + + if err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: cacheKey, + Value: result.Raw, + Created: time.Now(), + }, ResultTTL); err != nil { + return nil, fmt.Errorf("error caching ptr response: %w", err) + } + + return result, nil +} + +type Data struct { + Name string `json:"name,omitempty"` + RR []*dns.PTR `json:"rr,omitempty"` + Msg dns.Msg `json:"msg,omitempty"` +} + +type HostSearchResult struct { + Raw []byte `json:"raw,omitempty"` + Data Data `json:"data,omitempty"` +} + +func (data Data) MarshalJSON() ([]byte, error) { + type Header struct { + Name string `dns:"cdomain-name"` + Rrtype uint16 `json:"rrtype,omitempty"` + Class uint16 `json:"class,omitempty"` + Ttl uint32 `json:"ttl,omitempty"` // nolint:revive + Rdlength uint16 `json:"rdlength,omitempty"` + } + + type ptr struct { + Header Header `json:"header,omitempty"` + Ptr string `json:"ptr,omitempty"` + } + + type myData struct { + Name string `json:"name,omitempty"` + RR []*ptr `json:"rr,omitempty"` + } + + var res myData + + res.Name = data.Name + for _, r := range data.RR { + res.RR = append(res.RR, &ptr{ + Header: Header{ + Name: r.Header().Name, + Rrtype: r.Header().Rrtype, + Class: r.Header().Class, + Ttl: r.Header().Ttl, + Rdlength: r.Header().Rdlength, + }, + Ptr: r.Ptr, + }) + } + + out, err := json.Marshal(res) + if err != nil { + return nil, fmt.Errorf("error marshalling ptr data: %w", err) + } + + return out, nil +} diff --git a/providers/ptr/testdata/ptr_8_8_8_8_report.json b/providers/ptr/testdata/ptr_8_8_8_8_report.json new file mode 100644 index 0000000..3cb3334 --- /dev/null +++ b/providers/ptr/testdata/ptr_8_8_8_8_report.json @@ -0,0 +1,34 @@ +{ + "raw": "eyJwdHIiOlt7IkhkciI6eyJOYW1lIjoiOC44LjguOC5pbi1hZGRyLmFycGEuIiwiUnJ0eXBlIjoxMiwiQ2xhc3MiOjEsIlR0bCI6MzUwNDEsIlJkbGVuZ3RoIjoxMn0sIlB0ciI6ImRucy5nb29nbGUuIn1dLCJtc2ciOnsiSWQiOjAsIlJlc3BvbnNlIjpmYWxzZSwiT3Bjb2RlIjowLCJBdXRob3JpdGF0aXZlIjpmYWxzZSwiVHJ1bmNhdGVkIjpmYWxzZSwiUmVjdXJzaW9uRGVzaXJlZCI6ZmFsc2UsIlJlY3Vyc2lvbkF2YWlsYWJsZSI6ZmFsc2UsIlplcm8iOmZhbHNlLCJBdXRoZW50aWNhdGVkRGF0YSI6ZmFsc2UsIkNoZWNraW5nRGlzYWJsZWQiOmZhbHNlLCJSY29kZSI6MCwiUXVlc3Rpb24iOm51bGwsIkFuc3dlciI6bnVsbCwiTnMiOm51bGwsIkV4dHJhIjpudWxsfX0=", + "data": { + "ptr": [ + { + "Hdr": { + "Name": "8.8.8.8.in-addr.arpa.", + "Rrtype": 12, + "Class": 1, + "Ttl": 35041, + "Rdlength": 12 + }, + "Ptr": "dns.google." + } + ], + "msg": { + "Id": 0, + "Response": false, + "Opcode": 0, + "Authoritative": false, + "Truncated": false, + "RecursionDesired": false, + "RecursionAvailable": false, + "Zero": false, + "AuthenticatedData": false, + "CheckingDisabled": false, + "Rcode": 0, + "Question": null, + "Answer": null, + "Ns": null, + "Extra": null + } + } +} diff --git a/providers/shodan/shodan.go b/providers/shodan/shodan.go new file mode 100644 index 0000000..44f53aa --- /dev/null +++ b/providers/shodan/shodan.go @@ -0,0 +1,663 @@ +package shodan + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/netip" + "net/url" + "os" + "strings" + "time" + + "github.com/fatih/color" + "github.com/hashicorp/go-retryablehttp" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "shodan" + APIURL = "https://api.shodan.io" + HostIPPath = "/shodan/host" + MaxColumnWidth = 120 + IndentPipeHyphens = " |-----" + portLastModifiedFormat = "2006-01-02T15:04:05.999999" + ResultTTL = time.Duration(12 * time.Hour) +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +type Provider interface { + LoadData() ([]byte, error) + CreateTable([]byte) (*table.Writer, error) +} + +type ProviderClient struct { + session.Session +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.Shodan.Enabled +} + +func loadAPIResponse(ctx context.Context, c session.Session, apiKey string) (res *HostSearchResult, err error) { + urlPath, err := url.JoinPath(APIURL, HostIPPath, c.Host.String()) + if err != nil { + return nil, fmt.Errorf("failed to create shodan api url path: %w", err) + } + + sURL, err := url.Parse(urlPath) + if err != nil { + panic(err) + } + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + q := sURL.Query() + q.Add("key", apiKey) + sURL.RawQuery = q.Encode() + + req, err := retryablehttp.NewRequestWithContext(ctx, http.MethodGet, sURL.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to make request: %w", err) + } + + if resp.StatusCode == http.StatusNotFound { + return nil, providers.ErrNoMatchFound + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("shodan api request failed: %s", resp.Status) + } + + // read response body + rBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading shodan response: %w", err) + } + + defer resp.Body.Close() + + if rBody == nil { + return nil, providers.ErrNoDataFound + } + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + if err = os.WriteFile(fmt.Sprintf("%s/backups/shodan_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), rBody, 0o600); err != nil { + panic(err) + } + + c.Logger.Debug("backed up shodan response", "host", c.Host.String()) + } + + res, err = unmarshalResponse(rBody) + if err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + res.Raw = rBody + + if res.Raw == nil { + return nil, fmt.Errorf("shodan: %w", providers.ErrNoMatchFound) + } + + return res, nil +} + +func unmarshalResponse(data []byte) (*HostSearchResult, error) { + var res HostSearchResult + + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling shodan data: %w", err) + } + + res.Raw = data + + return &res, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + // get raw data + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("error reading shodan file: %w", err) + } + + // unmarshal data + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening shodan file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding shodan file: %w", err) + } + + res.Raw = raw + + return res, nil +} + +func (ssr *HostSearchResult) CreateTable() *table.Writer { + tw := table.NewWriter() + + return &tw +} + +type Client struct { + Config Config + HTTPClient *retryablehttp.Client +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func fetchData(c session.Session) (*HostSearchResult, error) { + var result *HostSearchResult + + var err error + + if c.UseTestData { + result, err = loadResultsFile("providers/shodan/testdata/shodan_google_dns_resp.json") + if err != nil { + return nil, fmt.Errorf("error loading shodan test data: %w", err) + } + + return result, nil + } + + // load data from cache + cacheKey := fmt.Sprintf("shodan_%s_report.json", strings.ReplaceAll(c.Host.String(), ".", "_")) + + var item *cache.Item + + if item, err = cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + if item.Value != nil && len(item.Value) > 0 { + result, err = unmarshalResponse(item.Value) + if err != nil { + return nil, fmt.Errorf("error unmarshalling cached shodan response: %w", err) + } + + c.Logger.Info("shodan response found in cache", "host", c.Host.String()) + + result.Raw = item.Value + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return result, nil + } + } + + result, err = loadAPIResponse(context.Background(), c, c.Providers.Shodan.APIKey) + if err != nil { + return nil, fmt.Errorf("loading shodan api response: %w", err) + } + + if err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: cacheKey, + Value: result.Raw, + Created: time.Now(), + }, ResultTTL); err != nil { + return nil, fmt.Errorf("error caching shodan response: %w", err) + } + + return result, nil +} + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising shodan client") + + if c.Providers.Shodan.APIKey == "" && !c.UseTestData { + return fmt.Errorf("shodan provider api key not set") + } + + return nil +} + +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := fetchData(c.Session) + if err != nil { + return nil, err + } + + c.Logger.Debug("shodan host match data", "size", len(result.Raw)) + + return result.Raw, nil +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var result *HostSearchResult + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("error unmarshalling shodan data: %w", err) + } + + if result == nil { + return nil, nil + } + + tw := table.NewWriter() + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 1, AutoMerge: true}, + }) + + var rows []table.Row + + tw.AppendRow(table.Row{"WHOIS", providers.DashIfEmpty(result.LastUpdate)}) + tw.AppendRow(table.Row{" - Org", providers.DashIfEmpty(result.Org)}) + tw.AppendRow(table.Row{" - Country", fmt.Sprintf("%s (%s)", providers.DashIfEmpty(result.CountryName), providers.DashIfEmpty(strings.ToUpper(result.CountryCode)))}) + tw.AppendRow(table.Row{" - Region", providers.DashIfEmpty(result.RegionCode)}) + tw.AppendRow(table.Row{" - City", providers.DashIfEmpty(result.City)}) + + var filteredPorts int + + for _, dr := range result.Data { + var ok bool + + _, ok, err := providers.PortMatchFilter(providers.PortMatchFilterInput{ + IncomingPort: fmt.Sprintf("%d/%s", dr.Port, dr.Transport), + MatchPorts: c.Config.Global.Ports, + ConfirmedDate: dr.Timestamp, + ConfirmedDateFormat: portLastModifiedFormat, + MaxAge: c.Config.Global.MaxAge, + }) + if err != nil { + return nil, fmt.Errorf("error checking port match filter: %w", err) + } + + if !ok { + filteredPorts++ + + continue + } + } + + if filteredPorts > 0 { + rows = append(rows, table.Row{"Ports", fmt.Sprintf("%d (%d filtered)", len(result.Data), filteredPorts)}) + } else { + rows = append(rows, table.Row{"Ports", len(result.Data)}) + } + + if len(result.Data) > 0 { + for _, dr := range result.Data { + _, ok, err := providers.PortMatchFilter(providers.PortMatchFilterInput{ + IncomingPort: fmt.Sprintf("%d/%s", dr.Port, dr.Transport), + MatchPorts: c.Config.Global.Ports, + ConfirmedDate: dr.Timestamp, + ConfirmedDateFormat: portLastModifiedFormat, + MaxAge: c.Config.Global.MaxAge, + }) + if err != nil { + return nil, fmt.Errorf("error checking port match filter: %w", err) + } + + if !ok { + filteredPorts++ + + continue + } + + rows = append(rows, table.Row{"", color.CyanString("%d/%s", dr.Port, dr.Transport)}) + if len(dr.Domains) > 0 { + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s Domains: %s", IndentPipeHyphens, strings.Join(dr.Domains, ", ")), + }) + } + + if dr.Timestamp != "" { + rows = append(rows, table.Row{"", fmt.Sprintf("%s Timestamp: %s", IndentPipeHyphens, dr.Timestamp)}) + } + + if len(dr.Hostnames) > 0 { + rows = append(rows, table.Row{"", fmt.Sprintf("%s HostNames: %s", IndentPipeHyphens, strings.Join(dr.Hostnames, ", "))}) + } + + if dr.SSH.Type != "" { + rows = append(rows, table.Row{"", fmt.Sprintf("%s SSH", IndentPipeHyphens)}) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sType: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.SSH.Type), + }) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sCipher: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.SSH.Cipher), + }) + } + + if dr.HTTP.Status != 0 { + rows = append(rows, table.Row{"", fmt.Sprintf("%s HTTP", IndentPipeHyphens)}) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sStatus: %d", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.HTTP.Status), + }) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sTitle: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.HTTP.Title), + }) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sServer: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.HTTP.Server), + }) + } + + if len(dr.Ssl.Versions) > 0 { + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s SSL", IndentPipeHyphens), + }) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sIssuer: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.Ssl.Cert.Issuer.Cn), + }) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sSubject: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.Ssl.Cert.Subject.Cn), + }) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sVersions: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), strings.Join(dr.Ssl.Versions, ", ")), + }) + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sExpires: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.Ssl.Cert.Expires), + }) + } + + if dr.DNS.ResolverHostname != nil { + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s DNS", + IndentPipeHyphens), + }) + + if dr.DNS.ResolverHostname != "" { + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sResolver Hostname: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.DNS.ResolverHostname), + }) + } + + if dr.DNS.Software != nil { + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sResolver Software: %s", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.DNS.Software), + }) + } + + rows = append(rows, table.Row{ + "", + fmt.Sprintf("%s%sRecursive: %t", + IndentPipeHyphens, strings.Repeat(" ", 2*c.Config.Global.IndentSpaces), dr.DNS.Recursive), + }) + } + } + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: true, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + } + + tw.SetAutoIndex(false) + // tw.SetStyle(table.StyleColoredDark) + // tw.Style().Options.DrawBorder = true + tw.SetTitle("SHODAN | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("SHODAN | Host: %s", result.Data[0].IPStr) + } + + c.Logger.Debug("shodan table created", "host", c.Host.String()) + + return &tw, nil +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating shodan client") + + tc := &ProviderClient{ + c, + } + + return tc, nil +} + +func (c *Client) GetConfig() *session.Session { + return &c.Config.Session +} + +func (c *Client) GetData() (result *HostSearchResult, err error) { + result, err = loadResultsFile("shodan/testdata/shodan_google_dns_resp.json") + if err != nil { + return nil, err + } + + return result, nil +} + +type HostSearchResultData struct { + Hash int `json:"hash"` + Opts struct{} `json:"opts,omitempty"` + Timestamp string `json:"timestamp"` + Isp string `json:"isp"` + Data string `json:"data"` + Shodan struct { + Region string `json:"region"` + Module string `json:"module"` + Ptr bool `json:"ptr"` + Options struct{} `json:"options"` + ID string `json:"id"` + Crawler string `json:"crawler"` + } `json:"_shodan,omitempty"` + Port int `json:"port"` + Hostnames []string `json:"hostnames"` + Location struct { + City string `json:"city"` + RegionCode string `json:"region_code"` + AreaCode any `json:"area_code"` + Longitude float64 `json:"longitude"` + CountryName string `json:"country_name"` + CountryCode string `json:"country_code"` + Latitude float64 `json:"latitude"` + } `json:"location"` + DNS struct { + ResolverHostname any `json:"resolver_hostname"` + Recursive bool `json:"recursive"` + ResolverID any `json:"resolver_id"` + Software any `json:"software"` + } `json:"dns,omitempty"` + SSH struct { + Hassh string `json:"hassh"` + Fingerprint string `json:"fingerprint"` + Mac string `json:"mac"` + Cipher string `json:"cipher"` + Key string `json:"key"` + Kex struct { + Languages []string `json:"languages"` + ServerHostKeyAlgorithms []string `json:"server_host_key_algorithms"` + EncryptionAlgorithms []string `json:"encryption_algorithms"` + KexFollows bool `json:"kex_follows"` + Unused int `json:"unused"` + KexAlgorithms []string `json:"kex_algorithms"` + CompressionAlgorithms []string `json:"compression_algorithms"` + MacAlgorithms []string `json:"mac_algorithms"` + } `json:"kex"` + Type string `json:"type"` + } `json:"ssh"` + HTTP struct { + Status int `json:"status"` + RobotsHash int `json:"robots_hash"` + Redirects []struct { + Host string `json:"host"` + Data string `json:"data"` + Location string `json:"location"` + } + SecurityTxt string `json:"security_txt"` + Title string `json:"title"` + SitemapHash string `json:"sitemap_hash"` + HTMLHash int `json:"html_hash"` + Robots string `json:"robots"` + Favicon struct { + Hash int `json:"hash"` + Data string `json:"data"` + Location string `json:"location"` + } `json:"favicon"` + HeadersHash int `json:"headers_hash"` + Host string `json:"host"` + HTML string `json:"html"` + Location string `json:"location"` + Components struct{} `json:"components"` + Server string `json:"server"` + Sitemap string `json:"sitemap"` + SecurityTxtHash int `json:"securitytxt_hash"` + } `json:"http,omitempty"` + IP int `json:"ip"` + Domains []string `json:"domains"` + Org string `json:"org"` + Os any `json:"os"` + Asn string `json:"asn"` + Transport string `json:"transport"` + IPStr string `json:"ip_str"` + Ssl struct { + ChainSha256 []string `json:"chain_sha256"` + Jarm string `json:"jarm"` + Chain []string `json:"chain"` + Dhparams any `json:"dhparams"` + Versions []string `json:"versions"` + AcceptableCas []any `json:"acceptable_cas"` + Tlsext []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"tlsext"` + Ja3S string `json:"ja3s"` + Cert struct { + SigAlg string `json:"sig_alg"` + Issued string `json:"issued"` + Expires string `json:"expires"` + Expired bool `json:"expired"` + Version int `json:"version"` + Extensions []struct { + Critical bool `json:"critical,omitempty"` + Data string `json:"data"` + Name string `json:"name"` + } `json:"extensions"` + Fingerprint struct { + Sha256 string `json:"sha256"` + Sha1 string `json:"sha1"` + } `json:"fingerprint"` + Serial json.RawMessage `json:"serial"` + Subject struct { + Cn string `json:"CN"` + } `json:"subject"` + Pubkey struct { + Type string `json:"type"` + Bits int `json:"bits"` + } `json:"pubkey"` + Issuer struct { + C string `json:"C"` + Cn string `json:"CN"` + O string `json:"O"` + } `json:"issuer"` + } `json:"cert"` + Cipher struct { + Version string `json:"version"` + Bits int `json:"bits"` + Name string `json:"name"` + } `json:"cipher"` + Trust struct { + Revoked bool `json:"revoked"` + Browser any `json:"browser"` + } `json:"trust"` + HandshakeStates []string `json:"handshake_states"` + Alpn []any `json:"alpn"` + Ocsp struct{} `json:"ocsp"` + } `json:"ssl,omitempty"` +} + +type HostSearchResult struct { + Raw []byte `json:"raw"` + City string `json:"city"` + RegionCode string `json:"region_code"` + Os any `json:"os"` + Tags []any `json:"tags"` + IP int `json:"ip"` + Isp string `json:"isp"` + AreaCode any `json:"area_code"` + Longitude float64 `json:"longitude"` + LastUpdate string `json:"last_update"` + Ports []int `json:"ports"` + Latitude float64 `json:"latitude"` + Hostnames []string `json:"hostnames"` + CountryCode string `json:"country_code"` + CountryName string `json:"country_name"` + Domains []string `json:"domains"` + Org string `json:"org"` + Data []HostSearchResultData + Asn string `json:"asn"` + IPStr string `json:"ip_str"` + Error string `json:"error"` +} diff --git a/providers/shodan/shodan_test.go b/providers/shodan/shodan_test.go new file mode 100644 index 0000000..37d545c --- /dev/null +++ b/providers/shodan/shodan_test.go @@ -0,0 +1,153 @@ +package shodan + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestShodanHostDNSQuery(t *testing.T) { + t.Parallel() + + jf, err := os.Open("testdata/shodan_google_dns_resp.json") + require.NoError(t, err) + defer jf.Close() + + decoder := json.NewDecoder(jf) + + var sr HostSearchResult + err = decoder.Decode(&sr) + require.NoError(t, err) + + require.Equal(t, "Mountain View", sr.City) + require.Equal(t, "CA", sr.RegionCode) + require.Nil(t, sr.Os) + require.Empty(t, sr.Tags) + require.Equal(t, 134744072, sr.IP) + require.Equal(t, "Google LLC", sr.Isp) + require.Nil(t, sr.AreaCode) + require.Equal(t, -122.0775, sr.Longitude) + require.Equal(t, "2024-04-28T12:50:55.650683", sr.LastUpdate) + require.Equal(t, []int{443, 53}, sr.Ports) + require.Equal(t, 37.4056, sr.Latitude) + require.Equal(t, []string{"dns.google"}, sr.Hostnames) + require.Equal(t, "US", sr.CountryCode) + require.Equal(t, "United States", sr.CountryName) + require.Equal(t, []string{"dns.google"}, sr.Domains) + require.Equal(t, "Google LLC", sr.Org) + require.Equal(t, -553166942, sr.Data[0].Hash) + require.Empty(t, sr.Data[0].Opts) + require.Equal(t, "2024-04-28T09:18:08.964972", sr.Data[0].Timestamp) + require.Equal(t, "Google LLC", sr.Data[0].Isp) + require.Equal(t, "\nRecursion: enabled", sr.Data[0].Data) + require.NotNil(t, sr.Data[0].Shodan) + require.Equal(t, "na", sr.Data[0].Shodan.Region) + require.Equal(t, "dns-tcp", sr.Data[0].Shodan.Module) + require.True(t, sr.Data[0].Shodan.Ptr) + require.Empty(t, sr.Data[0].Shodan.Options) + require.Equal(t, "750fb3c2-8743-44db-b1ff-25e9028e4345", sr.Data[0].Shodan.ID) + require.Equal(t, "d89a99e2d29a6c9d44f01585e74209a6c7d174c7", sr.Data[0].Shodan.Crawler) + require.Equal(t, 53, sr.Data[0].Port) + require.Equal(t, []string{"dns.google"}, sr.Data[0].Hostnames) + require.Equal(t, "2024-04-28T09:18:08.964972", sr.Data[0].Timestamp) + require.NotEmpty(t, sr.Data[0].Location) + require.Equal(t, "Mountain View", sr.Data[0].Location.City) + require.Equal(t, "CA", sr.Data[0].Location.RegionCode) + require.Nil(t, sr.Data[0].Location.AreaCode) + require.Equal(t, -122.0775, sr.Data[0].Location.Longitude) + require.Equal(t, 37.4056, sr.Data[0].Location.Latitude) + require.Equal(t, "United States", sr.Data[0].Location.CountryName) + require.Equal(t, "US", sr.Data[0].Location.CountryCode) + require.Nil(t, sr.Data[0].DNS.ResolverHostname) + require.True(t, sr.Data[0].DNS.Recursive) + require.Nil(t, sr.Data[0].DNS.ResolverID) + require.Nil(t, sr.Data[0].DNS.Software) + require.Equal(t, 134744072, sr.Data[0].IP) + require.Equal(t, "dns.google", sr.Data[0].Domains[0]) + require.Equal(t, "Google LLC", sr.Data[0].Org) + require.Nil(t, sr.Data[0].Os) + require.Equal(t, "AS15169", sr.Data[0].Asn) + require.Equal(t, "tcp", sr.Data[0].Transport) + require.Equal(t, "8.8.8.8", sr.Data[0].IPStr) + // /// + require.Equal(t, -553166942, sr.Data[1].Hash) + require.Empty(t, sr.Data[1].Opts) + require.Equal(t, "2024-04-28T12:50:55.650683", sr.Data[1].Timestamp) + require.Equal(t, "Google LLC", sr.Data[1].Isp) + require.Equal(t, "\nRecursion: enabled", sr.Data[1].Data) + require.NotNil(t, sr.Data[1].Shodan) + require.Equal(t, "eu", sr.Data[1].Shodan.Region) + require.Equal(t, "dns-udp", sr.Data[1].Shodan.Module) + require.True(t, sr.Data[1].Shodan.Ptr) + require.Empty(t, sr.Data[1].Shodan.Options) + require.Equal(t, "8a5051c2-01a9-44fc-850a-25335e57326e", sr.Data[1].Shodan.ID) + require.Equal(t, "487814a778c983e2dcef234806292d88c5cbf3ec", sr.Data[1].Shodan.Crawler) + require.Equal(t, 53, sr.Data[1].Port) + require.Equal(t, []string{"dns.google"}, sr.Data[1].Hostnames) + require.Equal(t, "2024-04-28T12:50:55.650683", sr.Data[1].Timestamp) + require.NotEmpty(t, sr.Data[1].Location) + require.Equal(t, "Mountain View", sr.Data[1].Location.City) + require.Equal(t, "CA", sr.Data[1].Location.RegionCode) + require.Nil(t, sr.Data[1].Location.AreaCode) + require.Equal(t, -122.0775, sr.Data[1].Location.Longitude) + require.Equal(t, 37.4056, sr.Data[1].Location.Latitude) + require.Equal(t, "United States", sr.Data[1].Location.CountryName) + require.Equal(t, "US", sr.Data[1].Location.CountryCode) + require.Nil(t, sr.Data[1].DNS.ResolverHostname) + require.True(t, sr.Data[1].DNS.Recursive) + require.Nil(t, sr.Data[1].DNS.ResolverID) + require.Nil(t, sr.Data[1].DNS.Software) + require.Equal(t, 134744072, sr.Data[1].IP) + require.Equal(t, "dns.google", sr.Data[1].Domains[0]) + require.Equal(t, "Google LLC", sr.Data[1].Org) + require.Nil(t, sr.Data[1].Os) + require.Equal(t, "AS15169", sr.Data[1].Asn) + require.Equal(t, "udp", sr.Data[1].Transport) + require.Equal(t, "8.8.8.8", sr.Data[1].IPStr) + // /// + require.Equal(t, -1020052518, sr.Data[2].Hash) + require.Empty(t, sr.Data[2].Opts) + require.Equal(t, "2024-04-28T09:25:24.474444", sr.Data[2].Timestamp) + require.Equal(t, "Google LLC", sr.Data[2].Isp) + require.Equal(t, "HTTP/1.1 200 OK\r\nContent-Security-Policy: object-src 'none';base-uri 'self';script-src 'nonce-iRRB3YCy7UKnm1gs9cWAWA' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'\r\nStrict-Transport-Security: max-age=31536000; includeSubDomains; preload\r\nX-Content-Type-Options: nosniff\r\nContent-Type: text/html; charset=UTF-8\r\nDate: Sun, 28 Apr 2024 09:25:24 GMT\r\nServer: scaffolding on HTTPServer2\r\nCache-Control: private\r\nX-XSS-Protection: 0\r\nX-Frame-Options: SAMEORIGIN\r\nAlt-Svc: h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000\r\nAccept-Ranges: none\r\nVary: Accept-Encoding\r\nTransfer-Encoding: chunked\r\n\r\n", sr.Data[2].Data) + require.NotNil(t, sr.Data[2].Shodan) + require.Equal(t, "na", sr.Data[2].Shodan.Region) + require.Equal(t, "https", sr.Data[2].Shodan.Module) + require.True(t, sr.Data[2].Shodan.Ptr) + require.Empty(t, sr.Data[2].Shodan.Options) + require.Equal(t, "80b7f761-7a88-4a9b-acd0-dfb1f260c5aa", sr.Data[2].Shodan.ID) + require.Equal(t, "ea0db9f26a43b503e537983ddb14b4f59b0f8632", sr.Data[2].Shodan.Crawler) + require.Equal(t, 443, sr.Data[2].Port) + require.Equal(t, []string{"dns.google"}, sr.Data[2].Hostnames) + require.Equal(t, "2024-04-28T09:25:24.474444", sr.Data[2].Timestamp) + require.NotEmpty(t, sr.Data[2].Location) + require.Equal(t, "Mountain View", sr.Data[2].Location.City) + require.Equal(t, "CA", sr.Data[2].Location.RegionCode) + require.Nil(t, sr.Data[2].Location.AreaCode) + require.Equal(t, -122.0775, sr.Data[2].Location.Longitude) + require.Equal(t, 37.4056, sr.Data[2].Location.Latitude) + require.Equal(t, "United States", sr.Data[2].Location.CountryName) + require.Equal(t, "US", sr.Data[2].Location.CountryCode) + require.Equal(t, 200, sr.Data[2].HTTP.Status) + require.Empty(t, sr.Data[2].HTTP.RobotsHash) + require.Equal(t, "help.emarketer.com", sr.Data[2].HTTP.Redirects[0].Host) + require.Equal(t, "HTTP/1.1 302 Found\r\nX-Content-Type-Options: nosniff\r\nAccess-Control-Allow-Origin: *\r\nLocation: https://dns.google/\r\nDate: Sun, 28 Apr 2024 09:25:22 GMT\r\nContent-Type: text/html; charset=UTF-8\r\nServer: HTTP server (unknown)\r\nContent-Length: 216\r\nX-XSS-Protection: 0\r\nX-Frame-Options: SAMEORIGIN\r\nAlt-Svc: h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000\r\n\r\n", sr.Data[2].HTTP.Redirects[0].Data) + require.Equal(t, "/", sr.Data[2].HTTP.Location) + require.Equal(t, "", sr.Data[2].HTTP.SecurityTxt) + require.Equal(t, "Google Public DNS", sr.Data[2].HTTP.Title) + require.Equal(t, "", sr.Data[2].HTTP.SitemapHash) + require.Equal(t, -82718941, sr.Data[2].HTTP.HTMLHash) + require.Empty(t, sr.Data[2].HTTP.Robots) + require.Equal(t, 56641965, sr.Data[2].HTTP.Favicon.Hash) + require.Equal(t, "iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAMAAAD04JH5AAABa1BMVEUAAAA0p100qFM0qFY1pW81\npmg1p141p2M2pHw2pH02pXU3oYs3ook4n5k4oY85np86m6s6m6w7mLo7mLs7mbY7mbc7p1M8lcQ8\nlsM+ks4+ks8+oYo/j9g/j9k/kNg/p1NAiudAi+VAjOFAjeBBh+9BiO1BietChfRChvJEhfRIhvNI\npVJJonNKhPJMhPJMpVJRkcdTh/BXo1FZie5ag+xeie1fhORmlZ5on1Bsnk90juV4m05+jOCAmE2D\ngMuMkdmNk0yQkkuQlNeWltSXj0qliEiubLqvgka0f0W1lbm4m7e7mrS9eUPBdkLHcEDOm5vSZj3S\noJXTZD3UoJLZXTvdWTrgonzhUzjkTjflp3LmSjfoRjboRzbqQzXqRDXqRTXqRjXqRzXsYDHtYjHt\nYzHtqlruqlfveSzveizweyzwr0/yjSbyjib1nSD1nh/1nx/3tSz4rBf4rRb4tyb6uQn6uQr6uhn7\nugj7vAU/At79AAAAAXRSTlMAQObYZgAAArlJREFUeNrt2PVz1EAYxvFlcSnFXQoUXgoUdyvu7ra4\nFy22fz53NJ27Xmx38z7vwpHnl860yXw/c9PJJVGqXuXp9KLGBRG6bHHrWIJ2XeQ8iKB1VIH2Xuw+\nryCkr7dH7vcSRe3PWE9cgqD+xJVETIKgvp5PxCQI68/eyAUI60/uI+IRhPX1MmICBPbnEjEJnC52\nqd9O7+cCuF9p2/8wZS3hAUXHLiZiEnh+zSR/7CE4oPD4SRvYAP7fss1L8AoCA0pOWUjEJQi5y9DL\nB+MC1F4CA8pOIj5A4G1elwNUDfjfAaoGhAGUJAB8Ja4BQX3JL0PO/l8K0P59QQBv31tAJAPQYn2v\n5yIC9D2eDImkAe0EIlTf4fXEPiJkvxSwZTe2XyZIXgUC+yWCefh+oWDmoEBfHXd5FQjsK3V5Ww5g\nqUxfqdtHMvtzhPJKmYfnVqf70/qF+qa5Gwc6+xPWyfRNsgenOwBLZP4BTGtXdrb3ewYkLgFm3O4e\na/XTrwIRAtOxx5c2j12CVxHhBSa9W4dGAQuI8AKTtUdnFzX6s7YSXmBydm2/nrqG8ACTu3unjhLB\nBaZoRHBBYd+caT/08CYAoDCfPvz1navnh3ZwCnzqf2Yb+/Kq6djFAfDNJ4LRfXt//+bFE3uqCAL6\n4whjjusXTh4cYASUnWaz9lOunyOwcn0swOlUIEDJASp8ANkCwQ+gOwCV+pmCbgCoGiAIqNiPDrA1\nIDLAdinACPb/bQDPHRnz/ZgggKkffEdmsQAj1g8EWD6ACngutJx9/ydTa4UA2QTL3S9+O+JThwBa\nisbPr5i+gyDZR1DfGTAM6jsL3sQGPEf1XQVPfqD6roIRWN9R8AnXdxMM4/JugrfIvovgBbTvIHj6\nHZl3EYxg++WEz+B8KeEDPF8ieIfPFxteStSLDM9+ydTzEapelf0GmFdLbOXMqToAAAAASUVORK5C\nYII=\n", sr.Data[2].HTTP.Favicon.Data) + require.Equal(t, "https://dns.google:443/static/93dd5954/favicon.png", sr.Data[2].HTTP.Favicon.Location) + require.Equal(t, 134744072, sr.Data[2].IP) + require.Equal(t, "dns.google", sr.Data[2].Domains[0]) + require.Equal(t, "Google LLC", sr.Data[2].Org) + require.Nil(t, sr.Data[2].Os) + require.Equal(t, "AS15169", sr.Data[2].Asn) + require.Equal(t, "tcp", sr.Data[2].Transport) + require.Equal(t, "8.8.8.8", sr.Data[2].IPStr) +} diff --git a/providers/shodan/testdata/shodan_google_dns_resp.json b/providers/shodan/testdata/shodan_google_dns_resp.json new file mode 100644 index 0000000..6eda6c6 --- /dev/null +++ b/providers/shodan/testdata/shodan_google_dns_resp.json @@ -0,0 +1 @@ +{"region_code": "CA", "tags": [], "ip": 134744072, "area_code": null, "domains": ["dns.google"], "hostnames": ["dns.google"], "country_code": "US", "org": "Google LLC", "data": [{"asn": "AS15169", "hash": -553166942, "os": null, "timestamp": "2024-04-28T09:18:08.964972", "isp": "Google LLC", "transport": "tcp", "_shodan": {"region": "na", "module": "dns-tcp", "ptr": true, "options": {}, "id": "750fb3c2-8743-44db-b1ff-25e9028e4345", "crawler": "d89a99e2d29a6c9d44f01585e74209a6c7d174c7"}, "hostnames": ["dns.google"], "location": {"city": "Mountain View", "region_code": "CA", "area_code": null, "longitude": -122.0775, "latitude": 37.4056, "country_code": "US", "country_name": "United States"}, "dns": {"software": null, "recursive": true, "resolver_id": null, "resolver_hostname": null}, "ip": 134744072, "domains": ["dns.google"], "org": "Google LLC", "data": "\nRecursion: enabled", "port": 53, "opts": {}, "ip_str": "8.8.8.8"}, {"asn": "AS15169", "hash": -553166942, "os": null, "timestamp": "2024-04-28T12:50:55.650683", "isp": "Google LLC", "transport": "udp", "_shodan": {"region": "eu", "module": "dns-udp", "ptr": true, "options": {}, "id": "8a5051c2-01a9-44fc-850a-25335e57326e", "crawler": "487814a778c983e2dcef234806292d88c5cbf3ec"}, "hostnames": ["dns.google"], "location": {"city": "Mountain View", "region_code": "CA", "area_code": null, "longitude": -122.0775, "latitude": 37.4056, "country_code": "US", "country_name": "United States"}, "dns": {"software": null, "recursive": true, "resolver_id": null, "resolver_hostname": null}, "ip": 134744072, "domains": ["dns.google"], "org": "Google LLC", "data": "\nRecursion: enabled", "port": 53, "opts": {"raw": "34ef81820001000000000000026964067365727665720000100003"}, "ip_str": "8.8.8.8"}, {"hash": -1020052518, "asn": "AS15169", "http": {"status": 200, "robots_hash": null, "redirects": [{"host": "help.emarketer.com", "data": "HTTP/1.1 302 Found\r\nX-Content-Type-Options: nosniff\r\nAccess-Control-Allow-Origin: *\r\nLocation: https://dns.google/\r\nDate: Sun, 28 Apr 2024 09:25:22 GMT\r\nContent-Type: text/html; charset=UTF-8\r\nServer: HTTP server (unknown)\r\nContent-Length: 216\r\nX-XSS-Protection: 0\r\nX-Frame-Options: SAMEORIGIN\r\nAlt-Svc: h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000\r\n\r\n", "location": "/"}], "securitytxt": null, "title": "Google Public DNS", "sitemap_hash": null, "robots": null, "favicon": {"data": "iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAMAAAD04JH5AAABa1BMVEUAAAA0p100qFM0qFY1pW81\npmg1p141p2M2pHw2pH02pXU3oYs3ook4n5k4oY85np86m6s6m6w7mLo7mLs7mbY7mbc7p1M8lcQ8\nlsM+ks4+ks8+oYo/j9g/j9k/kNg/p1NAiudAi+VAjOFAjeBBh+9BiO1BietChfRChvJEhfRIhvNI\npVJJonNKhPJMhPJMpVJRkcdTh/BXo1FZie5ag+xeie1fhORmlZ5on1Bsnk90juV4m05+jOCAmE2D\ngMuMkdmNk0yQkkuQlNeWltSXj0qliEiubLqvgka0f0W1lbm4m7e7mrS9eUPBdkLHcEDOm5vSZj3S\noJXTZD3UoJLZXTvdWTrgonzhUzjkTjflp3LmSjfoRjboRzbqQzXqRDXqRTXqRjXqRzXsYDHtYjHt\nYzHtqlruqlfveSzveizweyzwr0/yjSbyjib1nSD1nh/1nx/3tSz4rBf4rRb4tyb6uQn6uQr6uhn7\nugj7vAU/At79AAAAAXRSTlMAQObYZgAAArlJREFUeNrt2PVz1EAYxvFlcSnFXQoUXgoUdyvu7ra4\nFy22fz53NJ27Xmx38z7vwpHnl860yXw/c9PJJVGqXuXp9KLGBRG6bHHrWIJ2XeQ8iKB1VIH2Xuw+\nryCkr7dH7vcSRe3PWE9cgqD+xJVETIKgvp5PxCQI68/eyAUI60/uI+IRhPX1MmICBPbnEjEJnC52\nqd9O7+cCuF9p2/8wZS3hAUXHLiZiEnh+zSR/7CE4oPD4SRvYAP7fss1L8AoCA0pOWUjEJQi5y9DL\nB+MC1F4CA8pOIj5A4G1elwNUDfjfAaoGhAGUJAB8Ja4BQX3JL0PO/l8K0P59QQBv31tAJAPQYn2v\n5yIC9D2eDImkAe0EIlTf4fXEPiJkvxSwZTe2XyZIXgUC+yWCefh+oWDmoEBfHXd5FQjsK3V5Ww5g\nqUxfqdtHMvtzhPJKmYfnVqf70/qF+qa5Gwc6+xPWyfRNsgenOwBLZP4BTGtXdrb3ewYkLgFm3O4e\na/XTrwIRAtOxx5c2j12CVxHhBSa9W4dGAQuI8AKTtUdnFzX6s7YSXmBydm2/nrqG8ACTu3unjhLB\nBaZoRHBBYd+caT/08CYAoDCfPvz1navnh3ZwCnzqf2Yb+/Kq6djFAfDNJ4LRfXt//+bFE3uqCAL6\n4whjjusXTh4cYASUnWaz9lOunyOwcn0swOlUIEDJASp8ANkCwQ+gOwCV+pmCbgCoGiAIqNiPDrA1\nIDLAdinACPb/bQDPHRnz/ZgggKkffEdmsQAj1g8EWD6ACngutJx9/ydTa4UA2QTL3S9+O+JThwBa\nisbPr5i+gyDZR1DfGTAM6jsL3sQGPEf1XQVPfqD6roIRWN9R8AnXdxMM4/JugrfIvovgBbTvIHj6\nHZl3EYxg++WEz+B8KeEDPF8ieIfPFxteStSLDM9+ydTzEapelf0GmFdLbOXMqToAAAAASUVORK5C\nYII=\n", "hash": 56641965, "location": "https://dns.google:443/static/93dd5954/favicon.png"}, "headers_hash": 818523308, "host": "dns.google", "html": "\n Google Public DNS
Public DNS
", "location": "/", "components": {}, "securitytxt_hash": null, "server": "scaffolding on HTTPServer2", "sitemap": null, "html_hash": -82718941}, "os": null, "timestamp": "2024-04-28T09:25:24.474444", "isp": "Google LLC", "transport": "tcp", "_shodan": {"region": "na", "module": "https", "ptr": true, "options": {"hostname": "help.emarketer.com", "scan": "WMdePZTNmeWcb4ml"}, "id": "80b7f761-7a88-4a9b-acd0-dfb1f260c5aa", "crawler": "ea0db9f26a43b503e537983ddb14b4f59b0f8632"}, "ssl": {"chain_sha256": ["5a0b6f72280c3e1371b6c58be4649c7bf0318e5eb860aca55815045b832cd2a4", "23ecb03eec17338c4e33a6b48a41dc3cda12281bbc3ff813c0589d6cc2387522", "3ee0278df71fa3c125c4cd487f01d774694e6fc57e0cd94c24efd769133918e5"], "jarm": "29d3fd00029d29d00042d43d00041d598ac0c1012db967bb1ad0ff2491b3ae", "chain": ["-----BEGIN CERTIFICATE-----\nMIIF4jCCBMqgAwIBAgIQSOHA7KHUOOEQhUzBPoQ3czANBgkqhkiG9w0BAQsFADBG\nMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM\nQzETMBEGA1UEAxMKR1RTIENBIDFDMzAeFw0yNDA0MDgwNzM0MDZaFw0yNDA3MDEw\nNzM0MDVaMBUxEzARBgNVBAMTCmRucy5nb29nbGUwggEiMA0GCSqGSIb3DQEBAQUA\nA4IBDwAwggEKAoIBAQDEdRRznL18x+idDVqTILeS8t55zY8TBFxha9y6yycLX1n1\n6EDcgmHTy+sH35nb/vXOWG9EcGSqB9Flo/E5fEq9pAQ3+YpvBCavQk/HwDTUtlg4\niltU8QLvhQNFTSKv82wUPajz+BWORzYwAoWOK5p86uagS8mgrdKos/ObxYCV/T4P\njzyOSBfeqptXk+n3GzuooUUXfxYZXPDntCOkagPUgJ04bgB87m9ZO0J040VbZtbJ\nUBd+EVoLhqd44vdvKuShuFggQPcCUwuPdZmO0lSGlhGcVveI69hCPlm6jAK7z4HZ\n9kHPTtNFXwKxJdA7U+HsBpOyla4WpAbGIj0idj5TAgMBAAGjggL7MIIC9zAOBgNV\nHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAd\nBgNVHQ4EFgQUeQqPTxzWkZVIqOz1OzYLcDnVUJ4wHwYDVR0jBBgwFoAUinR/r4XN\n7pXNPZzQ4kYU83E1HScwagYIKwYBBQUHAQEEXjBcMCcGCCsGAQUFBzABhhtodHRw\nOi8vb2NzcC5wa2kuZ29vZy9ndHMxYzMwMQYIKwYBBQUHMAKGJWh0dHA6Ly9wa2ku\nZ29vZy9yZXBvL2NlcnRzL2d0czFjMy5kZXIwgawGA1UdEQSBpDCBoYIKZG5zLmdv\nb2dsZYIOZG5zLmdvb2dsZS5jb22CECouZG5zLmdvb2dsZS5jb22CCzg4ODguZ29v\nZ2xlghBkbnM2NC5kbnMuZ29vZ2xlhwQICAgIhwQICAQEhxAgAUhgSGAAAAAAAAAA\nAIiIhxAgAUhgSGAAAAAAAAAAAIhEhxAgAUhgSGAAAAAAAAAAAGRkhxAgAUhgSGAA\nAAAAAAAAAABkMCEGA1UdIAQaMBgwCAYGZ4EMAQIBMAwGCisGAQQB1nkCBQMwPAYD\nVR0fBDUwMzAxoC+gLYYraHR0cDovL2NybHMucGtpLmdvb2cvZ3RzMWMzL1FxRnhi\naTlNNDhjLmNybDCCAQQGCisGAQQB1nkCBAIEgfUEgfIA8AB2AO7N0GTV2xrOxVy3\nnbTNE6Iyh0Z8vOzew1FIWUZxH7WbAAABjrzX4QIAAAQDAEcwRQIhALFR5p9qDfQl\np3CRKYPJHWpIjFDacGSEevAQ4fNS7gpCAiAMm9RkSkmA1O08CVXEEqRdrCw3vvdS\n25+vKB5F1XocVgB2AN/hVuuqBa+1nA+GcY2owDJOrlbZbqf1pWoB0cE7vlJcAAAB\njrzX4d0AAAQDAEcwRQIhAI4+y+2L3QpvolLjKo0Xk70i1La5HzQPPbt7ieL8YzLK\nAiAPWfUoPwfQF4Cra4fv14VSi5G+6X658+p4EBEIVTGDBDANBgkqhkiG9w0BAQsF\nAAOCAQEAEtpxret9LD/n0/CFBj7UH0iIiEMdbnFd1iXjgiMgUPAc8XNPTc6n45wU\nUQrU219pHMjBUFoI+Vtvwg9ajpMOHm0Q+vzUzBnN/XwmUf/gc8N071auF0z2Y939\njsSzIK54ilpABX0ltc2F0ZHUrzzhaJ7elf5z9uId6HEPvvX6gpDCRYKw6lktKiro\nPalDQeJ+XvT8CPX1exkuAKGld+NmZsvWUndtRDPlkDmTK+rYD0Jl7c4an0gO128K\ngu1wnNw0skIHxp7NvDKLFTEemvlIXunTvPUYJJ0gN9FMKlBuvu6mZJwprpg7wMuT\nTE43aEBTsOCmXUc3TnLAU/3vE3p2lQ==\n-----END CERTIFICATE-----\n", "-----BEGIN CERTIFICATE-----\nMIIFljCCA36gAwIBAgINAgO8U1lrNMcY9QFQZjANBgkqhkiG9w0BAQsFADBHMQsw\nCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU\nMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMjAwODEzMDAwMDQyWhcNMjcwOTMwMDAw\nMDQyWjBGMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp\nY2VzIExMQzETMBEGA1UEAxMKR1RTIENBIDFDMzCCASIwDQYJKoZIhvcNAQEBBQAD\nggEPADCCAQoCggEBAPWI3+dijB43+DdCkH9sh9D7ZYIl/ejLa6T/belaI+KZ9hzp\nkgOZE3wJCor6QtZeViSqejOEH9Hpabu5dOxXTGZok3c3VVP+ORBNtzS7XyV3NzsX\nlOo85Z3VvMO0Q+sup0fvsEQRY9i0QYXdQTBIkxu/t/bgRQIh4JZCF8/ZK2VWNAcm\nBA2o/X3KLu/qSHw3TT8An4Pf73WELnlXXPxXbhqW//yMmqaZviXZf5YsBvcRKgKA\ngOtjGDxQSYflispfGStZloEAoPtR28p3CwvJlk/vcEnHXG0g/Zm0tOLKLnf9LdwL\ntmsTDIwZKxeWmLnwi/agJ7u2441Rj72ux5uxiZ0CAwEAAaOCAYAwggF8MA4GA1Ud\nDwEB/wQEAwIBhjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwEgYDVR0T\nAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUinR/r4XN7pXNPZzQ4kYU83E1HScwHwYD\nVR0jBBgwFoAU5K8rJnEaK0gnhS9SZizv8IkTcT4waAYIKwYBBQUHAQEEXDBaMCYG\nCCsGAQUFBzABhhpodHRwOi8vb2NzcC5wa2kuZ29vZy9ndHNyMTAwBggrBgEFBQcw\nAoYkaHR0cDovL3BraS5nb29nL3JlcG8vY2VydHMvZ3RzcjEuZGVyMDQGA1UdHwQt\nMCswKaAnoCWGI2h0dHA6Ly9jcmwucGtpLmdvb2cvZ3RzcjEvZ3RzcjEuY3JsMFcG\nA1UdIARQME4wOAYKKwYBBAHWeQIFAzAqMCgGCCsGAQUFBwIBFhxodHRwczovL3Br\naS5nb29nL3JlcG9zaXRvcnkvMAgGBmeBDAECATAIBgZngQwBAgIwDQYJKoZIhvcN\nAQELBQADggIBAIl9rCBcDDy+mqhXlRu0rvqrpXJxtDaV/d9AEQNMwkYUuxQkq/BQ\ncSLbrcRuf8/xam/IgxvYzolfh2yHuKkMo5uhYpSTld9brmYZCwKWnvy15xBpPnrL\nRklfRuFBsdeYTWU0AIAaP0+fbH9JAIFTQaSSIYKCGvGjRFsqUBITTcFTNvNCCK9U\n+o53UxtkOCcXCb1YyRt8OS1b887U7ZfbFAO/CVMkH8IMBHmYJvJh8VNS/UKMG2Yr\nPxWhu//2m+OBmgEGcYk1KCTd4b3rGS3hSMs9WYNRtHTGnXzGsYZbr8w0xNPM1IER\nlQCh9BIiAfq0g3GvjLeMcySsN1PCAJA/Ef5c7TaUEDu9Ka7ixzpiO2xj2YC/WXGs\nYye5TBeg2vZzFb8q3o/zpWwygTMD0IZRcZk0upONXbVRWPeyk+gB9lm+cZv9TSjO\nz23HFtz30dZGm6fKa+l3D/2gthsjgx0QGtkJAITgRNOidSOzNIb2ILCkXhAd4FJG\nAJ2xDx8hcFH1mt0G/FX0Kw4zd8NLQsLxdxP8c4CU6x+7Nz/OAipmsHMdMqUybDKw\njuDEI/9bfU1lcKwrmz3O2+BtjjKAvpafkmO8l7tdufThcV4q5O8DIrGKZTqPwJNl\n1IXNDw9bg1kWRxYtnCQ6yICmJhSFm/Y3m6xv+cXDBlHz4n/FsRC6UfTd\n-----END CERTIFICATE-----\n", "-----BEGIN CERTIFICATE-----\nMIIFYjCCBEqgAwIBAgIQd70NbNs2+RrqIQ/E8FjTDTANBgkqhkiG9w0BAQsFADBX\nMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTEQMA4GA1UE\nCxMHUm9vdCBDQTEbMBkGA1UEAxMSR2xvYmFsU2lnbiBSb290IENBMB4XDTIwMDYx\nOTAwMDA0MloXDTI4MDEyODAwMDA0MlowRzELMAkGA1UEBhMCVVMxIjAgBgNVBAoT\nGUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxFDASBgNVBAMTC0dUUyBSb290IFIx\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAthECix7joXebO9y/lD63\nladAPKH9gvl9MgaCcfb2jH/76Nu8ai6Xl6OMS/kr9rH5zoQdsfnFl97vufKj6bwS\niV6nqlKr+CMny6SxnGPb15l+8Ape62im9MZaRw1NEDPjTrETo8gYbEvs/AmQ351k\nKSUjB6G00j0uYODP0gmHu81I8E3CwnqIiru6z1kZ1q+PsAewnjHxgsHA3y6mbWwZ\nDrXYfiYaRQM9sHmklCitD38m5agI/pboPGiUU+6DOogrFZYJsuB6jC511pzrp1Zk\nj5ZPaK49l8KEj8C8QMALXL32h7M1bKwYUH+E4EzNktMg6TO8UpmvMrUpsyUqtEj5\ncuHKZPfmghCN6J3Cioj6OGaK/GP5Afl4/Xtcd/p2h/rs37EOeZVXtL0m79YB0esW\nCruOC7XFxYpVq9Os6pFLKcwZpDIlTirxZUTQAs6qzkm06p98g7BAe+dDq6dso499\niYH6TKX/1Y7DzkvgtdizjkXPdsDtQCv9Uw+wp9U7DbGKogPeMa3Md+pvez7W35Ei\nEua++tgy/BBjFFFy3l3WFpO9KWgz7zpm7AeKJt8T11dleCfeXkkUAKIAf5qoIbap\nsZWwpbkNFhHax2xIPEDgfg1azVY80ZcFuctL7TlLnMQ/0lUTbiSw1nH69MG6zO0b\n9f6BQdgAmD06yK56mDcYBZUCAwEAAaOCATgwggE0MA4GA1UdDwEB/wQEAwIBhjAP\nBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTkrysmcRorSCeFL1JmLO/wiRNxPjAf\nBgNVHSMEGDAWgBRge2YaRQ2XyolQL30EzTSo//z9SzBgBggrBgEFBQcBAQRUMFIw\nJQYIKwYBBQUHMAGGGWh0dHA6Ly9vY3NwLnBraS5nb29nL2dzcjEwKQYIKwYBBQUH\nMAKGHWh0dHA6Ly9wa2kuZ29vZy9nc3IxL2dzcjEuY3J0MDIGA1UdHwQrMCkwJ6Al\noCOGIWh0dHA6Ly9jcmwucGtpLmdvb2cvZ3NyMS9nc3IxLmNybDA7BgNVHSAENDAy\nMAgGBmeBDAECATAIBgZngQwBAgIwDQYLKwYBBAHWeQIFAwIwDQYLKwYBBAHWeQIF\nAwMwDQYJKoZIhvcNAQELBQADggEBADSkHrEoo9C0dhemMXoh6dFSPsjbdBZBiLg9\nNR3t5P+T4Vxfq7vqfM/b5A3Ri1fyJm9bvhdGaJQ3b2t6yMAYN/olUazsaL+yyEn9\nWprKASOshIArAoyZl+tJaox118fessmXn1hIVw41oeQa1v1vg4Fv74zPl6/AhSrw\n9U5pCZEt4Wi4wStz6dTZ/CLANx8LZh1J7QJVj2fhMtfTJr9w4z30Z209fOU0iOMy\n+qduBmpvvYuR7hZL6Dupszfnw0Skfths18dG9ZKb59UhvmaSGZRVbNQpsg3BZlvi\nd0lIKO2d1xozclOzgjXPYovJJIultzkMu34qQb9Sz/yilrbCgj8=\n-----END CERTIFICATE-----\n"], "dhparams": null, "versions": ["-TLSv1", "-SSLv2", "-SSLv3", "-TLSv1.1", "TLSv1.2", "TLSv1.3"], "acceptable_cas": [], "tlsext": [{"id": 51, "name": "key_share"}, {"id": 43, "name": "supported_versions"}], "alpn": [], "cert": {"sig_alg": "sha256WithRSAEncryption", "issued": "20240408073406Z", "expires": "20240701073405Z", "pubkey": {"bits": 2048, "type": "rsa"}, "version": 2, "extensions": [{"critical": true, "data": "\\x03\\x02\\x05\\xa0", "name": "keyUsage"}, {"data": "0\\n\\x06\\x08+\\x06\\x01\\x05\\x05\\x07\\x03\\x01", "name": "extendedKeyUsage"}, {"critical": true, "data": "0\\x00", "name": "basicConstraints"}, {"data": "\\x04\\x14y\\n\\x8fO\\x1c\\xd6\\x91\\x95H\\xa8\\xec\\xf5;6\\x0bp9\\xd5P\\x9e", "name": "subjectKeyIdentifier"}, {"data": "0\\x16\\x80\\x14\\x8at\\x7f\\xaf\\x85\\xcd\\xee\\x95\\xcd=\\x9c\\xd0\\xe2F\\x14\\xf3q5\\x1d\\'", "name": "authorityKeyIdentifier"}, {"data": "0\\\\0\\'\\x06\\x08+\\x06\\x01\\x05\\x05\\x070\\x01\\x86\\x1bhttp://ocsp.pki.goog/gts1c301\\x06\\x08+\\x06\\x01\\x05\\x05\\x070\\x02\\x86%http://pki.goog/repo/certs/gts1c3.der", "name": "authorityInfoAccess"}, {"data": "0\\x81\\xa1\\x82\\ndns.google\\x82\\x0edns.google.com\\x82\\x10*.dns.google.com\\x82\\x0b8888.google\\x82\\x10dns64.dns.google\\x87\\x04\\x08\\x08\\x08\\x08\\x87\\x04\\x08\\x08\\x04\\x04\\x87\\x10 \\x01H`H`\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x88\\x88\\x87\\x10 \\x01H`H`\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x88D\\x87\\x10 \\x01H`H`\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00dd\\x87\\x10 \\x01H`H`\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00d", "name": "subjectAltName"}, {"data": "0\\x180\\x08\\x06\\x06g\\x81\\x0c\\x01\\x02\\x010\\x0c\\x06\\n+\\x06\\x01\\x04\\x01\\xd6y\\x02\\x05\\x03", "name": "certificatePolicies"}, {"data": "0301\\xa0/\\xa0-\\x86+http://crls.pki.goog/gts1c3/QqFxbi9M48c.crl", "name": "crlDistributionPoints"}, {"data": "\\x04\\x81\\xf2\\x00\\xf0\\x00v\\x00\\xee\\xcd\\xd0d\\xd5\\xdb\\x1a\\xce\\xc5\\\\\\xb7\\x9d\\xb4\\xcd\\x13\\xa22\\x87F|\\xbc\\xec\\xde\\xc3QHYFq\\x1f\\xb5\\x9b\\x00\\x00\\x01\\x8e\\xbc\\xd7\\xe1\\x02\\x00\\x00\\x04\\x03\\x00G0E\\x02!\\x00\\xb1Q\\xe6\\x9fj\\r\\xf4%\\xa7p\\x91)\\x83\\xc9\\x1djH\\x8cP\\xdapd\\x84z\\xf0\\x10\\xe1\\xf3R\\xee\\nB\\x02 \\x0c\\x9b\\xd4dJI\\x80\\xd4\\xed<\\tU\\xc4\\x12\\xa4]\\xac,7\\xbe\\xf7R\\xdb\\x9f\\xaf(\\x1eE\\xd5z\\x1cV\\x00v\\x00\\xdf\\xe1V\\xeb\\xaa\\x05\\xaf\\xb5\\x9c\\x0f\\x86q\\x8d\\xa8\\xc02N\\xaeV\\xd9n\\xa7\\xf5\\xa5j\\x01\\xd1\\xc1;\\xbeR\\\\\\x00\\x00\\x01\\x8e\\xbc\\xd7\\xe1\\xdd\\x00\\x00\\x04\\x03\\x00G0E\\x02!\\x00\\x8e>\\xcb\\xed\\x8b\\xdd\\no\\xa2R\\xe3*\\x8d\\x17\\x93\\xbd\"\\xd4\\xb6\\xb9\\x1f4\\x0f=\\xbb{\\x89\\xe2\\xfcc2\\xca\\x02 \\x0fY\\xf5(?\\x07\\xd0\\x17\\x80\\xabk\\x87\\xef\\xd7\\x85R\\x8b\\x91\\xbe\\xe9~\\xb9\\xf3\\xeax\\x10\\x11\\x08U1\\x83\\x04", "name": "ct_precert_scts"}], "fingerprint": {"sha256": "5a0b6f72280c3e1371b6c58be4649c7bf0318e5eb860aca55815045b832cd2a4", "sha1": "e1fb59f3a2b1b6089413fa68400347e07ba1ca75"}, "serial": 96876595460258181011824296766757681011, "issuer": {"C": "US", "CN": "GTS CA 1C3", "O": "Google Trust Services LLC"}, "expired": false, "subject": {"CN": "dns.google"}}, "cipher": {"version": "TLSv1.3", "bits": 256, "name": "TLS_AES_256_GCM_SHA384"}, "trust": {"revoked": false, "browser": null}, "handshake_states": ["before SSL initialization", "SSLv3/TLS write client hello", "SSLv3/TLS read server hello", "TLSv1.3 read encrypted extensions", "SSLv3/TLS read server certificate", "TLSv1.3 read server certificate verify", "SSLv3/TLS read finished", "SSLv3/TLS write change cipher spec", "SSLv3/TLS write finished", "SSL negotiation finished successfully"], "ja3s": "66e33336e3e99f75410126f42d44cc81", "ocsp": {}}, "hostnames": ["dns.google"], "location": {"city": "Mountain View", "region_code": "CA", "area_code": null, "longitude": -122.0775, "latitude": 37.4056, "country_code": "US", "country_name": "United States"}, "ip": 134744072, "domains": ["dns.google"], "org": "Google LLC", "data": "HTTP/1.1 200 OK\r\nContent-Security-Policy: object-src 'none';base-uri 'self';script-src 'nonce-iRRB3YCy7UKnm1gs9cWAWA' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/honest_dns/1_0;frame-ancestors 'none'\r\nStrict-Transport-Security: max-age=31536000; includeSubDomains; preload\r\nX-Content-Type-Options: nosniff\r\nContent-Type: text/html; charset=UTF-8\r\nDate: Sun, 28 Apr 2024 09:25:24 GMT\r\nServer: scaffolding on HTTPServer2\r\nCache-Control: private\r\nX-XSS-Protection: 0\r\nX-Frame-Options: SAMEORIGIN\r\nAlt-Svc: h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000\r\nAccept-Ranges: none\r\nVary: Accept-Encoding\r\nTransfer-Encoding: chunked\r\n\r\n", "port": 443, "opts": {"vulns": [], "heartbleed": "2024/04/28 09:25:38 8.8.8.8:443 - SAFE\n"}, "ip_str": "8.8.8.8"}], "asn": "AS15169", "city": "Mountain View", "latitude": 37.4056, "isp": "Google LLC", "longitude": -122.0775, "last_update": "2024-04-28T12:50:55.650683", "country_name": "United States", "ip_str": "8.8.8.8", "os": null, "ports": [443, 53]} \ No newline at end of file diff --git a/providers/whois/whois.go b/providers/whois/whois.go new file mode 100644 index 0000000..326aeef --- /dev/null +++ b/providers/whois/whois.go @@ -0,0 +1 @@ +package whois diff --git a/session/config.yaml b/session/config.yaml new file mode 100644 index 0000000..e9e2197 --- /dev/null +++ b/session/config.yaml @@ -0,0 +1,48 @@ +--- +global: + indent_spaces: 2 + max_value_chars: 300 + max_age: 90d + max_reports: 5 + output: table + # ports: [80] + +providers: + annotated: + enabled: false + paths: + # - + aws: + enabled: true + # url: override the default URL + azure: + enabled: true + # url: override the default URL + criminalip: + enabled: false + digitalocean: + enabled: true + # url: override the default URL + gcp: + enabled: true + ipurl: + enabled: true + urls: + - "https://iplists.firehol.org/files/firehol_level1.netset" + - "https://iplists.firehol.org/files/firehol_level2.netset" + - "https://iplists.firehol.org/files/firehol_anonymous.netset" + - "https://iplists.firehol.org/files/blocklist_de.ipset" + - "https://iplists.firehol.org/files/socks_proxy_7d.ipset" + - "https://iplists.firehol.org/files/sslproxies_7d.ipset" + - "https://iplists.firehol.org/files/tor_exits_7d.ipset" + icloudpr: + enabled: true + # url: override the default URL + linode: + enabled: true + # url: override the default URL + ptr: + enabled: true + shodan: + enabled: false + max_ports: 10 \ No newline at end of file diff --git a/session/session.go b/session/session.go new file mode 100644 index 0000000..fb75e03 --- /dev/null +++ b/session/session.go @@ -0,0 +1,244 @@ +package session + +import ( + _ "embed" + "fmt" + "log/slog" + "net/netip" + "os" + "path" + "path/filepath" + "sync" + "time" + + "github.com/dgraph-io/badger/v4" + "github.com/hashicorp/go-retryablehttp" + "github.com/mitchellh/go-homedir" + "gopkg.in/yaml.v2" +) + +const ( + AppName = "ipscout" + DefaultIndentSpaces = 2 + DefaultMaxReports = 5 + DefaultConfigFileName = "config.yaml" + // DefaultConfigFileRoot = ".session/ipscout" +) + +//go:embed config.yaml +var defaultConfig string + +type Stats struct { + Mu sync.Mutex + InitialiseDuration map[string]time.Duration + InitialiseUsedCache map[string]bool + FindHostDuration map[string]time.Duration + FindHostUsedCache map[string]bool + CreateTableDuration map[string]time.Duration +} + +func CreateStats() *Stats { + return &Stats{ + InitialiseDuration: make(map[string]time.Duration), + InitialiseUsedCache: make(map[string]bool), + FindHostDuration: make(map[string]time.Duration), + FindHostUsedCache: make(map[string]bool), + CreateTableDuration: make(map[string]time.Duration), + } +} + +func New() *Session { + return &Session{ + Stats: CreateStats(), + } +} + +func (c *Session) Validate() error { + switch { + case c.Logger == nil: + return fmt.Errorf("logger not set") + case c.Stats == nil: + return fmt.Errorf("stats not set") + case c.Cache == nil: + return fmt.Errorf("cache not set") + } + + return nil +} + +type Config struct { + Global GlobalConfig `mapstructure:"global"` +} + +type GlobalConfig struct { + LogLevel string `mapstructure:"log-level"` + Output string `mapstructure:"output"` + IndentSpaces int `mapstructure:"indent-spaces"` + Ports []string `mapstructure:"ports"` + MaxValueChars int32 `mapstructure:"max-value-chars"` + MaxAge string `mapstructure:"max-age"` + MaxReports int `mapstructure:"max-reports"` + DisableCache bool `mapstructure:"disable-cache"` +} + +type Session struct { + App struct { + Version string + SemVer string + } + Logger *slog.Logger + Stats *Stats + Target *os.File + Output string + Cache *badger.DB + Config Config + + HTTPClient *retryablehttp.Client + Host netip.Addr + Providers Providers `mapstructure:"providers"` + HideProgress bool `mapstructure:"hide-progress"` + + // MaxWidth int + UseTestData bool +} + +type Providers struct { + AbuseIPDB struct { + Enabled bool `mapstructure:"enabled"` + APIKey string `mapstructure:"api-key"` + MaxAge int `mapstructure:"max-age"` + } `mapstructure:"abuseipdb"` + Annotated struct { + Enabled bool `mapstructure:"enabled"` + Paths []string `mapstructure:"paths"` + } `mapstructure:"annotated"` + AWS struct { + Enabled bool `mapstructure:"enabled"` + URL string `mapstructure:"url"` + } `mapstructure:"aws"` + Azure struct { + Enabled bool `mapstructure:"enabled"` + URL string `mapstructure:"url"` + } `mapstructure:"azure"` + CriminalIP struct { + APIKey string `mapstructure:"api-key"` + Enabled bool `mapstructure:"enabled"` + } `mapstructure:"criminalip"` + DigitalOcean struct { + Enabled bool `mapstructure:"enabled"` + URL string + } `mapstructure:"digitalocean"` + GCP struct { + Enabled bool `mapstructure:"enabled"` + URL string + } `mapstructure:"gcp"` + ICloudPR struct { + Enabled bool `mapstructure:"enabled"` + URL string `mapstructure:"url"` + } `mapstructure:"icloudpr"` + IPURL struct { + Enabled bool `mapstructure:"enabled"` + URLs []string `mapstructure:"urls"` + } `mapstructure:"ipurl"` + Linode struct { + Enabled bool `mapstructure:"enabled"` + URL string + } `mapstructure:"linode"` + Shodan struct { + APIKey string `mapstructure:"api-key"` + Enabled bool `mapstructure:"enabled"` + } `mapstructure:"shodan"` + PTR struct { + Enabled bool `mapstructure:"enabled"` + } `mapstructure:"ptr"` +} + +func unmarshalConfig(data []byte) (*Session, error) { + var conf Session + if err := yaml.Unmarshal(data, &conf); err != nil { + return nil, fmt.Errorf("failed to unmarshal session: %w", err) + } + + return &conf, nil +} + +func CreateDefaultConfigIfMissing(path string) error { + if path == "" { + return fmt.Errorf("session path not specified") + } + + var err error + + // check if session already exists + _, err = os.Stat(filepath.Join(path, DefaultConfigFileName)) + + switch { + case err == nil: + return nil + case os.IsNotExist(err): + // check default session is valid + if _, err = unmarshalConfig([]byte(defaultConfig)); err != nil { + return fmt.Errorf("default session invalid: %w", err) + } + + // create dir specified in path argument if missing + if _, err = os.Stat(path); os.IsNotExist(err) { + if err = os.MkdirAll(path, 0o700); err != nil { + return fmt.Errorf("failed to create session directory: %w", err) + } + } + + if err = os.WriteFile(filepath.Join(path, DefaultConfigFileName), []byte(defaultConfig), 0o600); err != nil { + return fmt.Errorf("failed to write default session: %w", err) + } + case err != nil: + return fmt.Errorf("failed to stat session directory: %w", err) + } + + return nil +} + +// CreateConfigPathStructure creates all the necessary paths under session root if they don't exist +// and returns an error if it fails to create the directory, or the session root does not exist +func CreateConfigPathStructure(configRoot string) error { + // check session root exists + _, err := os.Stat(configRoot) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("session root does not exist: %v", err) + } + } + + for _, dir := range []string{"backups", "cache"} { + _, err = os.Stat(path.Join(configRoot, dir)) + if err != nil { + if os.IsNotExist(err) { + mErr := os.MkdirAll(path.Join(configRoot, dir), 0o700) + if mErr != nil { + return fmt.Errorf("failed to create %s directory: %v", dir, mErr) + } + } else { + return fmt.Errorf("failed to stat %s directory: %v", dir, err) + } + } + } + + return nil +} + +// GetConfigRoot returns the root path for the app's session directory +// if root is specified, it will use that, otherwise it will use the user's home directory +func GetConfigRoot(root string, appName string) string { + // if root specified then use that + if root != "" { + return path.Join(root, ".config", appName) + } + + // otherwise, use the user's home directory + home, err := homedir.Dir() + if err != nil { + os.Exit(1) + } + + return path.Join(home, ".config", appName) +} diff --git a/session/session_test.go b/session/session_test.go new file mode 100644 index 0000000..5eee1d3 --- /dev/null +++ b/session/session_test.go @@ -0,0 +1,126 @@ +package session + +import ( + "os" + "path/filepath" + "testing" + + "github.com/mitchellh/go-homedir" + "github.com/stretchr/testify/require" +) + +func TestNew(t *testing.T) { + conf := New() + require.NotNil(t, conf) + require.NotNil(t, conf.Stats) + require.NotNil(t, conf.Stats.InitialiseDuration) + require.NotNil(t, conf.Stats.InitialiseUsedCache) + require.NotNil(t, conf.Stats.FindHostDuration) + require.NotNil(t, conf.Stats.FindHostUsedCache) + require.NotNil(t, conf.Stats.CreateTableDuration) +} + +func TestUnmarshalConfig(t *testing.T) { + t.Run("ValidConfig", func(t *testing.T) { + data := []byte(defaultConfig) + conf, err := unmarshalConfig(data) + require.NoError(t, err) + require.NotNil(t, conf) + }) + + t.Run("InvalidConfig", func(t *testing.T) { + data := []byte("invalid session") + conf, err := unmarshalConfig(data) + require.Error(t, err) + require.Nil(t, conf) + }) +} + +func TestCreateDefaultConfig(t *testing.T) { + t.Run("PathExists", func(t *testing.T) { + path := "/tmp" + err := CreateDefaultConfigIfMissing(path) + require.NoError(t, err) + }) + + t.Run("PathDoesNotExist", func(t *testing.T) { + path := "/tmp/nonexistent" + err := CreateDefaultConfigIfMissing(path) + require.NoError(t, err) + _, err = os.Stat(path) + require.NoError(t, err) + }) + + t.Run("InvalidPath", func(t *testing.T) { + path := "" + err := CreateDefaultConfigIfMissing(path) + require.Error(t, err) + }) +} + +func TestCreateCachePathIfNotExist(t *testing.T) { + t.Run("PathExists", func(t *testing.T) { + tempDir := t.TempDir() + + configRoot := GetConfigRoot(tempDir, AppName) + + // create session root (required for cache path) + require.NoError(t, CreateDefaultConfigIfMissing(configRoot)) + + // check session root exists + _, err := os.Stat(configRoot) + require.NoError(t, err) + + // check cache path does not exist + _, err = os.Stat(filepath.Join(configRoot, "cache")) + require.ErrorIs(t, err, os.ErrNotExist) + + // create cache path + require.NoError(t, CreateConfigPathStructure(configRoot)) + // check cache path exists + for _, dir := range []string{"backups", "cache"} { + _, err = os.Stat(filepath.Join(configRoot, dir)) + require.NoError(t, err) + } + }) + + t.Run("PathDoesNotExist", func(t *testing.T) { + tempDir := t.TempDir() + configRoot := GetConfigRoot(tempDir, AppName) + + // create session root (required for cache path) + require.NoError(t, CreateDefaultConfigIfMissing(configRoot)) + + err := CreateConfigPathStructure(configRoot) + require.NoError(t, err) + + for _, dir := range []string{"backups", "cache"} { + _, err = os.Stat(filepath.Join(configRoot, dir)) + require.NoError(t, err) + } + }) + + t.Run("InvalidPath", func(t *testing.T) { + path := "" + err := CreateConfigPathStructure(path) + require.Error(t, err) + }) +} + +func TestGetConfigRoot(t *testing.T) { + t.Run("ValidAppName", func(t *testing.T) { + dir := t.TempDir() + appName := "test" + path := GetConfigRoot(dir, appName) + require.Equal(t, filepath.Join(dir, ".config", appName), path) + }) + + t.Run("EmptyAppName", func(t *testing.T) { + appName := "" + dir, err := homedir.Dir() + require.NoError(t, err) + + path := GetConfigRoot("", appName) + require.Equal(t, filepath.Join(dir, ".config", appName), path) + }) +}