From aa016dedf16a5f0c39fd5d16797836d8d766c6fe Mon Sep 17 00:00:00 2001 From: Jon Hadfield Date: Sat, 4 May 2024 06:05:06 +0100 Subject: [PATCH] add iCloud Private Relay. --- cmd/root.go | 2 + go.mod | 2 +- go.sum | 4 +- process/process.go | 2 + providers/aws/aws.go | 4 +- providers/digitalocean/digitalocean.go | 4 +- providers/gcp/gcp.go | 4 +- providers/icloudpr/icloudpr.go | 355 ++++++++++++++++++ .../icloudpr_172_224_224_60_report.json | 10 + providers/ipurl/ipurl.go | 4 +- providers/linode/linode.go | 4 +- session/config.yaml | 3 + session/session.go | 4 + 13 files changed, 389 insertions(+), 13 deletions(-) create mode 100644 providers/icloudpr/icloudpr.go create mode 100644 providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json diff --git a/cmd/root.go b/cmd/root.go index 6852747..6af41f0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -156,6 +156,8 @@ func initConfig(cmd *cobra.Command) error { sess.Providers.DigitalOcean.URL = v.GetString("providers.digitalocean.url") sess.Providers.GCP.Enabled = v.GetBool("providers.gcp.enabled") sess.Providers.GCP.URL = v.GetString("providers.gcp.url") + sess.Providers.ICloudPR.Enabled = v.GetBool("providers.icloudpr.enabled") + sess.Providers.ICloudPR.URL = v.GetString("providers.icloudpr.url") sess.Providers.IPURL.Enabled = v.GetBool("providers.ipurl.enabled") sess.Providers.IPURL.URLs = v.GetStringSlice("providers.ipurl.urls") sess.Providers.Linode.Enabled = v.GetBool("providers.linode.enabled") diff --git a/go.mod b/go.mod index f3c8e2a..75b82e0 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/fatih/color v1.16.0 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/jedib0t/go-pretty/v6 v6.5.8 - github.com/jonhadfield/ip-fetcher v0.0.0-20240502200418-266a499cb230 + github.com/jonhadfield/ip-fetcher v0.0.0-20240503180217-558f6741c36f github.com/miekg/dns v1.1.59 github.com/mitchellh/go-homedir v1.1.0 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index 2f6c48d..80f8875 100644 --- a/go.sum +++ b/go.sum @@ -82,8 +82,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jedib0t/go-pretty/v6 v6.5.8 h1:8BCzJdSvUbaDuRba4YVh+SKMGcAAKdkcF3SVFbrHAtQ= github.com/jedib0t/go-pretty/v6 v6.5.8/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= -github.com/jonhadfield/ip-fetcher v0.0.0-20240502200418-266a499cb230 h1:jWf1Hhed32wTkXG4F5CVCMQWsjcHyh24D9v/rPXmN6g= -github.com/jonhadfield/ip-fetcher v0.0.0-20240502200418-266a499cb230/go.mod h1:xckxqrzpzUyfNGSRKc+Ugp47f5iwogDYQB4iUUxfXHg= +github.com/jonhadfield/ip-fetcher v0.0.0-20240503180217-558f6741c36f h1:HxE19TTjhpHylMfgeoply4qrztj/seduEAO+Q/pQcJA= +github.com/jonhadfield/ip-fetcher v0.0.0-20240503180217-558f6741c36f/go.mod h1:xckxqrzpzUyfNGSRKc+Ugp47f5iwogDYQB4iUUxfXHg= github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI= github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/process/process.go b/process/process.go index d660eaf..1a5e4bd 100644 --- a/process/process.go +++ b/process/process.go @@ -2,6 +2,7 @@ package process import ( "fmt" + "github.com/jonhadfield/ipscout/providers/icloudpr" "log/slog" "os" "path/filepath" @@ -51,6 +52,7 @@ func getProviderClients(sess session.Session) (map[string]providers.ProviderClie {Name: digitalocean.ProviderName, Enabled: sess.Providers.DigitalOcean.Enabled, APIKey: "", NewClient: digitalocean.NewProviderClient}, {Name: gcp.ProviderName, Enabled: sess.Providers.GCP.Enabled, APIKey: "", NewClient: gcp.NewProviderClient}, {Name: ipurl.ProviderName, Enabled: sess.Providers.IPURL.Enabled, APIKey: "", NewClient: ipurl.NewProviderClient}, + {Name: icloudpr.ProviderName, Enabled: sess.Providers.ICloudPR.Enabled, APIKey: "", NewClient: icloudpr.NewProviderClient}, {Name: linode.ProviderName, Enabled: sess.Providers.Linode.Enabled, APIKey: "", NewClient: linode.NewProviderClient}, {Name: shodan.ProviderName, Enabled: sess.Providers.Shodan.Enabled, APIKey: sess.Providers.Shodan.APIKey, NewClient: shodan.NewProviderClient}, {Name: ptr.ProviderName, Enabled: sess.Providers.PTR.Enabled, APIKey: "", NewClient: ptr.NewProviderClient}, diff --git a/providers/aws/aws.go b/providers/aws/aws.go index d298403..8884552 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -337,10 +337,10 @@ func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, }) tw.SetAutoIndex(false) - tw.SetTitle("AWS IP | Host: %s", c.Host.String()) + tw.SetTitle("AWS | Host: %s", c.Host.String()) if c.UseTestData { - tw.SetTitle("AWS IP | Host: 18.164.100.99") + tw.SetTitle("AWS | Host: 18.164.100.99") } return &tw, nil diff --git a/providers/digitalocean/digitalocean.go b/providers/digitalocean/digitalocean.go index daee4b4..fb8f745 100644 --- a/providers/digitalocean/digitalocean.go +++ b/providers/digitalocean/digitalocean.go @@ -290,10 +290,10 @@ func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, }) tw.SetAutoIndex(false) - tw.SetTitle("DigitalOcean IP | Host: %s", c.Host.String()) + tw.SetTitle("DIGITAL OCEAN | Host: %s", c.Host.String()) if c.UseTestData { - tw.SetTitle("DigitalOcean IP | Host: %s", "165.232.46.239") + tw.SetTitle("DIGITAL OCEAN | Host: %s", "165.232.46.239") } return &tw, nil diff --git a/providers/gcp/gcp.go b/providers/gcp/gcp.go index f384ec1..2e59b11 100644 --- a/providers/gcp/gcp.go +++ b/providers/gcp/gcp.go @@ -315,10 +315,10 @@ func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, }) tw.SetAutoIndex(false) - tw.SetTitle("GCP IP | Host: %s", c.Host.String()) + tw.SetTitle("GCP | Host: %s", c.Host.String()) if c.UseTestData { - tw.SetTitle("GCP IP | Host: %s", "34.128.62.2") + tw.SetTitle("GCP | Host: %s", "34.128.62.2") } return &tw, nil diff --git a/providers/icloudpr/icloudpr.go b/providers/icloudpr/icloudpr.go new file mode 100644 index 0000000..3e988af --- /dev/null +++ b/providers/icloudpr/icloudpr.go @@ -0,0 +1,355 @@ +package icloudpr + +import ( + "encoding/json" + "errors" + "fmt" + "net/netip" + "os" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jonhadfield/ip-fetcher/providers/icloudpr" + "github.com/jonhadfield/ipscout/cache" + "github.com/jonhadfield/ipscout/providers" + "github.com/jonhadfield/ipscout/session" +) + +const ( + ProviderName = "icloudpr" + DocTTL = 24 * time.Hour +) + +type Config struct { + _ struct{} + session.Session + Host netip.Addr + APIKey string +} + +type ProviderClient struct { + session.Session +} + +func NewProviderClient(c session.Session) (providers.ProviderClient, error) { + c.Logger.Debug("creating icloudpr client") + + tc := &ProviderClient{ + Session: c, + } + + return tc, nil +} + +func (c *ProviderClient) Enabled() bool { + return c.Session.Providers.ICloudPR.Enabled +} + +func (c *ProviderClient) GetConfig() *session.Session { + return &c.Session +} + +func unmarshalResponse(rBody []byte) (*HostSearchResult, error) { + var res *HostSearchResult + + if err := json.Unmarshal(rBody, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + res.Raw = rBody + + return res, nil +} + +func unmarshalProviderData(data []byte) (*icloudpr.Doc, error) { + var res *icloudpr.Doc + + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("error unmarshalling icloudpr data: %w", err) + } + + return res, nil +} + +func (c *ProviderClient) loadProviderData() error { + icloudprClient := icloudpr.New() + icloudprClient.Client = c.HTTPClient + + if c.Providers.ICloudPR.URL != "" { + icloudprClient.DownloadURL = c.Providers.ICloudPR.URL + c.Logger.Debug("overriding icloudpr source", "url", icloudprClient.DownloadURL) + } + + doc, err := icloudprClient.Fetch() + if err != nil { + return fmt.Errorf("error fetching icloudpr data: %w", err) + } + + data, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("error marshalling icloudpr provider doc: %w", err) + } + + err = cache.UpsertWithTTL(c.Logger, c.Cache, cache.Item{ + AppVersion: c.App.Version, + Key: providers.CacheProviderPrefix + ProviderName, + Value: data, + Version: doc.ETag, + Created: time.Now(), + }, DocTTL) + if err != nil { + return fmt.Errorf("error upserting icloudpr data: %w", err) + } + + return nil +} + +const ( + MaxColumnWidth = 120 +) + +func (c *ProviderClient) Initialise() error { + if c.Cache == nil { + return errors.New("cache not set") + } + + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.InitialiseDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + c.Logger.Debug("initialising icloudpr client") + + // load provider data into cache if not already present and fresh + ok, err := cache.CheckExists(c.Logger, c.Cache, providers.CacheProviderPrefix+ProviderName) + if err != nil { + return fmt.Errorf("checking icloudpr cache: %w", err) + } + + if ok { + c.Logger.Info("icloudpr provider data found in cache") + + return nil + } + + c.Logger.Info("loading icloudpr provider data from source") + + err = c.loadProviderData() + if err != nil { + return fmt.Errorf("loading icloudpr api response: %w", err) + } + + return nil +} + +func (c *ProviderClient) loadProviderDataFromCache() (*icloudpr.Doc, error) { + c.Logger.Info("loading icloudpr provider data from cache") + + cacheKey := providers.CacheProviderPrefix + ProviderName + + var doc *icloudpr.Doc + + if item, err := cache.Read(c.Logger, c.Cache, cacheKey); err == nil { + var uErr error + + doc, uErr = unmarshalProviderData(item.Value) + if uErr != nil { + defer func() { + _ = cache.Delete(c.Logger, c.Cache, cacheKey) + }() + + return nil, fmt.Errorf("error unmarshalling cached icloudpr provider doc: %w", uErr) + } + } else { + return nil, fmt.Errorf("error reading icloudpr cache: %w", err) + } + + c.Stats.Mu.Lock() + c.Stats.FindHostUsedCache[ProviderName] = true + c.Stats.Mu.Unlock() + + return doc, nil +} + +func loadTestData(c *ProviderClient) ([]byte, error) { + tdf, err := loadResultsFile("providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json") + if err != nil { + return nil, err + } + + c.Logger.Info("icloudpr match returned from test data", "host", "172.224.224.60") + + out, err := json.Marshal(tdf) + if err != nil { + return nil, fmt.Errorf("error marshalling test data: %w", err) + } + + return out, nil +} + +// FindHost searches for the host in the icloudpr data +func (c *ProviderClient) FindHost() ([]byte, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.FindHostDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + var result *HostSearchResult + + var err error + + // return cached report if test data is enabled + if c.UseTestData { + return loadTestData(c) + } + + doc, err := c.loadProviderDataFromCache() + if err != nil { + return nil, fmt.Errorf("loading icloudpr host data from cache: %w", err) + } + + // search in the data for the host + for _, record := range doc.Records { + if record.Prefix.Contains(c.Host) { + result = &HostSearchResult{ + Prefix: record.Prefix, + Alpha2Code: record.Alpha2Code, + Region: record.Region, + City: record.City, + PostalCode: record.PostalCode, + SyncToken: doc.ETag, + CreationTime: time.Time{}, + } + + c.Logger.Debug("returning icloudpr host match data") + + break + } + } + + if result == nil { + c.Logger.Debug("no icloudpr host match found") + return nil, providers.ErrNoMatchFound + } + + var raw []byte + + raw, err = json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("error marshalling response: %w", err) + } + + result.Raw = raw + + // TODO: remove before release + if os.Getenv("CCI_BACKUP_RESPONSES") == "true" { + c.Logger.Debug("backing up icloudpr host report") + + if err = os.WriteFile(fmt.Sprintf("%s/backups/icloudpr_%s_report.json", session.GetConfigRoot("", session.AppName), + strings.ReplaceAll(c.Host.String(), ".", "_")), raw, 0o600); err != nil { + panic(err) + } + } + + return result.Raw, nil +} + +func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { + start := time.Now() + defer func() { + c.Stats.Mu.Lock() + c.Stats.CreateTableDuration[ProviderName] = time.Since(start) + c.Stats.Mu.Unlock() + }() + + result, err := unmarshalResponse(data) + if err != nil { + return nil, fmt.Errorf("error unmarshalling response: %w", err) + } + + tw := table.NewWriter() + + var rows []table.Row + + tw.AppendRow(table.Row{"Prefix", dashIfEmpty(result.Prefix.String())}) + tw.AppendRow(table.Row{"Alpha2Code", dashIfEmpty(result.Alpha2Code)}) + tw.AppendRow(table.Row{"Region", dashIfEmpty(result.Region)}) + tw.AppendRow(table.Row{"City", dashIfEmpty(result.City)}) + // tw.AppendRow(table.Row{"Postal Code", dashIfEmpty(result.PostalCode)}) + + if !result.CreationTime.IsZero() { + tw.AppendRow(table.Row{"Creation Time", dashIfEmpty(result.CreationTime.String())}) + } + + //if result.SyncToken != "" { + // tw.AppendRow(table.Row{"SyncToken", dashIfEmpty(result.SyncToken)}) + //} + + tw.AppendRows(rows) + tw.SetColumnConfigs([]table.ColumnConfig{ + {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, + }) + tw.SetAutoIndex(false) + tw.SetTitle("ICLOUD PRIVATE RELAY | Host: %s", c.Host.String()) + + if c.UseTestData { + tw.SetTitle("ICLOUD PRIVATE RELAY | Host: %s", "172.224.224.60") + } + + return &tw, nil +} + +func loadResultsFile(path string) (res *HostSearchResult, err error) { + jf, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening file: %w", err) + } + + defer jf.Close() + + decoder := json.NewDecoder(jf) + + err = decoder.Decode(&res) + if err != nil { + return res, fmt.Errorf("error decoding file: %w", err) + } + + return res, nil +} + +type HostSearchResult struct { + Raw []byte + Prefix netip.Prefix `json:"ip_prefix"` + Alpha2Code string `json:"alpha2code"` + Region string `json:"region"` + City string `json:"city"` + PostalCode string `json:"postal_code"` + SyncToken string `json:"synctoken"` + CreationTime time.Time `json:"creation_time"` +} + +func dashIfEmpty(value interface{}) string { + switch v := value.(type) { + case string: + if len(v) == 0 { + return "-" + } + + return v + case *string: + if v == nil || len(*v) == 0 { + return "-" + } + + return *v + case int: + return fmt.Sprintf("%d", v) + default: + return "-" + } +} diff --git a/providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json b/providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json new file mode 100644 index 0000000..5fda0f0 --- /dev/null +++ b/providers/icloudpr/testdata/icloudpr_172_224_224_60_report.json @@ -0,0 +1,10 @@ +{ + "Raw": "eyJSYXciOm51bGwsImlwX3ByZWZpeCI6IjE3Mi4yMjQuMjI0LjYwLzMxIiwiYWxwaGEyY29kZSI6IkdCIiwicmVnaW9uIjoiR0ItV0EiLCJjaXR5IjoiQ2FyZGlmZiIsInBvc3RhbF9jb2RlIjoiIiwic3luY3Rva2VuIjoiXCI2NjJmNmQ4Mi1iMDFmZWJcIiIsImNyZWF0aW9uX3RpbWUiOiIwMDAxLTAxLTAxVDAwOjAwOjAwWiJ9", + "ip_prefix": "172.224.224.60/31", + "alpha2code": "GB", + "region": "GB-WA", + "city": "Cardiff", + "postal_code": "", + "synctoken": "\"662f6d82-b01feb\"", + "creation_time": "0001-01-01T00:00:00Z" +} diff --git a/providers/ipurl/ipurl.go b/providers/ipurl/ipurl.go index f105f2b..235688c 100644 --- a/providers/ipurl/ipurl.go +++ b/providers/ipurl/ipurl.go @@ -376,10 +376,10 @@ func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { {Number: 2, AutoMerge: true, WidthMax: MaxColumnWidth, WidthMin: 10}, }) tw.SetAutoIndex(false) - tw.SetTitle("IP URLs | Host: %s", c.Host.String()) + tw.SetTitle("IP URL | Host: %s", c.Host.String()) if c.UseTestData { - tw.SetTitle("IP URLs | Host: 5.105.62.60") + tw.SetTitle("IP URL | Host: 5.105.62.60") } return &tw, nil diff --git a/providers/linode/linode.go b/providers/linode/linode.go index 4738217..030a03b 100644 --- a/providers/linode/linode.go +++ b/providers/linode/linode.go @@ -295,10 +295,10 @@ func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) { {Number: 2, AutoMerge: false, WidthMax: MaxColumnWidth, WidthMin: 50}, }) tw.SetAutoIndex(false) - tw.SetTitle("LINODE IP | Host: %s", c.Host.String()) + tw.SetTitle("LINODE | Host: %s", c.Host.String()) if c.UseTestData { - tw.SetTitle("LINODE IP | Host: %s", "69.164.198.1") + tw.SetTitle("LINODE | Host: %s", "69.164.198.1") } return &tw, nil diff --git a/session/config.yaml b/session/config.yaml index 6adf318..d239b6a 100644 --- a/session/config.yaml +++ b/session/config.yaml @@ -34,6 +34,9 @@ providers: - "https://iplists.firehol.org/files/socks_proxy_7d.ipset" - "https://iplists.firehol.org/files/sslproxies_7d.ipset" - "https://iplists.firehol.org/files/tor_exits_7d.ipset" + icloudpr: + enabled: true + # url: override the default URL linode: enabled: true # url: override the default URL diff --git a/session/session.go b/session/session.go index 70995c0..1299da0 100644 --- a/session/session.go +++ b/session/session.go @@ -130,6 +130,10 @@ type Providers struct { Enabled bool `mapstructure:"enabled"` URL string } `mapstructure:"gcp"` + ICloudPR struct { + Enabled bool `mapstructure:"enabled"` + URL string `mapstructure:"url"` + } `mapstructure:"icloudpr"` IPURL struct { Enabled bool `mapstructure:"enabled"` URLs []string `mapstructure:"urls"`