From 2c03eef5979c97a8f3b91683a1b77efc19229fcd Mon Sep 17 00:00:00 2001 From: Bradley Kemp Date: Sat, 25 Jul 2020 14:15:13 +0100 Subject: [PATCH] Major refactor to enable usage as a library --- abwhose.go | 140 +++++++++++++++------------------- email_template.go | 59 ++++++++++++++ matchers.go | 137 --------------------------------- matchers/email_fallback.go | 34 +++++++++ matchers/provider_contacts.go | 50 ++++++++++++ matchers/shared_hosting.go | 38 +++++++++ matchers/types.go | 26 +++++++ matchers/whois_matchers.go | 49 ++++++++++++ 8 files changed, 316 insertions(+), 217 deletions(-) create mode 100644 email_template.go delete mode 100644 matchers.go create mode 100644 matchers/email_fallback.go create mode 100644 matchers/provider_contacts.go create mode 100644 matchers/shared_hosting.go create mode 100644 matchers/types.go create mode 100644 matchers/whois_matchers.go diff --git a/abwhose.go b/abwhose.go index b9c33dd..c755ef4 100644 --- a/abwhose.go +++ b/abwhose.go @@ -2,14 +2,12 @@ package main import ( "fmt" - "net" "net/url" "os" - "os/exec" "strings" "text/tabwriter" - "golang.org/x/net/publicsuffix" + "github.com/bradleyjkemp/abwhose/matchers" ) func main() { @@ -23,101 +21,83 @@ func main() { } } -var tabWriter = tabwriter.NewWriter(os.Stdout, 12, 2, 1, ' ', tabwriter.TabIndent) - -func run(abuseURL string) error { - var abusive *url.URL - var err error - if strings.Contains(abuseURL, "/") { - // This looks like a full URL instead of a plain domain - if !strings.HasPrefix(abuseURL, "http://") && !strings.HasPrefix(abuseURL, "https://") { - // Doesn't have a protocol so won't url.Parse properly - abuseURL = "http://" + abuseURL - } - - abusive, err = url.Parse(abuseURL) - if err != nil { - return fmt.Errorf("couldn't parse URL: %w", err) - } - } else { - // This is a plain domain name so we construct a URL directly - abusive = &url.URL{ - Scheme: "http", - Host: abuseURL, - } - } - - if abusive.Hostname() == "" { - return fmt.Errorf("%s doesn't look like a valid URL (hostname is empty)", abuseURL) +func run(query string) error { + u, err := parseURL(query) + if err != nil { + return err } - // First look up abuse details for the domain itself (this will be the registrar) - rootDomain, _ := publicsuffix.EffectiveTLDPlusOne(abusive.Hostname()) - - // First check if this is a shared host - var sharedHost bool - for _, matcher := range sharedHostMatchers { - if match, display := matcher(rootDomain); match { - if !sharedHost { - fmt.Println("Report abuse to shared hosting provider:") - } - display() - sharedHost = true - } - } - // If this is a shared host then skip the WHOIS lookup - // as that information isn't useful. - if sharedHost { + if ok, contact := matchers.IsSharedHostingProvider(u); ok { + fmt.Println("Report abuse to shared hosting provider:") + printContactDetails(u, contact) + // If this is a shared host then skip the rest of the lookups + // as that information isn't useful. return nil } - err = getAbuseReportDetails("Report abuse to domain registrar:", abusive, rootDomain) + contacts, err := matchers.Registrar(u) if err != nil { - return fmt.Errorf("failed to get registrar abuse details: %w", err) - } - - // Now look up the IP in order to find the hosting provider - ips, err := net.LookupIP(abusive.Hostname()) - if err != nil { - return fmt.Errorf("failed to find hosting provider: %w", err) + return err } + fmt.Println("Report abuse to domain registrar:") + printContactDetails(u, contacts...) - // Abuse details for the IP should be the hosting provider - err = getAbuseReportDetails("Report abuse to host:", abusive, ips[0].String()) + contacts, err = matchers.HostingProvider(u) if err != nil { - return fmt.Errorf("failed to get host abuse details: %w", err) + return err } + fmt.Println("Report abuse to hosting provider:") + printContactDetails(u, contacts...) return nil } -func getAbuseReportDetails(header string, abusive *url.URL, query string) error { - rawWhois, err := exec.Command("whois", query).CombinedOutput() +func parseURL(input string) (*url.URL, error) { + if !strings.Contains(input, "/") { + // This is likely a plain domain name so we construct a URL directly + return &url.URL{ + Scheme: "http", + Host: input, + }, nil + } + + // This looks like a full URL instead of a plain domain + if !strings.HasPrefix(input, "http://") && !strings.HasPrefix(input, "https://") { + // Doesn't have a protocol so won't url.Parse properly + input = "http://" + input + } + + u, err := url.Parse(input) if err != nil { - return err + return nil, fmt.Errorf("couldn't parse URL: %w", err) + } + if u.Hostname() == "" { + return nil, fmt.Errorf("%s doesn't look like a valid URL (hostname is empty)", input) } - gotMatch := false - for _, matcher := range whoisMatchers { - if match, display := matcher(string(rawWhois)); match { - if !gotMatch { - fmt.Println(header) - gotMatch = true + return u, nil +} + +var tabWriter = tabwriter.NewWriter(os.Stdout, 12, 2, 1, ' ', tabwriter.TabIndent) + +func printContactDetails(u *url.URL, contacts ...matchers.ProviderContact) { + for _, contact := range contacts { + switch c := contact.(type) { + case matchers.AbuseEmail: + if emailTemplateConfigured() { + offerToSendEmail(u, c) + } else { + fmt.Fprintf(tabWriter, " Email:\t%s\n", c.Email) } - display() + + case matchers.OnlineForm: + fmt.Fprintf(tabWriter, " %s:\tFill out abuse form %s\n", contact.Name(), c.URL) + + default: + panic(fmt.Sprintf("unknown contact type: %T", contact)) } } - if gotMatch { - return nil + if len(contacts) == 0 { + fmt.Fprintf(tabWriter, " Couldn't find any contact details\n") } - - // None of the specific matchers hit so use a generic one - found, display := fallbackEmailMatcher(header, abusive, string(rawWhois)) - if found { - display() - return nil - } - - fmt.Println(header) - fmt.Println(" couldn't find any abuse contact details") - return nil + tabWriter.Flush() } diff --git a/email_template.go b/email_template.go new file mode 100644 index 0000000..5203521 --- /dev/null +++ b/email_template.go @@ -0,0 +1,59 @@ +package main + +import ( + "bytes" + "fmt" + "html/template" + "io/ioutil" + "net/url" + "os" + "os/exec" + "strings" + + "github.com/bradleyjkemp/abwhose/matchers" +) + +func emailTemplateConfigured() bool { + _, configured := os.LookupEnv("ABWHOSE_MAILTO_TEMPLATE") + return configured +} + +func offerToSendEmail(u *url.URL, contact matchers.AbuseEmail) { + emailTemplateFile, _ := os.LookupEnv("ABWHOSE_MAILTO_TEMPLATE") + emailTemplateContents, err := ioutil.ReadFile(emailTemplateFile) + if err != nil { + fmt.Printf("Failed reading email template: %v\n", err) + return + } + mailto := &bytes.Buffer{} + err = template.Must(template.New("email").Parse(string(emailTemplateContents))).Execute(mailto, map[string]interface{}{ + "domain": strings.Replace(u.Hostname(), ".", "[.]", -1), + "url": strings.Replace(u.Hostname(), ".", "[.]", -1) + u.RawPath + u.RawQuery, + "recipient": contact.Email, + }) + if err != nil { + fmt.Printf("Error templating email: %v\n", err) + return + } + fmt.Printf(" Send email to %s? [Y/n] ", contact.Email) + if userSaysYes() { + exec.Command("open", mailto.String()).Run() + } +} + +func userSaysYes() bool { + var response string + _, err := fmt.Scanln(&response) + if err != nil && err.Error() != "unexpected newline" { + panic(err) + } + okayResponses := map[string]bool{ + "": true, + "y": true, + "yes": true, + } + if okayResponses[strings.ToLower(response)] { + return true + } + return false +} diff --git a/matchers.go b/matchers.go deleted file mode 100644 index fa41cce..0000000 --- a/matchers.go +++ /dev/null @@ -1,137 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "net/url" - "os" - "os/exec" - "regexp" - "sort" - "strings" - "text/template" -) - -type matcher func(string) (match bool, display func()) - -var sharedHostMatchers = []matcher{ - domainMatcher("000webhost.com", onlineFormMessage("000webhost", "https://www.000webhost.com/report-abuse")), - domainMatcher("000webhostapp.com", onlineFormMessage("00webhost", "https://www.000webhost.com/report-abuse")), - domainMatcher("blogger.com", onlineFormMessage("Blogger", "https://support.google.com/blogger/answer/76315")), - domainMatcher("blogspot.com", onlineFormMessage("Blogger", "https://support.google.com/blogger/answer/76315")), - domainMatcher("weebly.com", onlineFormMessage("Weebly", "https://www.weebly.com/uk/spam")), - domainMatcher("appspot.com", onlineFormMessage("Google Cloud", "https://support.google.com/code/contact/cloud_platform_report")), - domainMatcher("googleapis.com", onlineFormMessage("Google Cloud", "https://support.google.com/code/contact/cloud_platform_report")), -} - -var whoisMatchers = []matcher{ - containsMatcher("abuse@cloudflare.com", onlineFormMessage("Cloudflare", "https://www.cloudflare.com/abuse/form")), - containsMatcher("abuse@godaddy.com", onlineFormMessage("GoDaddy", "https://supportcenter.godaddy.com/AbuseReport")), - containsMatcher("abuse@namecheap.com", onlineFormMessage("Namecheap", "https://support.namecheap.com/index.php?/Tickets/Submit")), - containsMatcher("abuse@namesilo.com", onlineFormMessage("Namesilo", "https://www.namesilo.com/report_abuse.php or https://new.namesilo.com/phishing_report.php")), - containsMatcher("abuse-contact@publicdomainregistry.com", onlineFormMessage("PublicDomainRegistry", "http://publicdomainregistry.com/report-abuse-complain/")), - containsMatcher("abuse@tucows.com", onlineFormMessage("Tucows", "https://tucowsdomains.com/report-abuse/")), - containsMatcher("domainabuse@tucows.com", onlineFormMessage("Tucows", "https://tucowsdomains.com/report-abuse/")), -} - -var emailMatchers = []*regexp.Regexp{ - regexp.MustCompile(`(?i)(abuse@[a-z0-9\-.]*)`), - regexp.MustCompile(`(?m)^OrgAbuseEmail:\s+(.*)$`), - regexp.MustCompile(`(?m)^Registrar Abuse Contact Email:\s+(.+)$`), -} - -func fallbackEmailMatcher(header string, abusive *url.URL, whois string) (bool, func()) { - var emails = map[string]struct{}{} - for _, matcher := range emailMatchers { - for _, email := range matcher.FindAllStringSubmatch(whois, -1) { - emails[strings.TrimSpace(email[1])] = struct{}{} - } - } - if len(emails) == 0 { - return false, nil - } - - sortedEmails := make([]string, 0, len(emails)) - for email := range emails { - sortedEmails = append(sortedEmails, email) - } - sort.Slice(sortedEmails, func(i, j int) bool { - return sortedEmails[i] < sortedEmails[j] - }) - - return true, func() { - emailTemplateFile, found := os.LookupEnv("ABWHOSE_MAILTO_TEMPLATE") - if !found { - fmt.Println(header) - fmt.Fprintf(tabWriter, " Email:\t%s\n", sortedEmails) - tabWriter.Flush() - return - } - - emailTemplateContents, err := ioutil.ReadFile(emailTemplateFile) - if err != nil { - fmt.Printf("Failed reading email template: %v\n", err) - return - } - mailto := &bytes.Buffer{} - err = template.Must(template.New("email").Parse(string(emailTemplateContents))).Execute(mailto, map[string]interface{}{ - "domain": strings.Replace(abusive.Hostname(), ".", "[.]", -1), - "url": strings.Replace(abusive.Hostname(), ".", "[.]", -1) + abusive.RawPath + abusive.RawQuery, - "recipient": strings.Join(sortedEmails, ";"), - }) - if err != nil { - fmt.Printf("Error templating email: %v\n", err) - return - } - fmt.Println(header) - fmt.Printf(" Send email to %s? [Y/n] ", sortedEmails) - if userSaysYes() { - exec.Command("open", mailto.String()).Run() - } - } -} - -func userSaysYes() bool { - var response string - _, err := fmt.Scanln(&response) - if err != nil && err.Error() != "unexpected newline" { - panic(err) - } - okayResponses := map[string]bool{ - "": true, - "y": true, - "yes": true, - } - if okayResponses[strings.ToLower(response)] { - return true - } - return false -} - -func containsMatcher(contents string, display func()) matcher { - return func(whois string) (bool, func()) { - if strings.Contains(whois, contents) { - return true, display - } - - return false, nil - } -} - -func domainMatcher(domain string, display func()) matcher { - return func(abusiveDomain string) (bool, func()) { - if abusiveDomain == domain || strings.HasSuffix(abusiveDomain, "."+domain) { - return true, display - } - - return false, nil - } -} - -func onlineFormMessage(name, url string) func() { - return func() { - fmt.Fprintf(tabWriter, " %s:\tFill out abuse form %s\n", name, url) - tabWriter.Flush() - } -} diff --git a/matchers/email_fallback.go b/matchers/email_fallback.go new file mode 100644 index 0000000..bbad75c --- /dev/null +++ b/matchers/email_fallback.go @@ -0,0 +1,34 @@ +package matchers + +import ( + "regexp" + "sort" + "strings" +) + +var emailRegexes = []*regexp.Regexp{ + regexp.MustCompile(`(?i)(abuse@[a-z0-9\-.]*)`), + regexp.MustCompile(`(?m)^OrgAbuseEmail:\s+(.*)$`), + regexp.MustCompile(`(?m)^Registrar Abuse Contact Email:\s+(.+)$`), +} + +func getRawEmailContacts(rawWhois string) []ProviderContact { + var emails = map[string]struct{}{} + for _, matcher := range emailRegexes { + for _, email := range matcher.FindAllStringSubmatch(rawWhois, -1) { + emails[strings.TrimSpace(email[1])] = struct{}{} + } + } + if len(emails) == 0 { + return nil + } + + sortedEmails := make([]ProviderContact, 0, len(emails)) + for email := range emails { + sortedEmails = append(sortedEmails, AbuseEmail{providerID(email), email}) + } + sort.Slice(sortedEmails, func(i, j int) bool { + return sortedEmails[i].(AbuseEmail).Email < sortedEmails[j].(AbuseEmail).Email + }) + return sortedEmails +} diff --git a/matchers/provider_contacts.go b/matchers/provider_contacts.go new file mode 100644 index 0000000..2c5baa0 --- /dev/null +++ b/matchers/provider_contacts.go @@ -0,0 +1,50 @@ +package matchers + +import ( + "fmt" + "net" + "net/url" + "sort" + + "golang.org/x/net/publicsuffix" +) + +// Gets the abuse contact details for the registrar of a domain name. +func Registrar(u *url.URL) ([]ProviderContact, error) { + // First look up abuse details for the domain itself + rootDomain, err := publicsuffix.EffectiveTLDPlusOne(u.Hostname()) + if err != nil { + return nil, fmt.Errorf("failed to get root domain: %w", err) + } + + return getContactsFromWHOIS(rootDomain) +} + +// Gets the abuse contact details for the hosting provider of a domain name. +func HostingProvider(u *url.URL) ([]ProviderContact, error) { + ips, err := net.LookupIP(u.Hostname()) + if err != nil { + return nil, fmt.Errorf("failed to find hosting provider: %w", err) + } + + contacts := map[string]ProviderContact{} + for _, ip := range ips { + ipContacts, err := getContactsFromWHOIS(ip.String()) + if err != nil { + return nil, err + } + + for _, contact := range ipContacts { + contacts[contact.Name()] = contact + } + } + + dedupedContacts := make([]ProviderContact, 0, len(contacts)) + for _, contact := range contacts { + dedupedContacts = append(dedupedContacts, contact) + } + sort.Slice(dedupedContacts, func(i, j int) bool { + return dedupedContacts[i].Name() < dedupedContacts[j].Name() + }) + return dedupedContacts, nil +} diff --git a/matchers/shared_hosting.go b/matchers/shared_hosting.go new file mode 100644 index 0000000..ab8e81e --- /dev/null +++ b/matchers/shared_hosting.go @@ -0,0 +1,38 @@ +package matchers + +import ( + "net/url" + "strings" +) + +// Returns the contact details of the shared hosting provider if it exists. +// If this matches, these contact details should be preferred over the +// registrar and hosting provider. +func IsSharedHostingProvider(u *url.URL) (bool, ProviderContact) { + for _, m := range sharedHostMatchers { + if m.matches(u.Host) { + return true, m.contact + } + } + return false, nil +} + +// Matches content served by shared hosting providers i.e. where the abusive content +// is not served by the domain/server owner. +// +// Try to keep this sorted alphabetically by providerID +var sharedHostMatchers = []matcher{ + {OnlineForm{"000webhost", "https://www.000webhost.com/report-abuse"}, isSubDomainOf("000webhost.com")}, + {OnlineForm{"00webhost", "https://www.000webhost.com/report-abuse"}, isSubDomainOf("000webhostapp.com")}, + {OnlineForm{"Blogger", "https://support.google.com/blogger/answer/76315"}, isSubDomainOf("blogger.com")}, + {OnlineForm{"Blogger", "https://support.google.com/blogger/answer/76315"}, isSubDomainOf("blogspot.com")}, + {OnlineForm{"Google Cloud", "https://support.google.com/code/contact/cloud_platform_report"}, isSubDomainOf("appspot.com")}, + {OnlineForm{"Google Cloud", "https://support.google.com/code/contact/cloud_platform_report"}, isSubDomainOf("googleapis.com")}, + {OnlineForm{"Weebly", "https://www.weebly.com/uk/spam"}, isSubDomainOf("weebly.com")}, +} + +func isSubDomainOf(domain string) func(string) bool { + return func(abusiveDomain string) bool { + return abusiveDomain == domain || strings.HasSuffix(abusiveDomain, "."+domain) + } +} diff --git a/matchers/types.go b/matchers/types.go new file mode 100644 index 0000000..d28b1f6 --- /dev/null +++ b/matchers/types.go @@ -0,0 +1,26 @@ +package matchers + +type ProviderContact interface { + Name() string // Returns a Name that uniquely identifies the recipient of an abuse report +} + +type OnlineForm struct { + providerID + URL string +} + +type AbuseEmail struct { + providerID + Email string +} + +type providerID string + +func (m providerID) Name() string { + return string(m) +} + +type matcher struct { + contact ProviderContact + matches func(string) bool +} diff --git a/matchers/whois_matchers.go b/matchers/whois_matchers.go new file mode 100644 index 0000000..86748b8 --- /dev/null +++ b/matchers/whois_matchers.go @@ -0,0 +1,49 @@ +package matchers + +import ( + "fmt" + "os/exec" + "strings" +) + +// Matches WHOIS data to the best way to report abuse to the registrar/hosting provider. +// +// Try to keep this sorted alphabetically by providerID +var whoisMatchers = []matcher{ + {OnlineForm{"Cloudflare", "https://www.cloudflare.com/abuse/form"}, whoisContains("abuse@cloudflare.com")}, + {OnlineForm{"GoDaddy", "https://supportcenter.godaddy.com/AbuseReport"}, whoisContains("abuse@godaddy.com")}, + {OnlineForm{"Namecheap", "https://support.namecheap.com/index.php?/Tickets/Submit"}, whoisContains("abuse@namecheap.com")}, + {OnlineForm{"Namesilo", "https://www.namesilo.com/report_abuse.php or https://new.namesilo.com/phishing_report.php"}, whoisContains("abuse@namesilo.com")}, + {AbuseEmail{"OrangeWebsite", "abuse-dept@orangewebsite.com"}, whoisContains("abuse@orangewebsite.com")}, + {OnlineForm{"PublicDomainRegistry", "https://publicdomainregistry.com/process-for-handling-abuse/"}, whoisContains("abuse-contact@publicdomainregistry.com")}, + {OnlineForm{"Tucows", "https://tucowsdomains.com/report-abuse/"}, whoisContains("abuse@tucows.com")}, + {OnlineForm{"Tucows", "https://tucowsdomains.com/report-abuse/"}, whoisContains("domainabuse@tucows.com")}, +} + +func whoisContains(contents string) func(string) bool { + return func(whois string) bool { + return strings.Contains(whois, contents) + } +} + +func getContactsFromWHOIS(query string) ([]ProviderContact, error) { + rawWhois, err := exec.Command("whois", query).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to query whois: %s, %w", string(rawWhois), err) + } + + var contacts []ProviderContact + for _, m := range whoisMatchers { + if m.matches(string(rawWhois)) { + contacts = append(contacts, m.contact) + } + } + + // One of the whoisMatchers matched so return that info + if len(contacts) > 0 { + return contacts, nil + } + + // Nothing matched so try and extract raw email addresses + return getRawEmailContacts(string(rawWhois)), nil +}