diff --git a/etc/primary_blacklists.yml b/etc/primary_blacklists.yml index cde419d..a371b03 100644 --- a/etc/primary_blacklists.yml +++ b/etc/primary_blacklists.yml @@ -21,6 +21,223 @@ # headers - dict of key-value pairs specyfing additional HTTP headers # Example of url_params and headers usage can be viewed below in Abuse IP DB blacklist specification. iplists: +#Proxy servers +#HTTP +- id: proxy_ercin_http + name: ErcinDedeoglu's HTTP proxy servers list + descr: HTTP protocol, Number of IP adresses = 26782, Updates every ~10min + provider_link: https://github.com/ErcinDedeoglu/proxies + url: https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt + regex: "^(\\A)" + time: + hour: "*" + minute: 0 + +- id: proxy_vakhov_http + name: Vakhov's HTTP proxy servers list + descr: HTTP protocol, Number of IP adresses = 11353, Updates every ~15-30min + provider_link: https://github.com/vakhov/fresh-proxy-list + url: https://raw.githubusercontent.com/vakhov/fresh-proxy-list/master/http.txt + regex: "^(\\A)" + time: + hour: "*" + minute: 0 + +- id: proxy_speed_http + name: TheSpeedX's HTTP proxy servers list + descr: HTTP protocol, Number of IP adresses = 3321, Updates every 3h + provider_link: https://github.com/TheSpeedX/PROXY-List + url: https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt + regex: "^(\\A)" + time: + hour: "*/3" + minute: 0 + +- id: proxy_officialputuid_http + name: officialputuid's HTTP proxy servers list + descr: HTTP protocol, Number of IP adresses = 2440, Updates every 5h + provider_link: https://github.com/officialputuid/KangProxy + url: https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/http/http.txt + regex: "^(\\A)" + time: + hour: "*/5" + minute: 0 + +- id: proxy_vann_http + name: Vann-Dev's HTTP proxy servers list + descr: HTTP protocol, Number of IP adresses = 881, Updates every 3h + provider_link: https://github.com/Vann-Dev/proxy-list + url: https://raw.githubusercontent.com/Vann-Dev/proxy-list/main/proxies/http.txt + regex: "^(\\A)" + time: + hour: "*/3" + minute: 0 + +#HTTPS +- id: proxy_ercin_https + name: ErcinDedeoglu's HTTPS proxy servers list + descr: HTTPS protocol, Number of IP adresses = 26782, Updates every ~10min + provider_link: https://github.com/ErcinDedeoglu/proxies + url: https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/http.txt + regex: "^(\\A)" + time: + hour: "*" + minute: 0 + +- id: proxy_officialputuid_https + name: officialputuid's HTTPS proxy servers list + descr: HTTPS protocol, Number of IP adresses = 3555, Updates every 5h + provider_link: https://github.com/officialputuid/KangProxy + url: https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/https/https.txt + regex: "^(\\A)" + time: + hour: "*/5" + minute: 0 + +- id: proxy_zaeem20_https + name: Zaeem20's HTTPS proxy servers list + descr: HTTPS protocol, Number of IP adresses = 3555, Updates every ~10min + provider_link: https://github.com/Zaeem20/FREE_PROXIES_LIST + url: https://raw.githubusercontent.com/Zaeem20/FREE_PROXIES_LIST/master/https.txt + regex: "^(\\A)" + time: + hour: "*" + minute: 0 + +#SOCKS4 +- id: proxy_ercin_socks4 + name: ErcinDedeoglu's SOCKS4 proxy servers list + descr: SOCKS4 protocol, Number of IP adresses = 8529, Updates every ~10min + provider_link: https://github.com/ErcinDedeoglu/proxies + url: https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks4.txt + regex: "^(\\A)" + time: + hour: "*" + minute: 0 + +- id: proxy_speed_socks4 + name: TheSpeedX's SOCKS4 proxy servers list + descr: SOCKS4 protocol, Number of IP adresses = 2832, Updates every 3hod + provider_link: https://github.com/TheSpeedX/PROXY-List + url: https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt + regex: "^(\\A)" + time: + hour: "*/3" + minute: 0 + +- id: proxy_officialputuid_socks4 + name: officialputuid's SOCKS4 proxy servers list + descr: SOCKS4 protocol, Number of IP adresses = 6732, Updates every 5h + provider_link: https://github.com/officialputuid/KangProxy + url: https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/socks4/socks4.txt + regex: "^(\\A)" + time: + hour: "*/5" + minute: 0 + +#SOCKS5 +- id: proxy_ercin_socks5 + name: ErcinDedeoglu's SOCKS5 proxy servers list + descr: SOCKS5 protocol, Number of IP adresses = 2345, Updates every ~10min + provider_link: https://github.com/ErcinDedeoglu/proxies + url: https://raw.githubusercontent.com/ErcinDedeoglu/proxies/main/proxies/socks5.txt + regex: "^(\\A)" + time: + hour: "*" + minute: 0 + +- id: proxy_speed_socks5 + name: TheSpeedX's SOCKS5 proxy servers list + descr: SOCKS5 protocol, Number of IP adresses = 2486, Updates every 3hod + provider_link: https://github.com/TheSpeedX/PROXY-List + url: https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt + regex: "^(\\A)" + time: + hour: "*/3" + minute: 0 + +- id: proxy_officialputuid_socks5 + name: officialputuid's SOCKS5 proxy servers list + descr: SOCKS5 protocol, Number of IP adresses = 3039, Updates every 5h + provider_link: https://github.com/officialputuid/KangProxy + url: https://raw.githubusercontent.com/officialputuid/KangProxy/KangProxy/socks5/socks5.txt + regex: "^(\\A)" + time: + hour: "*/5" + minute: 0 + +#UNKNOWN +- id: proxy_geonode + name: Geonode's proxy servers list + descr: Unknown protocols, Number of IP adresses = 7399, Updates every ? + provider_link: https://geonode.com/free-proxy-list + url: file:///data/blacklists/geonode_proxy_list.txt + regex: "" + time: + hour: "*" + minute: 0 + +- id: proxy_proxyscrape + name: Proxyscrape's proxy servers list + descr: Unknown protocols, Number of IP adresses = 1135, Updates every ? + provider_link: https://proxyscrape.com/ + url: https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&proxy_format=protocolipport&format=text + regex: "(\\A)" + time: + hour: "1" + minute: 0 + +- id: proxy_fineproxy + name: Fineproxy's proxy servers list + descr: Unknown protocols, Number of IP adresses = 2837, Updates every 30min, The source URL may change + provider_link: https://fineproxy.org/free-proxy/ + url: https://fineproxy.org/wp-admin/admin-ajax.php?action=proxylister_download&nonce=62a3af4c7b&format=txt&filter={} + regex: "^(\\A)" + time: + hour: "*" + minute: 0 + +- id: proxy_yemixzy + name: Yemixzy's proxy servers list + descr: Unknown protocols, Number of IP adresses = 10902, Updates every 3hod, Contains 46% duplicates (size is without them) + provider_link: https://github.imc.re/yemixzy/proxy-list + url: https://raw.githubusercontent.com/yemixzy/proxy-list/main/proxies/unchecked.txt + regex: "^(\\A)" + time: + hour: "*/3" + minute: 0 + +#VPN Servers +- id: vpn_X4BNet_datacenter + name: X4BNet's vpn datacenter servers list + descr: Number of IP adresses = 30082, Updates every day + provider_link: https://github.com/X4BNet/lists_vpn + url: https://raw.githubusercontent.com/X4BNet/lists_vpn/main/output/datacenter/ipv4.txt + regex: "^(\\A)" + time: + hour: "1" + minute: 0 + +- id: vpn_X4BNet_vpn + name: X4BNet's vpn servers list + descr: Number of IP adresses = 3756, Updates every day + provider_link: https://github.com/X4BNet/lists_vpn + url: https://raw.githubusercontent.com/X4BNet/lists_vpn/main/output/vpn/ipv4.txt + regex: "^(\\A)" + time: + hour: "1" + minute: 0 + +- id: vpn_az0 + name: az0's vpn servers list + descr: Number of IP adresses = 20657, Updates every 12h, Contains 25% duplicates + provider_link: https://github.com/az0/vpn_ip + url: https://raw.githubusercontent.com/az0/vpn_ip/main/data/output/ip.txt + regex: "^(\\A)" + time: + hour: "*/12" + minute: 0 + - id: feodo name: Feodo tracker descr: Feodo Tracker is a project of abuse.ch. Shares botnet C&C
servers associated with the Feodo malware family(Dridex, Emotet/Heodo)
as well as TrickBot and QakBot (aka QuakBot / Qbot) diff --git a/scripts/fetch_geonode.py b/scripts/fetch_geonode.py new file mode 100644 index 0000000..8ae91fa --- /dev/null +++ b/scripts/fetch_geonode.py @@ -0,0 +1,43 @@ +#Author: Adam.Stefanides@cesnet.cz + +import requests +import math +import sys + +MAX_N_OF_REQUESTS = 100 + +def get_page(n: int) -> dict: + ''' + Descr: Sends GET request to an url with a specified page and returns it as json obj + :param n: page number + :return: json obj [dict] + ''' + url = "https://proxylist.geonode.com/api/proxy-list?limit=500&page=" + response = requests.get(url + str(n)) + + return response.json() + +def get_file(n: int) -> str: + ''' + Descr: Donwloads N pages from an url, extracts ip adresses and returns them + :param n: number of pages + :return: string of ip adresses + ''' + output = "" + for i in range(1, n + 1): + obj = get_page(i) + for rec in obj["data"]: + # extracting ip + output += rec["ip"] + "\n" + return output[:-1] + +#Calculating number of pages of API to download +obj = get_page(1) +pages = math.ceil(obj["total"] / obj["limit"]) +if pages > MAX_N_OF_REQUESTS: + print("ERROR: number of pages exceeds the number of allowed requests!", file=sys.stderr) + sys.exit(1) + +#Write the ip aresses into a file +with open("/data/blacklists/geonode_proxy_list.txt", "w") as file: + file.write(get_file(pages)) \ No newline at end of file