From d419823b49c3ba781c88cedf72d5bc51fa47e4b5 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Fri, 19 Oct 2018 13:08:26 +0100 Subject: [PATCH 01/39] :bug: Regex not parsing response in `asn.py` The regex expression was not parsing the response correctly in `asn.py:get_nets_radb`. May issue seems to be the ipwhois data may have changed no long ends in
instead with \n. --- ipwhois/asn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 6f15fa1e..0ce8ef91 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -740,9 +740,9 @@ def get_nets_radb(self, response, is_http=False): nets = [] if is_http: - regex = r'route(?:6)?:[^\S\n]+(?P.+?)
' + regex = r'route(?:6)?:\s+(?P.+?).?\n' else: - regex = r'^route(?:6)?:[^\S\n]+(?P.+|.+)$' + regex = r'^route(?:6)?:\s+(?P.+|.+)$' # Iterate through all of the networks found, storing the CIDR value # and the start and end positions. From 48619682d0ef1356a046d2c3fddf91ceaa821161 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Fri, 19 Oct 2018 14:35:46 +0100 Subject: [PATCH 02/39] :white_check_mark: Fixed unit test I think unit test should be passing, fixed unit test so it passes. The following test test_asn:test__get_nets_radb was asserting an empty list [], but I think the data should be parseable. --- ipwhois/asn.py | 4 ++-- ipwhois/tests/test_asn.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 0ce8ef91..9babae12 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -740,9 +740,9 @@ def get_nets_radb(self, response, is_http=False): nets = [] if is_http: - regex = r'route(?:6)?:\s+(?P.+?).?\n' + regex = r'route(?:6)?:[^\S\n]+(?P.+?)\n' else: - regex = r'^route(?:6)?:\s+(?P.+|.+)$' + regex = r'^route(?:6)?:[^\S\n]+(?P.+|.+)$' # Iterate through all of the networks found, storing the CIDR value # and the start and end positions. diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index 5bdb134b..0485abb2 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -242,7 +242,10 @@ def test__get_nets_radb(self): obj._get_nets_radb(multi_net_response) self.assertEqual(obj._get_nets_radb(multi_net_response, is_http=True), - []) + [{'cidr': '66.249.64.0/20', 'description': None, 'maintainer': None, 'updated': None, + 'source': None, 'start': 2, 'end': 29}, + {'cidr': '66.249.80.0/20', 'description': None, 'maintainer': None, 'updated': None, + 'source': None, 'start': 175, 'end': 202}]) net = Net('2001:43f8:7b0::') obj = ASNOrigin(net) From 600b4b09e47ab913c4841dac0d5a14a735ba989f Mon Sep 17 00:00:00 2001 From: secynic Date: Fri, 1 Feb 2019 23:59:44 -0600 Subject: [PATCH 03/39] remove deprecated (#230) --- ASN.rst | 12 ---- CHANGES.rst | 13 ++++ CLI.rst | 8 +-- RDAP.rst | 6 -- README.rst | 5 -- UPGRADING.rst | 14 +++++ WHOIS.rst | 6 -- ipwhois/asn.py | 91 ++-------------------------- ipwhois/ipwhois.py | 25 ++------ ipwhois/net.py | 31 +--------- ipwhois/nir.py | 40 ------------ ipwhois/scripts/ipwhois_cli.py | 17 ------ ipwhois/tests/online/test_ipwhois.py | 5 -- ipwhois/whois.py | 40 ------------ 14 files changed, 41 insertions(+), 272 deletions(-) diff --git a/ASN.rst b/ASN.rst index f2ef8f16..bcfed90d 100644 --- a/ASN.rst +++ b/ASN.rst @@ -24,12 +24,6 @@ Arguments supported by IPASN.lookup(). | | | resets, etc. are encountered. | | | | Defaults to 3. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN | | | | REST (ASN fallback HTTP lookup) does not | @@ -157,12 +151,6 @@ Arguments supported by ASNOrigin.lookup(). | | | ['description', 'maintainer', 'updated', | | | | 'source']. If None, defaults to all. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | asn_methods | list | ASN lookup types to attempt, in order. If | | | | None, defaults to all ['whois', 'http']. | +------------------------+--------+-------------------------------------------+ diff --git a/CHANGES.rst b/CHANGES.rst index 0a2ec4f0..024080b5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,19 @@ Changelog ========= +1.2.0 (TBD) +------------------ + +- Removed deprecated functions: asn.IPASN._parse_fields_http, + asn.IPASN._parse_fields_dns, asn.IPASN._parse_fields_whois, + asn.ASNOrigin._parse_fields, asn.ASNOrigin._get_nets_radb, + net.Net.lookup_asn, whois.Whois._parse_fields, whois.Whois._get_nets_arin + whois.Whois._get_nets_lacnic, whois.Whois._get_nets_other, + nir.NIRWhois._parse_fields, nir.NIRWhois._get_nets_jpnic + nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact (#230) +- Removed deprecated asn_alts parameter (#230) +- Removed deprecated allow_permutations parameter (#230) + 1.1.0 (2019-02-01) ------------------ diff --git a/CLI.rst b/CLI.rst index cf4a986e..03902a24 100644 --- a/CLI.rst +++ b/CLI.rst @@ -22,7 +22,7 @@ ipwhois_cli.py [-h] [--whois] [--exclude_nir] [--json] [--hr] [--proxy_http "PROXY_HTTP"] [--proxy_https "PROXY_HTTPS"] [--inc_raw] [--retry_count RETRY_COUNT] - [--asn_alts "ASN_ALTS"] [--asn_methods "ASN_METHODS"] + [--asn_methods "ASN_METHODS"] [--extra_org_map "EXTRA_ORG_MAP"] [--skip_asn_description] [--depth COLOR_DEPTH] [--excluded_entities "EXCLUDED_ENTITIES"] [--bootstrap] @@ -66,12 +66,6 @@ Common settings (RDAP & Legacy Whois): --retry_count RETRY_COUNT The number of times to retry in case socket errors, timeouts, connection resets, etc. are encountered. - --asn_alts ASN_ALTS - A comma delimited list of additional lookup types to - attempt if the ASN dns lookup fails. Allow - permutations must be enabled. Defaults to all: - "whois,http". *WARNING* deprecated in favor of new - argument asn_methods. --asn_methods ASN_METHODS List of ASN lookup types to attempt, in order. Defaults to all ['dns', 'whois', 'http']. diff --git a/RDAP.rst b/RDAP.rst index f216b3a9..9d32b50a 100644 --- a/RDAP.rst +++ b/RDAP.rst @@ -42,12 +42,6 @@ Arguments supported by IPWhois.lookup_rdap(). | | | when a rate limit notice is returned via | | | | rdap+json. Defaults to 120. | +--------------------+--------+-----------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the ASN | -| | | dns lookup fails. Allow permutations must be | -| | | enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated in | -| | | favor of new argument asn_methods. | -+--------------------+--------+-----------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN REST | | | | (ASN fallback HTTP lookup) does not show an | diff --git a/README.rst b/README.rst index eca1a080..712c9392 100644 --- a/README.rst +++ b/README.rst @@ -170,11 +170,6 @@ Input | proxy_opener | object | The urllib.request.OpenerDirector request for | | | | proxy support or None. | +--------------------+--------+-----------------------------------------------+ -| allow_permutations | bool | Allow net.Net() to use additional methods if | -| | | DNS lookups to Cymru fail. *WARNING* | -| | | deprecated in favor of new argument | -| | | asn_methods. Defaults to False. | -+--------------------+--------+-----------------------------------------------+ RDAP (HTTP) ----------- diff --git a/UPGRADING.rst b/UPGRADING.rst index 679db1a4..d26f5f20 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -9,6 +9,20 @@ any changes that may affect user experience when upgrading to a new release. This page is new as of version 1.0.0. Any information on older versions is likely missing or incomplete. +****** +v1.2.0 +****** + +- Removed deprecated functions: asn.IPASN._parse_fields_http, + asn.IPASN._parse_fields_dns, asn.IPASN._parse_fields_whois, + asn.ASNOrigin._parse_fields, asn.ASNOrigin._get_nets_radb, + net.Net.lookup_asn, whois.Whois._parse_fields, whois.Whois._get_nets_arin + whois.Whois._get_nets_lacnic, whois.Whois._get_nets_other, + nir.NIRWhois._parse_fields, nir.NIRWhois._get_nets_jpnic + nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact +- Removed deprecated asn_alts parameter +- Removed deprecated allow_permutations parameter + ****** v1.1.0 ****** diff --git a/WHOIS.rst b/WHOIS.rst index bb7a9b0f..0d7b4963 100644 --- a/WHOIS.rst +++ b/WHOIS.rst @@ -53,12 +53,6 @@ Arguments supported by IPWhois.lookup_whois(). | | | 'postal_code', 'emails', 'created', | | | | 'updated']. If None, defaults to all. | +------------------------+--------+-------------------------------------------+ -| asn_alts | list | Additional lookup types to attempt if the | -| | | ASN dns lookup fails. Allow permutations | -| | | must be enabled. If None, defaults to all | -| | | ['whois', 'http']. *WARNING* deprecated | -| | | in favor of new argument asn_methods. | -+------------------------+--------+-------------------------------------------+ | extra_org_map | dict | Dictionary mapping org handles to RIRs. | | | | This is for limited cases where ARIN | | | | REST (ASN fallback HTTP lookup) does not | diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 03946fe3..d6266626 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -169,16 +169,6 @@ def parse_fields_dns(self, response): return ret - def _parse_fields_dns(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_dns() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_dns().') - return self.parse_fields_dns(*args, **kwargs) - def parse_fields_verbose_dns(self, response): """ The function for parsing ASN fields from a verbose dns response. @@ -293,16 +283,6 @@ def parse_fields_whois(self, response): return ret - def _parse_fields_whois(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_whois() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_whois().') - return self.parse_fields_whois(*args, **kwargs) - def parse_fields_http(self, response, extra_org_map=None): """ The function for parsing ASN fields from a http response. @@ -403,19 +383,8 @@ def parse_fields_http(self, response, extra_org_map=None): return asn_data - def _parse_fields_http(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('IPASN._parse_fields_http() has been deprecated and will be ' - 'removed. You should now use IPASN.parse_fields_http().') - return self.parse_fields_http(*args, **kwargs) - - def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, - extra_org_map=None, asn_methods=None, - get_asn_description=True): + def lookup(self, inc_raw=False, retry_count=3, extra_org_map=None, + asn_methods=None, get_asn_description=True): """ The wrapper function for retrieving and parsing ASN information for an IP address. @@ -426,10 +395,6 @@ def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, retry_count (:obj:`int`): The number of times to retry in case socket errors, timeouts, connection resets, etc. are encountered. Defaults to 3. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - Defaults to all ['whois', 'http']. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to None. extra_org_map (:obj:`dict`): Mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC (which is @@ -466,17 +431,7 @@ def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, if asn_methods is None: - if asn_alts is None: - - lookups = ['dns', 'whois', 'http'] - - else: - - from warnings import warn - warn('IPASN.lookup() asn_alts argument has been deprecated ' - 'and will be removed. You should now use the asn_methods ' - 'argument.') - lookups = ['dns'] + asn_alts + lookups = ['dns', 'whois', 'http'] else: @@ -492,8 +447,7 @@ def lookup(self, inc_raw=False, retry_count=3, asn_alts=None, dns_success = False for index, lookup_method in enumerate(lookups): - if index > 0 and not asn_methods and not ( - self._net.allow_permutations): + if index > 0 and not asn_methods: raise ASNRegistryError('ASN registry lookup failed. ' 'Permutations not allowed.') @@ -706,16 +660,6 @@ def parse_fields(self, response, fields_dict, net_start=None, return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('ASNOrigin._parse_fields() has been deprecated and will be ' - 'removed. You should now use ASNOrigin.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_radb(self, response, is_http=False): """ The function for parsing network blocks from ASN origin data. @@ -769,18 +713,8 @@ def get_nets_radb(self, response, is_http=False): return nets - def _get_nets_radb(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('ASNOrigin._get_nets_radb() has been deprecated and will be ' - 'removed. You should now use ASNOrigin.get_nets_radb().') - return self.get_nets_radb(*args, **kwargs) - def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, - field_list=None, asn_alts=None, asn_methods=None): + field_list=None, asn_methods=None): """ The function for retrieving and parsing ASN origin whois information via port 43/tcp (WHOIS). @@ -797,9 +731,6 @@ def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, field_list (:obj:`list`): If provided, fields to parse: ['description', 'maintainer', 'updated', 'source'] If None, defaults to all. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN whois lookup fails. If None, defaults to all ['http']. - *WARNING* deprecated in favor of new argument asn_methods. asn_methods (:obj:`list`): ASN lookup types to attempt, in order. If None, defaults to all ['whois', 'http']. @@ -828,17 +759,7 @@ def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, if asn_methods is None: - if asn_alts is None: - - lookups = ['whois', 'http'] - - else: - - from warnings import warn - warn('ASNOrigin.lookup() asn_alts argument has been deprecated' - ' and will be removed. You should now use the asn_methods' - ' argument.') - lookups = ['whois'] + asn_alts + lookups = ['whois', 'http'] else: diff --git a/ipwhois/ipwhois.py b/ipwhois/ipwhois.py index 4a4d56bd..0c3e179d 100644 --- a/ipwhois/ipwhois.py +++ b/ipwhois/ipwhois.py @@ -42,17 +42,12 @@ class IPWhois: seconds. Defaults to 5. proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for proxy support. Defaults to None. - allow_permutations (:obj:`bool`): Allow net.Net() to use additional - methods if DNS lookups to Cymru fail. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to False. """ - def __init__(self, address, timeout=5, proxy_opener=None, - allow_permutations=False): + def __init__(self, address, timeout=5, proxy_opener=None): self.net = Net( - address=address, timeout=timeout, proxy_opener=proxy_opener, - allow_permutations=allow_permutations + address=address, timeout=timeout, proxy_opener=proxy_opener ) self.ipasn = IPASN(self.net) @@ -71,7 +66,7 @@ def __repr__(self): def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, extra_blacklist=None, ignore_referral_errors=False, - field_list=None, asn_alts=None, extra_org_map=None, + field_list=None, extra_org_map=None, inc_nir=True, nir_field_list=None, asn_methods=None, get_asn_description=True): """ @@ -95,10 +90,6 @@ def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, ['name', 'handle', 'description', 'country', 'state', 'city', 'address', 'postal_code', 'emails', 'created', 'updated'] If None, defaults to all. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - If None, defaults to all ['whois', 'http']. *WARNING* - deprecated in favor of new argument asn_methods. extra_org_map (:obj:`dict`): Dictionary mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC @@ -161,7 +152,7 @@ def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, log.debug('ASN lookup for {0}'.format(self.address_str)) asn_data = self.ipasn.lookup( - inc_raw=inc_raw, retry_count=retry_count, asn_alts=asn_alts, + inc_raw=inc_raw, retry_count=retry_count, extra_org_map=extra_org_map, asn_methods=asn_methods, get_asn_description=get_asn_description ) @@ -206,7 +197,7 @@ def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, excluded_entities=None, bootstrap=False, - rate_limit_timeout=120, asn_alts=None, extra_org_map=None, + rate_limit_timeout=120, extra_org_map=None, inc_nir=True, nir_field_list=None, asn_methods=None, get_asn_description=True): """ @@ -233,10 +224,6 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, rate_limit_timeout (:obj:`int`): The number of seconds to wait before retrying when a rate limit notice is returned via rdap+json. Defaults to 120. - asn_alts (:obj:`list`): Additional lookup types to attempt if the - ASN dns lookup fails. Allow permutations must be enabled. - If None, defaults to all ['whois', 'http']. *WARNING* - deprecated in favor of new argument asn_methods. extra_org_map (:obj:`dict`): Dictionary mapping org handles to RIRs. This is for limited cases where ARIN REST (ASN fallback HTTP lookup) does not show an RIR as the org handle e.g., DNIC @@ -303,7 +290,7 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, # Retrieve the ASN information. log.debug('ASN lookup for {0}'.format(self.address_str)) asn_data = self.ipasn.lookup( - inc_raw=inc_raw, retry_count=retry_count, asn_alts=asn_alts, + inc_raw=inc_raw, retry_count=retry_count, extra_org_map=extra_org_map, asn_methods=asn_methods, get_asn_description=get_asn_description ) diff --git a/ipwhois/net.py b/ipwhois/net.py index 87939213..ac23c350 100644 --- a/ipwhois/net.py +++ b/ipwhois/net.py @@ -103,17 +103,13 @@ class Net: seconds. Defaults to 5. proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for proxy support. Defaults to None. - allow_permutations (:obj:`bool`): Allow net.Net() to use additional - methods if DNS lookups to Cymru fail. *WARNING* deprecated in - favor of new argument asn_methods. Defaults to False. Raises: IPDefinedError: The address provided is defined (does not need to be resolved). """ - def __init__(self, address, timeout=5, proxy_opener=None, - allow_permutations=False): + def __init__(self, address, timeout=5, proxy_opener=None): # IPv4Address or IPv6Address if isinstance(address, IPv4Address) or isinstance( @@ -129,16 +125,6 @@ def __init__(self, address, timeout=5, proxy_opener=None, # Default timeout for socket connections. self.timeout = timeout - # Allow other than DNS lookups for ASNs. - self.allow_permutations = allow_permutations - - if self.allow_permutations: - - from warnings import warn - warn('allow_permutations has been deprecated and will be removed. ' - 'It is no longer needed, due to the deprecation of asn_alts, ' - 'and the addition of the asn_methods argument.') - self.dns_resolver = dns.resolver.Resolver() self.dns_resolver.timeout = timeout self.dns_resolver.lifetime = timeout @@ -219,21 +205,6 @@ def __init__(self, address, timeout=5, proxy_opener=None, self.dns_zone = IPV6_DNS_ZONE.format(self.reversed) - def lookup_asn(self, *args, **kwargs): - """ - Temporary wrapper for IP ASN lookups (moved to - asn.IPASN.lookup()). This will be removed in a future - release. - """ - - from warnings import warn - warn('Net.lookup_asn() has been deprecated and will be removed. ' - 'You should now use asn.IPASN.lookup() for IP ASN lookups.') - from .asn import IPASN - response = None - ipasn = IPASN(self) - return ipasn.lookup(*args, **kwargs), response - def get_asn_dns(self): """ The function for retrieving ASN information for an IP address from diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 910b4f8c..2c470acd 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -286,16 +286,6 @@ def parse_fields(self, response, fields_dict, net_start=None, return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._parse_fields() has been deprecated and will be ' - 'removed. You should now use NIRWhois.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_jpnic(self, response): """ The function for parsing network blocks from jpnic whois data. @@ -359,16 +349,6 @@ def get_nets_jpnic(self, response): return nets - def _get_nets_jpnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_nets_jpnic() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_nets_jpnic().') - return self.get_nets_jpnic(*args, **kwargs) - def get_nets_krnic(self, response): """ The function for parsing network blocks from krnic whois data. @@ -434,16 +414,6 @@ def get_nets_krnic(self, response): return nets - def _get_nets_krnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_nets_krnic() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_nets_krnic().') - return self.get_nets_krnic(*args, **kwargs) - def get_contact(self, response=None, nir=None, handle=None, retry_count=3, dt_format=None): """ @@ -491,16 +461,6 @@ def get_contact(self, response=None, nir=None, handle=None, is_contact=True ) - def _get_contact(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('NIRWhois._get_contact() has been deprecated and will be ' - 'removed. You should now use NIRWhois.get_contact().') - return self.get_contact(*args, **kwargs) - def lookup(self, nir=None, inc_raw=False, retry_count=3, response=None, field_list=None, is_offline=False): """ diff --git a/ipwhois/scripts/ipwhois_cli.py b/ipwhois/scripts/ipwhois_cli.py index 5a06e9ab..74a08f93 100644 --- a/ipwhois/scripts/ipwhois_cli.py +++ b/ipwhois/scripts/ipwhois_cli.py @@ -166,17 +166,6 @@ help='The number of times to retry in case socket errors, timeouts, ' 'connection resets, etc. are encountered.' ) -group.add_argument( - '--asn_alts', - type=str, - nargs=1, - default='whois,http', - metavar='"ASN_ALTS"', - help='A comma delimited list of additional lookup types to attempt if the ' - 'ASN dns lookup fails. Allow permutations must be enabled. ' - 'Defaults to all: "whois,http" *WARNING* deprecated in ' - 'favor of new argument asn_methods.' -) group.add_argument( '--asn_methods', type=str, @@ -1456,9 +1445,6 @@ def lookup_whois(self, hr=True, show_name=False, colorize=True, **kwargs): field_list=script_args.field_list.split(',') if ( script_args.field_list and len(script_args.field_list) > 0) else None, - asn_alts=script_args.asn_alts.split(',') if ( - script_args.asn_alts and not script_args.asn_methods and - len(script_args.asn_alts) > 0) else None, extra_org_map=script_args.extra_org_map, inc_nir=(not script_args.exclude_nir), nir_field_list=script_args.nir_field_list.split(',') if ( @@ -1484,9 +1470,6 @@ def lookup_whois(self, hr=True, show_name=False, colorize=True, **kwargs): len(script_args.excluded_entities) > 0) else None, bootstrap=script_args.bootstrap, rate_limit_timeout=script_args.rate_limit_timeout, - asn_alts=script_args.asn_alts.split(',') if ( - script_args.asn_alts and not script_args.asn_methods and - len(script_args.asn_alts) > 0) else None, extra_org_map=script_args.extra_org_map, inc_nir=(not script_args.exclude_nir), nir_field_list=script_args.nir_field_list.split(',') if ( diff --git a/ipwhois/tests/online/test_ipwhois.py b/ipwhois/tests/online/test_ipwhois.py index d9f24a79..3627e9d7 100644 --- a/ipwhois/tests/online/test_ipwhois.py +++ b/ipwhois/tests/online/test_ipwhois.py @@ -174,8 +174,3 @@ def test_lookup_rdap(self): result = IPWhois(address='74.125.225.229', timeout=0, proxy_opener=opener) self.assertRaises(ASNRegistryError, result.lookup_rdap) - - log.debug('Testing allow_permutations') - result = IPWhois(address='74.125.225.229', timeout=0, - allow_permutations=False) - self.assertRaises(ASNRegistryError, result.lookup_rdap) diff --git a/ipwhois/whois.py b/ipwhois/whois.py index 882dd908..abc3361f 100644 --- a/ipwhois/whois.py +++ b/ipwhois/whois.py @@ -324,16 +324,6 @@ def parse_fields(self, response, fields_dict, net_start=None, return ret - def _parse_fields(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._parse_fields() has been deprecated and will be ' - 'removed. You should now use Whois.parse_fields().') - return self.parse_fields(*args, **kwargs) - def get_nets_arin(self, response): """ The function for parsing network blocks from ARIN whois data. @@ -415,16 +405,6 @@ def get_nets_arin(self, response): return nets - def _get_nets_arin(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_arin() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_arin().') - return self.get_nets_arin(*args, **kwargs) - def get_nets_lacnic(self, response): """ The function for parsing network blocks from LACNIC whois data. @@ -495,16 +475,6 @@ def get_nets_lacnic(self, response): return nets - def _get_nets_lacnic(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_lacnic() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_lacnic().') - return self.get_nets_lacnic(*args, **kwargs) - def get_nets_other(self, response): """ The function for parsing network blocks from generic whois data. @@ -577,16 +547,6 @@ def get_nets_other(self, response): return nets - def _get_nets_other(self, *args, **kwargs): - """ - Deprecated. This will be removed in a future release. - """ - - from warnings import warn - warn('Whois._get_nets_other() has been deprecated and will be ' - 'removed. You should now use Whois.get_nets_other().') - return self.get_nets_other(*args, **kwargs) - def lookup(self, inc_raw=False, retry_count=3, response=None, get_referral=False, extra_blacklist=None, ignore_referral_errors=False, asn_data=None, From aa2de4a3e152faf132f30b2dea9556d5f37a981b Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Mon, 4 Feb 2019 13:55:38 +0000 Subject: [PATCH 04/39] 100% Coverage Added tests to make 100% code coverage again. Also updated coveragerc to show missing lines in the coverage report. --- .coveragerc | 1 + ipwhois/tests/test_asn.py | 26 ++++++++++++++++++++++++-- ipwhois/tests/test_nir.py | 5 +++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 5f6fa559..3aab48d0 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,5 @@ [report] +show_missing = True omit = */python?.?/* */site-packages/nose/* diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index 0485abb2..1981de29 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -135,8 +135,30 @@ def test__parse_fields_http(self): self.fail('Unexpected exception raised: {0}'.format(e)) def test_lookup(self): - # TODO: need to modify asn.json for this. - return NotImplemented + data_dir = path.dirname(__file__) + + with io.open(str(data_dir) + '/asn.json', 'r') as \ + data_file: + data = json.load(data_file) + + for key, val in data.items(): + + log.debug('Testing: {0}'.format(key)) + net = Net(key) + obj = IPASN(net) + + try: + + self.assertIsInstance(obj.lookup(), dict) + self.assertIsInstance(obj.lookup(asn_alts=['http']), dict) + + except AssertionError as e: + + raise e + + except Exception as e: + + self.fail('Unexpected exception raised: {0}'.format(e)) class TestASNOrigin(TestCommon): diff --git a/ipwhois/tests/test_nir.py b/ipwhois/tests/test_nir.py index a855e828..9220309e 100644 --- a/ipwhois/tests/test_nir.py +++ b/ipwhois/tests/test_nir.py @@ -45,6 +45,11 @@ def test_lookup(self): inc_raw=True), dict) + self.assertIsInstance(obj.lookup( + nir=val['nir'], + response=val['response']), + dict) + except AssertionError as e: raise e From e947df634b55ec99a9ce141354327749724292f3 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Tue, 5 Feb 2019 11:14:46 +0000 Subject: [PATCH 05/39] Changes from secynic Changes from secynic. --- ipwhois/tests/test_asn.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index b21249f1..f83cce59 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -136,30 +136,7 @@ def test_parse_fields_http(self): self.fail('Unexpected exception raised: {0}'.format(e)) def test_lookup(self): - data_dir = path.dirname(__file__) - - with io.open(str(data_dir) + '/asn.json', 'r') as \ - data_file: - data = json.load(data_file) - - for key, val in data.items(): - - log.debug('Testing: {0}'.format(key)) - net = Net(key) - obj = IPASN(net) - - try: - - self.assertIsInstance(obj.lookup(), dict) - self.assertIsInstance(obj.lookup(asn_alts=['http']), dict) - - except AssertionError as e: - - raise e - - except Exception as e: - - self.fail('Unexpected exception raised: {0}'.format(e)) + raise NotImplemented class TestASNOrigin(TestCommon): From e20c06e1d3ac9fc6ca72aaad51718f0c709be9f8 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Mon, 11 Feb 2019 11:57:01 +0000 Subject: [PATCH 06/39] Code Review Changes Changes to test as suggest by the maintainer --- ipwhois/tests/test_asn.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index f83cce59..9f9425ab 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -136,7 +136,29 @@ def test_parse_fields_http(self): self.fail('Unexpected exception raised: {0}'.format(e)) def test_lookup(self): - raise NotImplemented + data_dir = path.dirname(__file__) + + with io.open(str(data_dir) + '/asn.json', 'r') as \ + data_file: + data = json.load(data_file) + + for key, val in data.items(): + + log.debug('Testing: {0}'.format(key)) + net = Net(key) + obj = IPASN(net) + + try: + + self.assertIsInstance(obj.lookup(), dict) + + except AssertionError as e: + + raise e + + except Exception as e: + + self.fail('Unexpected exception raised: {0}'.format(e)) class TestASNOrigin(TestCommon): From b27953ca79a03c17d5f73a89ad331cedebebaa37 Mon Sep 17 00:00:00 2001 From: Stanislav Levin Date: Sat, 16 Feb 2019 20:17:56 +0300 Subject: [PATCH 07/39] Make use of builtin sphinx napoleon extension Since 1.3b1 (released Oct 10, 2014) Sphinx has support for NumPy and Google style docstring support via sphinx.ext.napoleon extension. The sphinxcontrib.napoleon extension will continue to work with Sphinx <= 1.2. Signed-off-by: Stanislav Levin --- ipwhois/docs/requirements.txt | 1 - ipwhois/docs/source/conf.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ipwhois/docs/requirements.txt b/ipwhois/docs/requirements.txt index 8aba4d07..b7845bae 100644 --- a/ipwhois/docs/requirements.txt +++ b/ipwhois/docs/requirements.txt @@ -1,4 +1,3 @@ sphinx -sphinxcontrib-napoleon sphinx_rtd_theme dnspython diff --git a/ipwhois/docs/source/conf.py b/ipwhois/docs/source/conf.py index 164be02c..108ae4d9 100644 --- a/ipwhois/docs/source/conf.py +++ b/ipwhois/docs/source/conf.py @@ -40,7 +40,7 @@ 'sphinx.ext.doctest', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', - 'sphinxcontrib.napoleon' + 'sphinx.ext.napoleon' ] napoleon_google_docstring = True From 3b11faa8da71b22ba7e78e1291d1b818992e0dbf Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Fri, 17 May 2019 14:05:56 +0100 Subject: [PATCH 08/39] Updated gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 690d6bea..8b4d2934 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,7 @@ nosetests.xml .pydevproject MANIFEST -.idea \ No newline at end of file +.idea +.history +.vscode +.venv \ No newline at end of file From 153c402be48a0c9119ac2cfe39128a0fb05b4fb2 Mon Sep 17 00:00:00 2001 From: Haseeb Majid Date: Tue, 30 Jul 2019 10:18:18 +0100 Subject: [PATCH 09/39] Fixed changes Fixed changes from code review. --- ipwhois/tests/test_asn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipwhois/tests/test_asn.py b/ipwhois/tests/test_asn.py index 9f9425ab..bb730512 100644 --- a/ipwhois/tests/test_asn.py +++ b/ipwhois/tests/test_asn.py @@ -145,8 +145,8 @@ def test_lookup(self): for key, val in data.items(): log.debug('Testing: {0}'.format(key)) - net = Net(key) - obj = IPASN(net) + net = Net(key) + obj = IPASN(net) try: From 566dc3e280a57ac6db1259dd027352223ca1b036 Mon Sep 17 00:00:00 2001 From: secynic Date: Wed, 31 Jul 2019 13:29:11 -0500 Subject: [PATCH 10/39] Fixed ASNOrigin lookups (#216) --- CHANGES.rst | 2 ++ ipwhois/asn.py | 31 +++++++++++++++++++------------ ipwhois/tests/online/test_asn.py | 3 ++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 024080b5..663cb009 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,8 @@ Changelog nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact (#230) - Removed deprecated asn_alts parameter (#230) - Removed deprecated allow_permutations parameter (#230) +- Fixed ASNOrigin lookups (#216) +- Fixed bug in ASNOrigin lookups when multiple asn_methods provided (#216) 1.1.0 (2019-02-01) ------------------ diff --git a/ipwhois/asn.py b/ipwhois/asn.py index aaa4c336..60625e2b 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -61,21 +61,21 @@ ASN_ORIGIN_HTTP = { 'radb': { - 'url': 'http://www.radb.net/query/', + 'url': 'http://www.radb.net/query', 'form_data_asn_field': 'keywords', 'form_data': { 'advanced_query': '1', 'query': 'Query', - '-T option': 'inet-rtr', + # '-T option': 'inet-rtr', 'ip_option': '', '-i': '1', '-i option': 'origin' }, 'fields': { - 'description': r'(descr):[^\S\n]+(?P.+?)\', - 'maintainer': r'(mnt-by):[^\S\n]+(?P.+?)\', - 'updated': r'(changed):[^\S\n]+(?P.+?)\', - 'source': r'(source):[^\S\n]+(?P.+?)\', + 'description': r'(descr):[^\S\n]+(?P.+?)\n', + 'maintainer': r'(mnt-by):[^\S\n]+(?P.+?)\n', + 'updated': r'(changed):[^\S\n]+(?P.+?)\n', + 'source': r'(source):[^\S\n]+(?P.+?)\<', } }, } @@ -796,6 +796,8 @@ def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, asn=asn, retry_count=retry_count ) + break + except (WhoisLookupError, WhoisRateLimitError) as e: log.debug('ASN origin WHOIS lookup failed: {0}' @@ -809,17 +811,22 @@ def lookup(self, asn=None, inc_raw=False, retry_count=3, response=None, log.debug('Response not given, perform ASN origin ' 'HTTP lookup for: {0}'.format(asn)) - tmp = ASN_ORIGIN_HTTP['radb']['form_data'] - tmp[str(ASN_ORIGIN_HTTP['radb']['form_data_asn_field'] - )] = asn + # tmp = ASN_ORIGIN_HTTP['radb']['form_data'] + # tmp[str( + # ASN_ORIGIN_HTTP['radb']['form_data_asn_field'] + # )] = asn response = self._net.get_http_raw( - url=ASN_ORIGIN_HTTP['radb']['url'], + url=('{0}?advanced_query=1&keywords={1}&-T+option' + '=&ip_option=&-i=1&-i+option=origin' + ).format(ASN_ORIGIN_HTTP['radb']['url'], asn), retry_count=retry_count, - request_type='POST', - form_data=tmp + request_type='GET', + # form_data=tmp ) is_http = True # pragma: no cover + break + except HTTPLookupError as e: log.debug('ASN origin HTTP lookup failed: {0}' diff --git a/ipwhois/tests/online/test_asn.py b/ipwhois/tests/online/test_asn.py index 62cd8049..76bc8ce0 100644 --- a/ipwhois/tests/online/test_asn.py +++ b/ipwhois/tests/online/test_asn.py @@ -96,4 +96,5 @@ def test_lookup(self): net = Net(address='74.125.225.229') asnorigin = ASNOrigin(net) - asnorigin.lookup(asn='15169', asn_methods=['whois', 'http']) + asnorigin.lookup(asn='15169', asn_methods=['whois']) + asnorigin.lookup(asn='15169', asn_methods=['http']) From ba1bebdf687246ba5abb1d28562457efa024d1b3 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 6 Aug 2019 18:27:56 -0500 Subject: [PATCH 11/39] Fixed bug in KRNIC queries due to a change in their service (#243) --- CHANGES.rst | 1 + ipwhois/nir.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 663cb009..f0998b75 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -15,6 +15,7 @@ Changelog - Removed deprecated allow_permutations parameter (#230) - Fixed ASNOrigin lookups (#216) - Fixed bug in ASNOrigin lookups when multiple asn_methods provided (#216) +- Fixed bug in KRNIC queries due to a change in their service (#243) 1.1.0 (2019-02-01) ------------------ diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 2c470acd..ea539874 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -108,9 +108,14 @@ }, 'krnic': { 'country_code': 'KR', - 'url': 'https://whois.kisa.or.kr/eng/whois.jsc', + 'url': 'https://xn--c79as89aj0e29b77z.xn--3e0b707e/eng/whois.jsc', 'request_type': 'POST', - 'request_headers': {'Accept': 'text/html'}, + 'request_headers': { + 'Accept': 'text/html', + 'Referer': ( + 'https://xn--c79as89aj0e29b77z.xn--3e0b707e/eng/whois.jsp' + ), + }, 'form_data_ip_field': 'query', 'fields': { 'name': r'(Organization Name)[\s]+\:[^\S\n]+(?P.+?)\n', From 05708cda9b235ca374f59cdbb3730bd8f8cb61c2 Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Mon, 10 Aug 2020 15:29:57 +0100 Subject: [PATCH 12/39] Fix issue where bulk_lookup_rdap() returns only one item result Add rdap lookups to the returned dictionary results --- ipwhois/experimental.py | 10 +++++----- ipwhois/rdap.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index a048bdf9..85fff9b1 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -253,15 +253,15 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, try: - results = ipasn.parse_fields_whois(asn_result) + asn_parsed = ipasn.parse_fields_whois(asn_result) except ASNRegistryError: # pragma: no cover continue # Add valid IP ASN result to asn_parsed_results for RDAP lookup - asn_parsed_results[ip] = results - stats[results['asn_registry']]['total'] += 1 + asn_parsed_results[ip] = asn_parsed + stats[asn_parsed['asn_registry']]['total'] += 1 # Set the list of IPs that are not allocated/failed ASN lookup stats['unallocated_addresses'] = list(k for k in addresses if k not in @@ -362,7 +362,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Perform the RDAP lookup. retry_count is set to 0 # here since we handle that in this function - results = rdap.lookup( + rdap_result = rdap.lookup( inc_raw=inc_raw, retry_count=0, asn_data=asn_data, depth=depth, excluded_entities=excluded_entities ) @@ -373,7 +373,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Lookup was successful, add to result. Set the nir # key to None as this is not supported # (yet - requires more queries) - results[ip] = results + results[ip] = rdap_result results[ip]['nir'] = None # Remove the IP from the lookup queue diff --git a/ipwhois/rdap.py b/ipwhois/rdap.py index 9bccbe1f..d019f6d2 100644 --- a/ipwhois/rdap.py +++ b/ipwhois/rdap.py @@ -553,7 +553,7 @@ def parse(self): self.vars[v] = self.json[v].strip() - except (KeyError, ValueError): + except (KeyError, ValueError, AttributeError): pass From b258ebd4c0073f2787e4defbb5fb3afce99aa58a Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Tue, 11 Aug 2020 09:51:58 +0100 Subject: [PATCH 13/39] Add test --- ipwhois/tests/online/test_experimental.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ipwhois/tests/online/test_experimental.py b/ipwhois/tests/online/test_experimental.py index 31ad57ec..fc5042ca 100644 --- a/ipwhois/tests/online/test_experimental.py +++ b/ipwhois/tests/online/test_experimental.py @@ -67,8 +67,22 @@ def test_bulk_lookup_rdap(self): '115.1.2.3' # KRNIC ] + expected_stats = {'ip_input_total': 12, 'ip_unique_total': 12, 'ip_lookup_total': 12, + 'lacnic': {'failed': [], 'rate_limited': [], 'total': 2}, + 'ripencc': {'failed': [], 'rate_limited': [], 'total': 2}, + 'apnic': {'failed': [], 'rate_limited': [], 'total': 4}, + 'afrinic': {'failed': [], 'rate_limited': [], 'total': 2}, + 'arin': {'failed': [], 'rate_limited': [], 'total': 2}, + 'unallocated_addresses': []} + try: - self.assertIsInstance(bulk_lookup_rdap(addresses=ips), tuple) + result = bulk_lookup_rdap(addresses=ips) + self.assertIsInstance(result, tuple) + + results, stats = result + self.assertEqual(stats, expected_stats) + self.assertEqual(len(results), 12) + except ASNLookupError: pass except AssertionError as e: From a3253af24a3d67cafd2a0d90dfa01c239dee4142 Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Tue, 11 Aug 2020 23:28:09 +0100 Subject: [PATCH 14/39] Add asn data to rdap results --- ipwhois/experimental.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 85fff9b1..1bd989d2 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -373,7 +373,8 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Lookup was successful, add to result. Set the nir # key to None as this is not supported # (yet - requires more queries) - results[ip] = rdap_result + results[ip]['asn'] = asn_data + results[ip]['rdap'] = rdap_result results[ip]['nir'] = None # Remove the IP from the lookup queue From fa9330ba4a645c880761becdefc983be6e364f9d Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Tue, 11 Aug 2020 23:58:50 +0100 Subject: [PATCH 15/39] Add asn data at the parent level --- ipwhois/experimental.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 1bd989d2..33ee5465 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -373,7 +373,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # Lookup was successful, add to result. Set the nir # key to None as this is not supported # (yet - requires more queries) - results[ip]['asn'] = asn_data + results[ip] = asn_data results[ip]['rdap'] = rdap_result results[ip]['nir'] = None From fe7dd4fde02c170b22bbf08274677070b71d9659 Mon Sep 17 00:00:00 2001 From: Imed Mnif Date: Wed, 12 Aug 2020 09:22:39 +0100 Subject: [PATCH 16/39] merge asn and rdap data into the results --- ipwhois/experimental.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 33ee5465..9d597e9c 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -374,7 +374,8 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, # key to None as this is not supported # (yet - requires more queries) results[ip] = asn_data - results[ip]['rdap'] = rdap_result + results[ip].update(rdap_result) + results[ip]['nir'] = None # Remove the IP from the lookup queue From 40c69ccc3bda2dab73180eadb95e1266c37f66d6 Mon Sep 17 00:00:00 2001 From: Philip Hane Date: Wed, 2 Sep 2020 14:57:59 -0500 Subject: [PATCH 17/39] Update changes for #262 --- CHANGES.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f0998b75..e9c2319c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,8 @@ Changelog - Fixed ASNOrigin lookups (#216) - Fixed bug in ASNOrigin lookups when multiple asn_methods provided (#216) - Fixed bug in KRNIC queries due to a change in their service (#243) +- Fixed bug in experimental.bulk_lookup_rdap where only the last + result was returned (#262 - ameidatou) 1.1.0 (2019-02-01) ------------------ @@ -233,4 +235,4 @@ Changelog - Added support for IPv4Address or IPv6Address as the address arg in IPWhois. - Fixed file open encoding bug. Moved from open to io.open. - Fixed parameter in IPWhois ip defined checks. -- Fixed TestIPWhois.test_ip_invalid() assertions. \ No newline at end of file +- Fixed TestIPWhois.test_ip_invalid() assertions. From 29479698de478bf49039710ab4193cbdf33e7def Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Fri, 11 Sep 2020 14:08:22 +0000 Subject: [PATCH 18/39] Fix deprecation warnings due to invalid escape sequences. --- ipwhois/nir.py | 10 +++++----- ipwhois/utils.py | 36 ++++++++++++++++++------------------ ipwhois/whois.py | 18 +++++++++--------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/ipwhois/nir.py b/ipwhois/nir.py index ea539874..3d2f6035 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -87,9 +87,9 @@ 'updated': r'(\[Last Update\])[^\S\n]+(?P.*?)\n', 'nameservers': r'(\[Nameserver\])[^\S\n]+(?P.*?)\n', 'contact_admin': r'(\[Administrative Contact\])[^\S\n]+.+?\>' - '(?P.+?)\<\/A\>\n', + '(?P.+?)\\<\\/A\\>\n', 'contact_tech': r'(\[Technical Contact\])[^\S\n]+.+?\>' - '(?P.+?)\<\/A\>\n' + '(?P.+?)\\<\\/A\\>\n' }, 'contact_fields': { 'name': r'(\[Last, First\])[^\S\n]+(?P.*?)\n', @@ -125,9 +125,9 @@ 'postal_code': r'(Zip Code)[\s]+\:[^\S\n]+(?P.+?)\n', 'created': r'(Registration Date)[\s]+\:[^\S\n]+(?P.+?)\n', 'contact_admin': r'(id="eng_isp_contact").+?\>(?P.*?)\<' - '\/div\>\n', + '\\/div\\>\n', 'contact_tech': r'(id="eng_user_contact").+?\>(?P.*?)\<' - '\/div\>\n' + '\\/div\\>\n' }, 'contact_fields': { 'name': r'(Name)[^\S\n]+?:[^\S\n]+?(?P.*?)\n', @@ -379,7 +379,7 @@ def get_nets_krnic(self, response): # and the start and end positions. for match in re.finditer( r'^(IPv4 Address)[\s]+:[^\S\n]+((.+?)[^\S\n]-[^\S\n](.+?)' - '[^\S\n]\((.+?)\)|.+)$', + '[^\\S\n]\\((.+?)\\)|.+)$', response, re.MULTILINE ): diff --git a/ipwhois/utils.py b/ipwhois/utils.py index 1df4a917..93297fd1 100644 --- a/ipwhois/utils.py +++ b/ipwhois/utils.py @@ -87,29 +87,29 @@ IP_REGEX = ( r'(?P' # IPv4 - '(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.)){3}' + r'(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.)){3}' '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' # IPv6 - '|\[?(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:)' - '{6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' - '2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]' - '{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d' - '\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|' - '((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' - '2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]' - '{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)' - '(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}((' - '(:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1' - '\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|((' + r'|\[?(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:)' + r'{6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' + r'2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]' + r'{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d' + r'\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|' + r'((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' + r'2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]' + r'{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)' + r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}((' + r'(:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1' + r'\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|((' '[0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4})' - '{0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]' - '?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((' - '25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})' - ')|:)))(%.+)?))\]?' + r'{0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]' + r'?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((' + r'25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})' + r')|:)))(%.+)?))\]?' # Optional IPv4 Port - '((:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}' + r'((:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}' # Optional CIDR block - '))|(\/(?:[012]\d?|3[012]?|[4-9])))?' + r'))|(\/(?:[012]\d?|3[012]?|[4-9])))?' ')' ) diff --git a/ipwhois/whois.py b/ipwhois/whois.py index abc3361f..99dc07c9 100644 --- a/ipwhois/whois.py +++ b/ipwhois/whois.py @@ -67,7 +67,7 @@ 'name': r'(NetName):[^\S\n]+(?P.+?)\n', 'handle': r'(NetHandle):[^\S\n]+(?P.+?)\n', 'description': r'(OrgName|CustName):[^\S\n]+(?P.+?)' - '(?=(\n\S):?)', + '(?=(\n\\S):?)', 'country': r'(Country):[^\S\n]+(?P.+?)\n', 'state': r'(StateProv):[^\S\n]+(?P.+?)\n', 'city': r'(City):[^\S\n]+(?P.+?)\n', @@ -75,7 +75,7 @@ 'postal_code': r'(PostalCode):[^\S\n]+(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(RegDate):[^\S\n]+(?P.+?)\n', 'updated': r'(Updated):[^\S\n]+(?P.+?)\n', @@ -92,7 +92,7 @@ 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': ( r'(created):[^\S\n]+(?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]' @@ -115,7 +115,7 @@ 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'updated': r'(changed):[^\S\n]+.*(?P[0-9]{8}).*?\n' }, @@ -129,7 +129,7 @@ 'country': r'(country):[^\S\n]+(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(created):[^\S\n]+(?P[0-9]{8}).*?\n', 'updated': r'(changed):[^\S\n]+(?P[0-9]{8}).*?\n' @@ -146,7 +146,7 @@ 'address': r'(address):[^\S\n]+(?P.+?)(?=(\n\S):?)', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), } } @@ -166,7 +166,7 @@ 'postal_code': r'(network:Postal-Code):(?P.+?)\n', 'emails': ( r'.+?:.*?[^\S\n]+(?P[\w\-\.]+?@[\w\-\.]+\.[\w\-]+)(' - '[^\S\n]+.*?)*?\n' + '[^\\S\n]+.*?)*?\n' ), 'created': r'(network:Created):(?P.+?)\n', 'updated': r'(network:Updated):(?P.+?)\n' @@ -454,7 +454,7 @@ def get_nets_lacnic(self, response): for addr in net_range.split(', '): count = addr.count('.') - if count is not 0 and count < 4: + if count != 0 and count < 4: addr_split = addr.strip().split('/') for i in range(count + 1, 4): @@ -627,7 +627,7 @@ def lookup(self, inc_raw=False, retry_count=3, response=None, # Only fetch the response if we haven't already. if response is None or (not is_offline and - asn_data['asn_registry'] is not 'arin'): + asn_data['asn_registry'] != 'arin'): log.debug('Response not given, perform WHOIS lookup for {0}' .format(self._net.address_str)) From 0516872ab79356774d5729dec8b3d61ee287717e Mon Sep 17 00:00:00 2001 From: Philip Hane Date: Mon, 14 Sep 2020 11:45:23 -0500 Subject: [PATCH 19/39] Fixed deprecation warnings due to invalid escape sequences --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e9c2319c..1c2a7e23 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,6 +18,8 @@ Changelog - Fixed bug in KRNIC queries due to a change in their service (#243) - Fixed bug in experimental.bulk_lookup_rdap where only the last result was returned (#262 - ameidatou) +- Fixed deprecation warnings due to invalid escape sequences + (#272 - tirkarthi) 1.1.0 (2019-02-01) ------------------ From 00879c931286ce87f3225a426437fc5f8e9b0554 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 12:30:10 -0500 Subject: [PATCH 20/39] Added support for Python 3.8 (#267) --- .travis.yml | 1 + CHANGES.rst | 1 + UPGRADING.rst | 1 + setup.py | 1 + 4 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index a363b5ec..4f08fb5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - 3.5 - 3.6 - 3.7 + - 3.8 install: - pip install --upgrade setuptools - pip install --upgrade pip diff --git a/CHANGES.rst b/CHANGES.rst index 1c2a7e23..7d17b5c2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,7 @@ Changelog result was returned (#262 - ameidatou) - Fixed deprecation warnings due to invalid escape sequences (#272 - tirkarthi) +- Added support for Python 3.8 (#267) 1.1.0 (2019-02-01) ------------------ diff --git a/UPGRADING.rst b/UPGRADING.rst index d26f5f20..181bb878 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -22,6 +22,7 @@ v1.2.0 nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact - Removed deprecated asn_alts parameter - Removed deprecated allow_permutations parameter +- Added support for Python 3.8 ****** v1.1.0 diff --git a/setup.py b/setup.py index 9103dec1..949dee52 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Internet', 'Topic :: Software Development', ] From 67b9c89a12a9ae9f082c2bf7261cbe5db566a95d Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 12:55:25 -0500 Subject: [PATCH 21/39] 38 regex fix attempt (#267) --- ipwhois/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipwhois/utils.py b/ipwhois/utils.py index 93297fd1..aad51b8e 100644 --- a/ipwhois/utils.py +++ b/ipwhois/utils.py @@ -88,7 +88,7 @@ r'(?P' # IPv4 r'(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.)){3}' - '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' + r'(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' # IPv6 r'|\[?(((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:)' r'{6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|' @@ -110,7 +110,7 @@ r'((:(6553[0-5]|655[0-2]\d|65[0-4]\d{2}|6[0-4]\d{3}|[1-5]\d{4}|[1-9]\d{0,3}' # Optional CIDR block r'))|(\/(?:[012]\d?|3[012]?|[4-9])))?' - ')' + r')' ) From 5bdc0dcb65e1d9b39e478a6594be4f7d5ff58ba3 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 13:18:07 -0500 Subject: [PATCH 22/39] 38 fix tests (#267) --- ipwhois/tests/test_utils.py | 110 ++++++++++++++++++++++++++---------- ipwhois/utils.py | 6 +- 2 files changed, 86 insertions(+), 30 deletions(-) diff --git a/ipwhois/tests/test_utils.py b/ipwhois/tests/test_utils.py index 7a26950c..44b3fc37 100644 --- a/ipwhois/tests/test_utils.py +++ b/ipwhois/tests/test_utils.py @@ -64,35 +64,35 @@ def test_ipv4_is_defined(self): self.assertRaises(ValueError, ipv4_is_defined, '192.168.0.256') self.assertRaises(AddressValueError, ipv4_is_defined, 1234) - self.assertEquals(ipv4_is_defined('74.125.225.229'), (False, '', '')) + self.assertEqual(ipv4_is_defined('74.125.225.229'), (False, '', '')) - self.assertEquals(ipv4_is_defined('0.0.0.0'), + self.assertEqual(ipv4_is_defined('0.0.0.0'), (True, 'This Network', 'RFC 1122, Section 3.2.1.3')) - self.assertEquals(ipv4_is_defined('127.0.0.0'), + self.assertEqual(ipv4_is_defined('127.0.0.0'), (True, 'Loopback', 'RFC 1122, Section 3.2.1.3')) - self.assertEquals(ipv4_is_defined('169.254.0.0'), + self.assertEqual(ipv4_is_defined('169.254.0.0'), (True, 'Link Local', 'RFC 3927')) - self.assertEquals(ipv4_is_defined('192.0.0.0'), + self.assertEqual(ipv4_is_defined('192.0.0.0'), (True, 'IETF Protocol Assignments', 'RFC 5736')) - self.assertEquals(ipv4_is_defined('192.0.2.0'), + self.assertEqual(ipv4_is_defined('192.0.2.0'), (True, 'TEST-NET-1', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('192.88.99.0'), + self.assertEqual(ipv4_is_defined('192.88.99.0'), (True, '6to4 Relay Anycast', 'RFC 3068')) - self.assertEquals(ipv4_is_defined('198.18.0.0'), + self.assertEqual(ipv4_is_defined('198.18.0.0'), (True, 'Network Interconnect Device Benchmark Testing', 'RFC 2544')) - self.assertEquals(ipv4_is_defined('198.51.100.0'), + self.assertEqual(ipv4_is_defined('198.51.100.0'), (True, 'TEST-NET-2', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('203.0.113.0'), + self.assertEqual(ipv4_is_defined('203.0.113.0'), (True, 'TEST-NET-3', 'RFC 5737')) - self.assertEquals(ipv4_is_defined('224.0.0.0'), + self.assertEqual(ipv4_is_defined('224.0.0.0'), (True, 'Multicast', 'RFC 3171')) - self.assertEquals(ipv4_is_defined('255.255.255.255'), + self.assertEqual(ipv4_is_defined('255.255.255.255'), (True, 'Limited Broadcast', 'RFC 919, Section 7')) - self.assertEquals(ipv4_is_defined('192.168.0.1'), + self.assertEqual(ipv4_is_defined('192.168.0.1'), (True, 'Private-Use Networks', 'RFC 1918')) - self.assertEquals(ipv4_is_defined('198.97.38.0'), + self.assertEqual(ipv4_is_defined('198.97.38.0'), (True, 'IANA Reserved', '')) def test_ipv6_is_defined(self): @@ -105,31 +105,31 @@ def test_ipv6_is_defined(self): '2001:4860:4860::8888::1234') self.assertRaises(AddressValueError, ipv6_is_defined, 1234) - self.assertEquals(ipv6_is_defined('2001:4860:4860::8888'), + self.assertEqual(ipv6_is_defined('2001:4860:4860::8888'), (False, '', '')) - self.assertEquals(ipv6_is_defined('ff00::'), + self.assertEqual(ipv6_is_defined('ff00::'), (True, 'Multicast', 'RFC 4291, Section 2.7')) - self.assertEquals(ipv6_is_defined('0:0:0:0:0:0:0:0'), + self.assertEqual(ipv6_is_defined('0:0:0:0:0:0:0:0'), (True, 'Unspecified', 'RFC 4291, Section 2.5.2')) - self.assertEquals(ipv6_is_defined('0:0:0:0:0:0:0:1'), + self.assertEqual(ipv6_is_defined('0:0:0:0:0:0:0:1'), (True, 'Loopback', 'RFC 4291, Section 2.5.3')) - self.assertEquals(ipv6_is_defined('100::'), + self.assertEqual(ipv6_is_defined('100::'), (True, 'Reserved', 'RFC 4291')) - self.assertEquals(ipv6_is_defined('fe80::'), + self.assertEqual(ipv6_is_defined('fe80::'), (True, 'Link-Local', 'RFC 4291, Section 2.5.6')) - self.assertEquals(ipv6_is_defined('fec0::'), + self.assertEqual(ipv6_is_defined('fec0::'), (True, 'Site-Local', 'RFC 4291, Section 2.5.7')) - self.assertEquals(ipv6_is_defined('fc00::'), + self.assertEqual(ipv6_is_defined('fc00::'), (True, 'Unique Local Unicast', 'RFC 4193')) def test_unique_everseen(self): input_list = ['b', 'a', 'c', 'a', 'b', 'x', 'a'] - self.assertEquals(list(unique_everseen(input_list)), + self.assertEqual(list(unique_everseen(input_list)), ['b', 'a', 'c', 'x']) - self.assertEquals(list(unique_everseen(input_list, str.lower)), + self.assertEqual(list(unique_everseen(input_list, str.lower)), ['b', 'a', 'c', 'x']) def test_unique_addresses(self): @@ -150,14 +150,14 @@ def test_unique_addresses(self): '2001:4860:4860::8888': {'count': 2, 'ports': {'443': 1}} } - self.assertEquals(unique_addresses(input_data), expected_result) + self.assertEqual(unique_addresses(input_data), expected_result) data_dir = path.dirname(__file__) fp = str(data_dir) + '/rdap.json' # Expected result is different on 2.x vs 3.x, possible issues with # ipaddr vs ipaddress output. Investigation pending... - if sys.version_info >= (3, 3): + if (3, 3) <= sys.version_info < (3, 8): fp_expected_result = { '74.125.225.0/24': {'count': 1, 'ports': {}}, @@ -203,7 +203,59 @@ def test_unique_addresses(self): '210.0.0.0/8': {'count': 1, 'ports': {}} } - self.assertEquals(unique_addresses(file_path=fp), + self.assertEqual(unique_addresses(file_path=fp), + fp_expected_result) + + if sys.version_info >= (3, 8): + + fp_expected_result = { + '196.0.0.0': {'count': 1, 'ports': {}}, + '196.11.239.0': {'count': 2, 'ports': {}}, + '196.11.240.0/23': {'count': 1, 'ports': {}}, + '196.11.240.215': {'count': 2, 'ports': {}}, + '196.11.246.255': {'count': 2, 'ports': {}}, + '196.255.255.255': {'count': 1, 'ports': {}}, + '200.57.128.0/20': {'count': 1, 'ports': {}}, + '200.57.141.161': {'count': 7, 'ports': {}}, + '2001:200::/23': {'count': 2, 'ports': {}}, + '2001:240:10c:1::ca20:9d1d': + {'count': 2, 'ports': {}}, + '2001:240::': {'count': 1, 'ports': {}}, + '2001:240::/32': {'count': 6, 'ports': {}}, + '2001:240:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '2001:4200::/23': {'count': 1, 'ports': {}}, + '2001:43f8:7b0::': {'count': 3, 'ports': {}}, + '2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '2001:4860:4860::8888': {'count': 10, 'ports': {}}, + '2001:4860::': {'count': 2, 'ports': {}}, + '2001:4860::/32': {'count': 1, 'ports': {}}, + '2001:4860:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '210.0.0.0': {'count': 1, 'ports': {}}, + '210.0.0.0/8': {'count': 1, 'ports': {}}, + '210.107.0.0': {'count': 2, 'ports': {}}, + '210.107.0.0/17': {'count': 6, 'ports': {}}, + '210.107.127.255': {'count': 2, 'ports': {}}, + '210.107.73.73': {'count': 2, 'ports': {}}, + '210.255.255.255': {'count': 1, 'ports': {}}, + '2801:10:c000::': {'count': 7, 'ports': {}}, + '2a00:2380::/25': {'count': 1, 'ports': {}}, + '2a00:2381:ffff::1': {'count': 4, 'ports': {}}, + '62.239.0.0/16': {'count': 1, 'ports': {}}, + '62.239.237.0': {'count': 1, 'ports': {}}, + '62.239.237.0/32': {'count': 1, 'ports': {}}, + '62.239.237.1': {'count': 4, 'ports': {}}, + '62.239.237.255': {'count': 1, 'ports': {}}, + '62.239.237.255/32': {'count': 1, 'ports': {}}, + '74.125.0.0': {'count': 2, 'ports': {}}, + '74.125.225.0/24': {'count': 1, 'ports': {}}, + '74.125.225.229': {'count': 8, 'ports': {}}, + '74.125.255.255': {'count': 1, 'ports': {}} + } + + self.assertEqual(unique_addresses(file_path=fp), fp_expected_result) else: @@ -261,8 +313,8 @@ def test_unique_addresses(self): def test_ipv4_generate_random(self): - self.assertEquals(len(list(ipv4_generate_random(1000))), 1000) + self.assertEqual(len(list(ipv4_generate_random(1000))), 1000) def test_ipv6_generate_random(self): - self.assertEquals(len(list(ipv6_generate_random(1000))), 1000) + self.assertEqual(len(list(ipv6_generate_random(1000))), 1000) diff --git a/ipwhois/utils.py b/ipwhois/utils.py index aad51b8e..920ed0f8 100644 --- a/ipwhois/utils.py +++ b/ipwhois/utils.py @@ -101,7 +101,7 @@ r'(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}((' r'(:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1' r'\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|((' - '[0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4})' + r'[0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4})' r'{0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]' r'?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((' r'25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})' @@ -212,6 +212,7 @@ def get_countries(is_legacy_xml=False): # Read the file. data = f.read() + f.close() # Check if there is data. if not data: # pragma: no cover @@ -258,6 +259,8 @@ def get_countries(is_legacy_xml=False): # Add to the countries dictionary. countries[code] = name + f.close() + return countries @@ -506,6 +509,7 @@ def unique_addresses(data=None, file_path=None): # Read the file. file_data = f.read() + f.close() pattern = re.compile( str(IP_REGEX), From e3d85e97c2d6cb5d84d2a3c1df667c1a161010c1 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 13:31:42 -0500 Subject: [PATCH 23/39] 38 fix tests 2 (#267) --- ipwhois/tests/test_utils.py | 53 ++----------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/ipwhois/tests/test_utils.py b/ipwhois/tests/test_utils.py index 44b3fc37..fdbd4d1b 100644 --- a/ipwhois/tests/test_utils.py +++ b/ipwhois/tests/test_utils.py @@ -157,56 +157,7 @@ def test_unique_addresses(self): # Expected result is different on 2.x vs 3.x, possible issues with # ipaddr vs ipaddress output. Investigation pending... - if (3, 3) <= sys.version_info < (3, 8): - - fp_expected_result = { - '74.125.225.0/24': {'count': 1, 'ports': {}}, - '62.239.0.0/16': {'count': 1, 'ports': {}}, - '2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff': - {'count': 1, 'ports': {}}, - '210.0.0.0': {'count': 1, 'ports': {}}, - '196.11.240.0/23': {'count': 1, 'ports': {}}, - '2001:240:10c:1::ca20:9d1d': {'count': 2, 'ports': {}}, - '196.11.240.215': {'count': 2, 'ports': {}}, - '62.239.237.0/32': {'count': 1, 'ports': {}}, - '210.107.0.0/17': {'count': 6, 'ports': {}}, - '2001:4860::/32': {'count': 1, 'ports': {}}, - '210.107.73.73': {'count': 2, 'ports': {}}, - '210.107.0.0': {'count': 2, 'ports': {}}, - '2001:200::/23': {'count': 2, 'ports': {}}, - '2001:240:ffff:ffff:ffff:ffff:ffff:ffff': - {'count': 1, 'ports': {}}, - '210.255.255.255': {'count': 1, 'ports': {}}, - '2001:43f8:7b0::': {'count': 3, 'ports': {}}, - '196.255.255.255': {'count': 1, 'ports': {}}, - '2001:240::/32': {'count': 6, 'ports': {}}, - '196.0.0.0': {'count': 1, 'ports': {}}, - '2001:240::': {'count': 1, 'ports': {}}, - '196.11.246.255': {'count': 2, 'ports': {}}, - '196.11.239.0': {'count': 2, 'ports': {}}, - '2001:4200::/23': {'count': 1, 'ports': {}}, - '2a00:2380::/25': {'count': 1, 'ports': {}}, - '200.57.128.0/20': {'count': 1, 'ports': {}}, - '62.239.237.255': {'count': 1, 'ports': {}}, - '2001:4860:4860::8888': {'count': 10, 'ports': {}}, - '2001:4860::': {'count': 2, 'ports': {}}, - '2001:4860:ffff:ffff:ffff:ffff:ffff:ffff': - {'count': 1, 'ports': {}}, - '74.125.225.229': {'count': 8, 'ports': {}}, - '210.107.127.255': {'count': 2, 'ports': {}}, - '200.57.141.161': {'count': 7, 'ports': {}}, - '62.239.237.255/32': {'count': 1, 'ports': {}}, - '2801:10:c000::': {'count': 7, 'ports': {}}, - '2a00:2381:ffff::1': {'count': 4, 'ports': {}}, - '62.239.237.0': {'count': 1, 'ports': {}}, - '62.239.237.1': {'count': 4, 'ports': {}}, - '210.0.0.0/8': {'count': 1, 'ports': {}} - } - - self.assertEqual(unique_addresses(file_path=fp), - fp_expected_result) - - if sys.version_info >= (3, 8): + if sys.version_info >= (3, 3): fp_expected_result = { '196.0.0.0': {'count': 1, 'ports': {}}, @@ -256,7 +207,7 @@ def test_unique_addresses(self): } self.assertEqual(unique_addresses(file_path=fp), - fp_expected_result) + fp_expected_result) else: From 17039b8c3b6f2c6b5fcc2e86a768f63f2f3b519e Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 13:44:34 -0500 Subject: [PATCH 24/39] 38 fix tests 3 (#267) --- ipwhois/tests/test_utils.py | 51 ++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/ipwhois/tests/test_utils.py b/ipwhois/tests/test_utils.py index fdbd4d1b..9dc041f4 100644 --- a/ipwhois/tests/test_utils.py +++ b/ipwhois/tests/test_utils.py @@ -157,7 +157,56 @@ def test_unique_addresses(self): # Expected result is different on 2.x vs 3.x, possible issues with # ipaddr vs ipaddress output. Investigation pending... - if sys.version_info >= (3, 3): + if (3, 3) <= sys.version_info < (3, 8): + + fp_expected_result = { + '74.125.225.0/24': {'count': 1, 'ports': {}}, + '62.239.0.0/16': {'count': 1, 'ports': {}}, + '2001:43f8:7b0:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '210.0.0.0': {'count': 1, 'ports': {}}, + '196.11.240.0/23': {'count': 1, 'ports': {}}, + '2001:240:10c:1::ca20:9d1d': {'count': 2, 'ports': {}}, + '196.11.240.215': {'count': 2, 'ports': {}}, + '62.239.237.0/32': {'count': 1, 'ports': {}}, + '210.107.0.0/17': {'count': 6, 'ports': {}}, + '2001:4860::/32': {'count': 1, 'ports': {}}, + '210.107.73.73': {'count': 2, 'ports': {}}, + '210.107.0.0': {'count': 2, 'ports': {}}, + '2001:200::/23': {'count': 2, 'ports': {}}, + '2001:240:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '210.255.255.255': {'count': 1, 'ports': {}}, + '2001:43f8:7b0::': {'count': 3, 'ports': {}}, + '196.255.255.255': {'count': 1, 'ports': {}}, + '2001:240::/32': {'count': 6, 'ports': {}}, + '196.0.0.0': {'count': 1, 'ports': {}}, + '2001:240::': {'count': 1, 'ports': {}}, + '196.11.246.255': {'count': 2, 'ports': {}}, + '196.11.239.0': {'count': 2, 'ports': {}}, + '2001:4200::/23': {'count': 1, 'ports': {}}, + '2a00:2380::/25': {'count': 1, 'ports': {}}, + '200.57.128.0/20': {'count': 1, 'ports': {}}, + '62.239.237.255': {'count': 1, 'ports': {}}, + '2001:4860:4860::8888': {'count': 10, 'ports': {}}, + '2001:4860::': {'count': 2, 'ports': {}}, + '2001:4860:ffff:ffff:ffff:ffff:ffff:ffff': + {'count': 1, 'ports': {}}, + '74.125.225.229': {'count': 8, 'ports': {}}, + '210.107.127.255': {'count': 2, 'ports': {}}, + '200.57.141.161': {'count': 7, 'ports': {}}, + '62.239.237.255/32': {'count': 1, 'ports': {}}, + '2801:10:c000::': {'count': 7, 'ports': {}}, + '2a00:2381:ffff::1': {'count': 4, 'ports': {}}, + '62.239.237.0': {'count': 1, 'ports': {}}, + '62.239.237.1': {'count': 4, 'ports': {}}, + '210.0.0.0/8': {'count': 1, 'ports': {}} + } + + self.assertEqual(unique_addresses(file_path=fp), + fp_expected_result) + + elif sys.version_info >= (3, 8): fp_expected_result = { '196.0.0.0': {'count': 1, 'ports': {}}, From 512efd3a5c99ab97f4ce2313208e30453cff4be3 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 14:29:33 -0500 Subject: [PATCH 25/39] Fixed travis build warnings (#268) --- .travis.yml | 2 +- CHANGES.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f08fb5b..845f994b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -sudo: required +os: linux dist: xenial python: - 2.7 diff --git a/CHANGES.rst b/CHANGES.rst index 7d17b5c2..0bf8a514 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -21,6 +21,7 @@ Changelog - Fixed deprecation warnings due to invalid escape sequences (#272 - tirkarthi) - Added support for Python 3.8 (#267) +- Fixed travis build warnings (#268) 1.1.0 (2019-02-01) ------------------ From 70981aea8660a1362de3e9efd251a0aa90c9dc11 Mon Sep 17 00:00:00 2001 From: secynic Date: Mon, 14 Sep 2020 15:45:54 -0500 Subject: [PATCH 26/39] Pinned requirements (#274) --- CHANGES.rst | 1 + UPGRADING.rst | 1 + ipwhois/docs/requirements.txt | 2 +- requirements/python2.txt | 4 ++-- requirements/python3.txt | 2 +- setup.py | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0bf8a514..aa410a74 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -22,6 +22,7 @@ Changelog (#272 - tirkarthi) - Added support for Python 3.8 (#267) - Fixed travis build warnings (#268) +- Pinned requirements (#274) 1.1.0 (2019-02-01) ------------------ diff --git a/UPGRADING.rst b/UPGRADING.rst index 181bb878..d689a4f1 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -23,6 +23,7 @@ v1.2.0 - Removed deprecated asn_alts parameter - Removed deprecated allow_permutations parameter - Added support for Python 3.8 +- Pinned requirements ****** v1.1.0 diff --git a/ipwhois/docs/requirements.txt b/ipwhois/docs/requirements.txt index b7845bae..9a90c110 100644 --- a/ipwhois/docs/requirements.txt +++ b/ipwhois/docs/requirements.txt @@ -1,3 +1,3 @@ sphinx sphinx_rtd_theme -dnspython +dnspython<=2.0.0 diff --git a/requirements/python2.txt b/requirements/python2.txt index b7f70e79..6f6db6bc 100644 --- a/requirements/python2.txt +++ b/requirements/python2.txt @@ -1,2 +1,2 @@ -dnspython -ipaddr +dnspython<=2.0.0 +ipaddr==2.2.0 diff --git a/requirements/python3.txt b/requirements/python3.txt index 2f735967..337b63d0 100644 --- a/requirements/python3.txt +++ b/requirements/python3.txt @@ -1 +1 @@ -dnspython +dnspython<=2.0.0 diff --git a/setup.py b/setup.py index 949dee52..e651eaf9 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ PACKAGE_DATA = {'ipwhois': ['data/*.xml', 'data/*.csv']} -INSTALL_REQUIRES = ['dnspython', 'ipaddr;python_version<"3.3"'] +INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"'] setup( name=NAME, From 1f5ddcf01293985956b9e86f06a07b7e7105b53e Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 11:48:12 -0500 Subject: [PATCH 27/39] Added ip_failed_total to bulk_lookup_rdap (#235) --- CHANGES.rst | 2 ++ EXPERIMENTAL.rst | 26 ++++++++++++++------------ ipwhois/experimental.py | 7 ++++++- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index aa410a74..b6beac5a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -23,6 +23,8 @@ Changelog - Added support for Python 3.8 (#267) - Fixed travis build warnings (#268) - Pinned requirements (#274) +- Added ip_failed_total key to stats dictionary in + experimental.bulk_lookup_rdap (#235) 1.1.0 (2019-02-01) ------------------ diff --git a/EXPERIMENTAL.rst b/EXPERIMENTAL.rst index d95f6350..0c6179fb 100644 --- a/EXPERIMENTAL.rst +++ b/EXPERIMENTAL.rst @@ -68,21 +68,21 @@ Basic usage >>>> pprint(results.split('\n')) [ - "Bulk mode; whois.cymru.com [2017-07-30 23:02:21 +0000]", - "15169 | 74.125.225.229 | 74.125.225.0/24 | US | arin | 2007-03-13 | GOOGLE - Google Inc., US", - "15169 | 2001:4860:4860::8888 | 2001:4860::/32 | US | arin | 2005-03-14 | GOOGLE - Google Inc., US", + "Bulk mode; whois.cymru.com [2020-09-15 16:42:29 +0000]", + "15169 | 74.125.225.229 | 74.125.225.0/24 | US | arin | 2007-03-13 | GOOGLE, US", + "15169 | 2001:4860:4860::8888 | 2001:4860::/32 | US | arin | 2005-03-14 | GOOGLE, US", "2856 | 62.239.237.1 | 62.239.0.0/16 | GB | ripencc | 2001-01-02 | BT-UK-AS BTnet UK Regional network, GB", "2856 | 2a00:2381:ffff::1 | 2a00:2380::/25 | GB | ripencc | 2007-08-29 | BT-UK-AS BTnet UK Regional network, GB", - "3786 | 210.107.73.73 | 210.107.0.0/17 | KR | apnic | | LGDACOM LG DACOM Corporation, KR", + "3786 | 210.107.73.73 | 210.107.0.0/17 | KR | apnic | 1997-08-29 | LGDACOM LG DACOM Corporation, KR", "2497 | 2001:240:10c:1::ca20:9d1d | 2001:240::/32 | JP | apnic | 2000-03-08 | IIJ Internet Initiative Japan Inc., JP", "19373 | 200.57.141.161 | 200.57.128.0/20 | MX | lacnic | 2000-12-04 | Triara.com, S.A. de C.V., MX", "NA | 2801:10:c000:: | NA | CO | lacnic | 2013-10-29 | NA", - "12091 | 196.11.240.215 | 196.11.240.0/24 | ZA | afrinic | | MTNNS-1, ZA", + "12091 | 196.11.240.215 | 196.11.240.0/24 | ZA | afrinic | 1994-07-21 | MTNNS-1, ZA", "37578 | 2001:43f8:7b0:: | 2001:43f8:7b0::/48 | KE | afrinic | 2013-03-22 | Tespok, KE", - "4730 | 133.1.2.5 | 133.1.0.0/16 | JP | apnic | | ODINS Osaka University, JP", - "4134 | 115.1.2.3 | 115.0.0.0/14 | KR | apnic | 2008-07-01 | CHINANET-BACKBONE No.31,Jin-rong Street, CN", + "4730 | 133.1.2.5 | 133.1.0.0/16 | JP | apnic | 1997-03-01 | ODINS Osaka University, JP", + "4766 | 115.1.2.3 | 115.0.0.0/12 | KR | apnic | 2008-07-01 | KIXS-AS-KR Korea Telecom, KR", "" - } + ] .. GET_BULK_ASN_WHOIS_OUTPUT_BASIC END @@ -175,11 +175,14 @@ The stats dictionary returned by ipwhois.experimental.bulk_lookup_rdap() 'ip_lookup_total' (int) - The total number of addresses that lookups were attempted for, excluding any that failed ASN registry checks. + 'ip_failed_total' (int) - The total number of addresses that + lookups failed for. Excludes any that failed initially, but + succeeded after further retries. 'lacnic' (dict) - { 'failed' (list) - The addresses that failed to lookup. Excludes any that failed initially, but succeeded after - futher retries. + further retries. 'rate_limited' (list) - The addresses that encountered rate-limiting. Unless an address is also in 'failed', it eventually succeeded. @@ -222,9 +225,7 @@ Basic usage "total": 2 }, "apnic": { - "failed": [ - "115.1.2.3" - ], + "failed": [], "rate_limited": [], "total": 4 }, @@ -233,6 +234,7 @@ Basic usage "rate_limited": [], "total": 2 }, + "ip_failed_total": 0, "ip_input_total": 12, "ip_lookup_total": 12, "ip_unique_total": 12, diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 9d597e9c..8f6ce230 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -158,11 +158,14 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, 'ip_lookup_total' (int) - The total number of addresses that lookups were attempted for, excluding any that failed ASN registry checks. + 'ip_failed_total' (int) - The total number of addresses that + lookups failed for. Excludes any that failed initially, but + succeeded after further retries. 'lacnic' (dict) - { 'failed' (list) - The addresses that failed to lookup. Excludes any that failed initially, but succeeded after - futher retries. + further retries. 'rate_limited' (list) - The addresses that encountered rate-limiting. Unless an address is also in 'failed', it eventually succeeded. @@ -196,6 +199,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, 'ip_input_total': len(addresses), 'ip_unique_total': 0, 'ip_lookup_total': 0, + 'ip_failed_total': 0, 'lacnic': {'failed': [], 'rate_limited': [], 'total': 0}, 'ripencc': {'failed': [], 'rate_limited': [], 'total': 0}, 'apnic': {'failed': [], 'rate_limited': [], 'total': 0}, @@ -425,6 +429,7 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, del asn_parsed_results[ip] stats[rir]['failed'].append(ip) + stats['ip_failed_total'] += 1 if rir == 'lacnic': From c4e71b6dc17023e82b38a15e4aea9198df455cb1 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 11:58:13 -0500 Subject: [PATCH 28/39] Fixed test for ip_failed_total (#235) --- ipwhois/tests/online/test_experimental.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipwhois/tests/online/test_experimental.py b/ipwhois/tests/online/test_experimental.py index fc5042ca..6ea9b473 100644 --- a/ipwhois/tests/online/test_experimental.py +++ b/ipwhois/tests/online/test_experimental.py @@ -67,7 +67,8 @@ def test_bulk_lookup_rdap(self): '115.1.2.3' # KRNIC ] - expected_stats = {'ip_input_total': 12, 'ip_unique_total': 12, 'ip_lookup_total': 12, + expected_stats = {'ip_input_total': 12, 'ip_unique_total': 12, + 'ip_lookup_total': 12, 'ip_failed_total': 0, 'lacnic': {'failed': [], 'rate_limited': [], 'total': 2}, 'ripencc': {'failed': [], 'rate_limited': [], 'total': 2}, 'apnic': {'failed': [], 'rate_limited': [], 'total': 4}, From 3ffcdafe0dbdac60b7e116b506ee5b4580674e75 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 14:42:47 -0500 Subject: [PATCH 29/39] Fixed bug in root and sub-entities not getting queried/data (#247) --- CHANGES.rst | 4 + RDAP.rst | 13 +++ UPGRADING.rst | 4 + ipwhois/ipwhois.py | 8 +- ipwhois/rdap.py | 174 +++++++++++++++++++++++++------------ ipwhois/tests/test_rdap.py | 14 +-- 6 files changed, 155 insertions(+), 62 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b6beac5a..2b25efef 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,10 @@ Changelog result was returned (#262 - ameidatou) - Fixed deprecation warnings due to invalid escape sequences (#272 - tirkarthi) +- Fixed bug in root and sub-entities not getting queried/data (#247) +- Added new argument root_ent_check to IPWhois.lookup_rdap and + RDAP.lookup. Set this to False to revert to old functionality - missing data, + but less queries (#247) - Added support for Python 3.8 (#267) - Fixed travis build warnings (#268) - Pinned requirements (#274) diff --git a/RDAP.rst b/RDAP.rst index 9d32b50a..0f8d0fad 100644 --- a/RDAP.rst +++ b/RDAP.rst @@ -73,6 +73,10 @@ Arguments supported by IPWhois.lookup_rdap(). | | | pulling ASN information via dns, in order to | | | | get the ASN description. Defaults to True. | +--------------------+--------+-----------------------------------------------+ +| root_ent_check | bool | If True, will perform additional RDAP HTTP | +| | | queries for missing entity data at the root | +| | | level. Defaults to True. | ++--------------------+--------+-----------------------------------------------+ .. _rdap-output: @@ -593,3 +597,12 @@ this very low for bulk queries, or disable completely by setting retry_count=0. Note that setting this result too low may cause a larger number of IP lookups to fail. + +root_ent_check +^^^^^^^^^^^^^^ + +When root level entities (depth=0) are missing vcard data, additional +entity specific HTTP lookups are performed. In the past, you would expect +depth=0 to mean a single lookup per IP. This was a bug and has been fixed as of +v1.2.0. Set this to False to revert back to the old method, although you will be +missing entity specific data. diff --git a/UPGRADING.rst b/UPGRADING.rst index d689a4f1..300da87e 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -22,6 +22,10 @@ v1.2.0 nir.NIRWhois._get_nets_krnic, nir.NIRWhois._get_contact - Removed deprecated asn_alts parameter - Removed deprecated allow_permutations parameter +- Added new argument root_ent_check to IPWhois.lookup_rdap and + RDAP.lookup. Set this to False to revert to old functionality - missing data, + but less queries. If you leave this set to default of True, you will notice + more queries and potentially more rate-limiting. - Added support for Python 3.8 - Pinned requirements diff --git a/ipwhois/ipwhois.py b/ipwhois/ipwhois.py index 0c3e179d..d5368e8e 100644 --- a/ipwhois/ipwhois.py +++ b/ipwhois/ipwhois.py @@ -199,7 +199,7 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, excluded_entities=None, bootstrap=False, rate_limit_timeout=120, extra_org_map=None, inc_nir=True, nir_field_list=None, asn_methods=None, - get_asn_description=True): + get_asn_description=True, root_ent_check=True): """ The function for retrieving and parsing whois information for an IP address via HTTP (RDAP). @@ -247,6 +247,9 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, get_asn_description (:obj:`bool`): Whether to run an additional query when pulling ASN information via dns, in order to get the ASN description. Defaults to True. + root_ent_check (:obj:`bool`): If True, will perform + additional RDAP HTTP queries for missing entity data at the + root level. Defaults to True. Returns: dict: The IP RDAP lookup results @@ -305,7 +308,8 @@ def lookup_rdap(self, inc_raw=False, retry_count=3, depth=0, inc_raw=inc_raw, retry_count=retry_count, asn_data=asn_data, depth=depth, excluded_entities=excluded_entities, response=response, bootstrap=bootstrap, - rate_limit_timeout=rate_limit_timeout + rate_limit_timeout=rate_limit_timeout, + root_ent_check=root_ent_check ) # Add the RDAP information to the return dictionary. diff --git a/ipwhois/rdap.py b/ipwhois/rdap.py index d019f6d2..1e80338a 100644 --- a/ipwhois/rdap.py +++ b/ipwhois/rdap.py @@ -28,6 +28,7 @@ from .net import ip_address import logging import json +from collections import namedtuple log = logging.getLogger(__name__) @@ -688,9 +689,95 @@ def __init__(self, net): raise NetError('The provided net parameter is not an instance of ' 'ipwhois.net.Net') + def _get_entity(self, entity=None, roles=None, inc_raw=False, retry_count=3, + asn_data=None, bootstrap=False, rate_limit_timeout=120): + """ + The function for retrieving and parsing information for an entity via + RDAP (HTTP). + + Args: + entity (:obj:`str`): The entity name to lookup. + roles (:obj:`dict`): The mapping of entity handles to roles. + inc_raw (:obj:`bool`, optional): Whether to include the raw + results in the returned dictionary. Defaults to False. + retry_count (:obj:`int`): The number of times to retry in case + socket errors, timeouts, connection resets, etc. are + encountered. Defaults to 3. + asn_data (:obj:`dict`): Result from + :obj:`ipwhois.asn.IPASN.lookup`. Optional if the bootstrap + parameter is True. + bootstrap (:obj:`bool`): If True, performs lookups via ARIN + bootstrap rather than lookups based on ASN data. Defaults to + False. + rate_limit_timeout (:obj:`int`): The number of seconds to wait + before retrying when a rate limit notice is returned via + rdap+json. Defaults to 120. + + Returns: + namedtuple: + + :result (dict): Consists of the fields listed in the + ipwhois.rdap._RDAPEntity dict. The raw result is included for + each object if the inc_raw parameter is True. + :roles (dict): The mapping of entity handles to roles. + """ + + result = {} + + if bootstrap: + entity_url = '{0}/entity/{1}'.format( + BOOTSTRAP_URL, entity) + else: + tmp_reg = asn_data['asn_registry'] + entity_url = RIR_RDAP[tmp_reg]['entity_url'] + entity_url = str(entity_url).format(entity) + + try: + + # RDAP entity query + response = self._net.get_http_json( + url=entity_url, retry_count=retry_count, + rate_limit_timeout=rate_limit_timeout + ) + + # Parse the entity + result_ent = _RDAPEntity(response) + result_ent.parse() + result = result_ent.vars + + result['roles'] = None + try: + + result['roles'] = roles[entity] + + except KeyError: # pragma: no cover + + pass + + try: + + for tmp in response['entities']: + + if tmp['handle'] not in roles: + roles[tmp['handle']] = tmp['roles'] + + except (IndexError, KeyError): + + pass + + if inc_raw: + result['raw'] = response + + except (HTTPLookupError, InvalidEntityObject): + + pass + + return_tuple = namedtuple('return_tuple', ['result', 'roles']) + return return_tuple(result, roles) + def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, excluded_entities=None, response=None, bootstrap=False, - rate_limit_timeout=120): + rate_limit_timeout=120, root_ent_check=True): """ The function for retrieving and parsing information for an IP address via RDAP (HTTP). @@ -716,6 +803,9 @@ def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, rate_limit_timeout (:obj:`int`): The number of seconds to wait before retrying when a rate limit notice is returned via rdap+json. Defaults to 120. + root_ent_check (:obj:`bool`): If True, will perform + additional RDAP HTTP queries for missing entity data at the + root level. Defaults to True. Returns: dict: The IP RDAP lookup results @@ -792,10 +882,23 @@ def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, if ent['handle'] not in [results['entities'], excluded_entities]: - result_ent = _RDAPEntity(ent) - result_ent.parse() + if 'vcardArray' not in ent and root_ent_check: + entity_object, roles = self._get_entity( + entity=ent['handle'], + roles=roles, + inc_raw=inc_raw, + retry_count=retry_count, + asn_data=asn_data, + bootstrap=bootstrap, + rate_limit_timeout=rate_limit_timeout + ) + results['objects'][ent['handle']] = entity_object - results['objects'][ent['handle']] = result_ent.vars + else: + result_ent = _RDAPEntity(ent) + result_ent.parse() + + results['objects'][ent['handle']] = result_ent.vars results['entities'].append(ent['handle']) @@ -835,57 +938,18 @@ def lookup(self, inc_raw=False, retry_count=3, asn_data=None, depth=0, list(new_objects.keys()) + excluded_entities): - if bootstrap: - entity_url = '{0}/entity/{1}'.format( - BOOTSTRAP_URL, ent) - else: - tmp_reg = asn_data['asn_registry'] - entity_url = RIR_RDAP[tmp_reg]['entity_url'] - entity_url = str(entity_url).format(ent) - - try: - - # RDAP entity query - response = self._net.get_http_json( - url=entity_url, retry_count=retry_count, - rate_limit_timeout=rate_limit_timeout - ) - - # Parse the entity - result_ent = _RDAPEntity(response) - result_ent.parse() - new_objects[ent] = result_ent.vars - - new_objects[ent]['roles'] = None - try: - - new_objects[ent]['roles'] = roles[ent] - - except KeyError: # pragma: no cover - - pass - - try: - - for tmp in response['entities']: - - if tmp['handle'] not in roles: - - roles[tmp['handle']] = tmp['roles'] - - except (IndexError, KeyError): - - pass - - if inc_raw: - - new_objects[ent]['raw'] = response - - except (HTTPLookupError, InvalidEntityObject): - - pass - - except TypeError: + entity_object, roles = self._get_entity( + entity=ent, + roles=roles, + inc_raw=inc_raw, + retry_count=retry_count, + asn_data=asn_data, + bootstrap=bootstrap, + rate_limit_timeout=rate_limit_timeout + ) + new_objects[ent] = entity_object + + except (KeyError, TypeError): pass diff --git a/ipwhois/tests/test_rdap.py b/ipwhois/tests/test_rdap.py index 2ccdb451..6d4277db 100644 --- a/ipwhois/tests/test_rdap.py +++ b/ipwhois/tests/test_rdap.py @@ -82,7 +82,8 @@ def test_lookup(self): 'endAddress': '74.125.225.229' }, asn_data=val['asn_data'], - depth=0), dict) + depth=0, + root_ent_check=False), dict) log.debug('Testing rdap.lookup entitiy checks') net = Net('74.125.225.229') @@ -99,7 +100,8 @@ def test_lookup(self): 'entities': entity }, asn_data=val['asn_data'], - depth=1), dict) + depth=0, + root_ent_check=False), dict) self.assertIsInstance(obj.lookup(response={ 'handle': 'test', @@ -109,9 +111,10 @@ def test_lookup(self): 'entities': entity }, asn_data=val['asn_data'], - depth=1, + depth=0, bootstrap=True, - inc_raw=True), dict) + inc_raw=True, + root_ent_check=False), dict) # No sub entities. This is for coverage, but won't error out. entity = [{'handle': 'test', 'roles': [ @@ -125,7 +128,8 @@ def test_lookup(self): 'entities': entity }, asn_data=val['asn_data'], - depth=1), dict) + depth=0, + root_ent_check=False), dict) class TestRDAPContact(TestCommon): From a54f2ba15f90645f052f36d604392b0bca35aa83 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 15:18:25 -0500 Subject: [PATCH 30/39] Removed old permutations check --- ipwhois/asn.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ipwhois/asn.py b/ipwhois/asn.py index 60625e2b..e2cda715 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -447,11 +447,6 @@ def lookup(self, inc_raw=False, retry_count=3, extra_org_map=None, dns_success = False for index, lookup_method in enumerate(lookups): - if index > 0 and not asn_methods: - - raise ASNRegistryError('ASN registry lookup failed. ' - 'Permutations not allowed.') - if lookup_method == 'dns': try: From a0428701e7b04b6d7f82ce7d95e4363e3cfd21df Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 18:01:21 -0500 Subject: [PATCH 31/39] Added ipv4_generate_random and ipv6_generate_random to utils CLI (#236) --- CHANGES.rst | 1 + ipwhois/scripts/ipwhois_utils_cli.py | 49 ++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2b25efef..d9df2fae 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -29,6 +29,7 @@ Changelog - Pinned requirements (#274) - Added ip_failed_total key to stats dictionary in experimental.bulk_lookup_rdap (#235) +- Added ipv4_generate_random and ipv6_generate_random to utils CLI (#236) 1.1.0 (2019-02-01) ------------------ diff --git a/ipwhois/scripts/ipwhois_utils_cli.py b/ipwhois/scripts/ipwhois_utils_cli.py index 7f86b95a..769823b9 100644 --- a/ipwhois/scripts/ipwhois_utils_cli.py +++ b/ipwhois/scripts/ipwhois_utils_cli.py @@ -28,8 +28,9 @@ from collections import OrderedDict import json from ipwhois.utils import (ipv4_lstrip_zeros, calculate_cidr, get_countries, - ipv4_is_defined, ipv6_is_defined, unique_everseen, - unique_addresses) + ipv4_is_defined, ipv6_is_defined, + ipv4_generate_random, ipv6_generate_random, + unique_everseen, unique_addresses) # CLI ANSI rendering ANSI = { @@ -86,6 +87,22 @@ metavar='"IP ADDRESS"', help='Check if an IPv6 address is defined (in a reserved address range).' ) +parser.add_argument( + '--ipv4_generate_random', + type=int, + nargs=1, + metavar='"TOTAL"', + help='Generate random, unique IPv4 addresses that are not defined (can be ' + 'looked up using ipwhois).' +) +parser.add_argument( + '--ipv6_generate_random', + type=int, + nargs=1, + metavar='"TOTAL"', + help='Generate random, unique IPv6 addresses that are not defined (can be ' + 'looked up using ipwhois).' +) parser.add_argument( '--unique_everseen', type=json.loads, @@ -224,6 +241,34 @@ print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) +elif script_args.ipv4_generate_random: + + try: + + result = ipv4_generate_random(total=script_args.ipv4_generate_random[0]) + + for random_ip in result: + + print(random_ip) + + except Exception as e: + + print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) + +elif script_args.ipv6_generate_random: + + try: + + result = ipv6_generate_random(total=script_args.ipv6_generate_random[0]) + + for random_ip in result: + + print(random_ip) + + except Exception as e: + + print('{0}Error{1}: {2}'.format(ANSI['red'], ANSI['end'], str(e))) + elif script_args.unique_everseen: try: From e7b2013c22ecb07f89d20ea22810a02d897dc98f Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 18:02:31 -0500 Subject: [PATCH 32/39] Fix metavars (#236) --- ipwhois/scripts/ipwhois_utils_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipwhois/scripts/ipwhois_utils_cli.py b/ipwhois/scripts/ipwhois_utils_cli.py index 769823b9..166ea5d1 100644 --- a/ipwhois/scripts/ipwhois_utils_cli.py +++ b/ipwhois/scripts/ipwhois_utils_cli.py @@ -91,7 +91,7 @@ '--ipv4_generate_random', type=int, nargs=1, - metavar='"TOTAL"', + metavar='TOTAL', help='Generate random, unique IPv4 addresses that are not defined (can be ' 'looked up using ipwhois).' ) @@ -99,7 +99,7 @@ '--ipv6_generate_random', type=int, nargs=1, - metavar='"TOTAL"', + metavar='TOTAL', help='Generate random, unique IPv6 addresses that are not defined (can be ' 'looked up using ipwhois).' ) From ac71f351fff8aa1dfb92882a813abb4eac3ea9f5 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 18:59:00 -0500 Subject: [PATCH 33/39] ipv4_generate_random and ipv6_generate_random docs (#236) --- CLI.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/CLI.rst b/CLI.rst index 03902a24..9c6934fd 100644 --- a/CLI.rst +++ b/CLI.rst @@ -168,6 +168,12 @@ optional arguments: --ipv6_is_defined IPADDRESS Check if an IPv6 address is defined (in a reserved address range). + --ipv4_generate_random TOTAL + Generate random, unique IPv4 addresses that are not + defined (can be looked up using ipwhois). + --ipv6_generate_random TOTAL + Generate random, unique IPv6 addresses that are not + defined (can be looked up using ipwhois). --unique_everseen ITERABLE List unique elements from input iterable, preserving the order. @@ -261,6 +267,32 @@ ipv6_is_defined Name: Unique Local Unicast RFC: RFC 4193 +ipv4_generate_random +^^^^^^^^^^^^^^^^^^^^ + +:: + + >>>> ipwhois_utils_cli.py --ipv4_generate_random 5 + + 119.224.47.74 + 128.106.183.195 + 54.97.0.158 + 52.206.105.37 + 126.180.201.81 + +ipv6_generate_random +^^^^^^^^^^^^^^^^^^^^ + +:: + + >>>> ipwhois_utils_cli.py --ipv6_generate_random 5 + + 3e8c:dc93:49c8:57fd:31dd:2963:6332:426e + 2e3d:fd84:b57b:9282:91e6:5d4d:18d5:34f1 + 21d4:9d25:7dd6:e28b:77d7:7ce9:f85f:b34f + 3659:2b9:12ed:1eac:fd40:5756:3753:6d2d + 2e05:6d47:83fd:5de8:c6cb:85cb:912:fdb1 + unique_everseen ^^^^^^^^^^^^^^^ From f499cb1a135985d664ec0b51b1d9a3231d7c7b12 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 19:35:36 -0500 Subject: [PATCH 34/39] Added GitHub issues badge --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 712c9392..524bcd78 100644 --- a/README.rst +++ b/README.rst @@ -7,8 +7,10 @@ ipwhois .. image:: https://coveralls.io/repos/github/secynic/ipwhois/badge.svg?branch= master :target: https://coveralls.io/github/secynic/ipwhois?branch=master +.. image:: https://img.shields.io/github/issues-raw/secynic/ipwhois + :target: https://github.com/secynic/ipwhois/issues .. image:: https://codeclimate.com/github/secynic/ipwhois/badges/issue_count.svg - :target: https://codeclimate.com/github/secynic/ipwhois + :target: https://codeclimate.com/github/secynic/ipwhois .. image:: https://img.shields.io/badge/license-BSD%202--Clause-blue.svg :target: https://github.com/secynic/ipwhois/tree/master/LICENSE.txt .. image:: https://img.shields.io/badge/python-2.7%2C%203.4+-blue.svg From 429fb0cee6fd23f3f24de0300da815f527f23640 Mon Sep 17 00:00:00 2001 From: secynic Date: Tue, 15 Sep 2020 19:50:21 -0500 Subject: [PATCH 35/39] Added documentation note for ASN data (#278) --- ASN.rst | 19 +++++++++++++++++++ CHANGES.rst | 1 + 2 files changed, 20 insertions(+) diff --git a/ASN.rst b/ASN.rst index bcfed90d..23691ede 100644 --- a/ASN.rst +++ b/ASN.rst @@ -5,6 +5,25 @@ IP ASN Lookups This is new functionality as of v0.15.0. This functionality was migrated from net.Net and is still used by IPWhois.lookup*(). +.. note:: + + Cymru ASN data should not be considered a primary source for data points + like country code. + + Message from the Cymru site:: + + The country code, registry, and allocation date are all based on data + obtained directly from the regional registries including: ARIN, RIPE, + AFRINIC, APNIC, LACNIC. The information returned relating to these + categories will only be as accurate as the data present in the RIR + databases. + + IMPORTANT NOTE: Country codes are likely to vary significantly from + actual IP locations, and we must strongly advise that the IP to ASN + mapping tool not be used as an IP geolocation (GeoIP) service. + + https://team-cymru.com/community-services/ip-asn-mapping/ + .. _ip-asn-input: IP ASN Input diff --git a/CHANGES.rst b/CHANGES.rst index d9df2fae..c70c0303 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -30,6 +30,7 @@ Changelog - Added ip_failed_total key to stats dictionary in experimental.bulk_lookup_rdap (#235) - Added ipv4_generate_random and ipv6_generate_random to utils CLI (#236) +- Added documentation note for ASN data (#278) 1.1.0 (2019-02-01) ------------------ From 50df6efb9631cea7e9db38c5e31e0a6d0dc9ba33 Mon Sep 17 00:00:00 2001 From: secynic Date: Thu, 17 Sep 2020 09:45:15 -0500 Subject: [PATCH 36/39] Fixed NIR datetime parsing issue if only date is returned (#284) --- CHANGES.rst | 1 + ipwhois/nir.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c70c0303..8bcb171d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -21,6 +21,7 @@ Changelog - Fixed deprecation warnings due to invalid escape sequences (#272 - tirkarthi) - Fixed bug in root and sub-entities not getting queried/data (#247) +- Fixed NIR datetime parsing issue if only date is returned (#284) - Added new argument root_ent_check to IPWhois.lookup_rdap and RDAP.lookup. Set this to False to revert to old functionality - missing data, but less queries (#247) diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 3d2f6035..231f3795 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -265,12 +265,20 @@ def parse_fields(self, response, fields_dict, net_start=None, if field in ['created', 'updated'] and dt_format: - value = ( - datetime.strptime( - values[0], - str(dt_format) - ) - timedelta(hours=hourdelta) - ).isoformat('T') + try: + value = ( + datetime.strptime( + values[0], + str(dt_format) + ) - timedelta(hours=hourdelta) + ).isoformat('T') + except ValueError: + value = ( + datetime.strptime( + values[0], + '%Y/%m/%d' + ) + ).isoformat('T') elif field in ['nameservers']: From 4df5d37c249b0ec53002fd729d0806797df2d266 Mon Sep 17 00:00:00 2001 From: Philip Hane Date: Thu, 17 Sep 2020 10:40:23 -0500 Subject: [PATCH 37/39] Create python-publish-test.yml --- .github/workflows/python-publish-test.yml | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/python-publish-test.yml diff --git a/.github/workflows/python-publish-test.yml b/.github/workflows/python-publish-test.yml new file mode 100644 index 00000000..fd6650eb --- /dev/null +++ b/.github/workflows/python-publish-test.yml @@ -0,0 +1,26 @@ +name: Upload Python Package to Test PyPi + +on: workflow_dispatch + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: secynic + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload --repository testpypi dist/* From 9f0aaef0c92dbbf520c0fe2139841b24f4117c06 Mon Sep 17 00:00:00 2001 From: Philip Hane Date: Thu, 17 Sep 2020 10:58:20 -0500 Subject: [PATCH 38/39] Create python-publish.yml --- .github/workflows/python-publish.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 00000000..6de16a55 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,28 @@ +name: Upload Python Package to PyPi + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: secynic + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* From 40cf38a46bec1fc140904a7c72499082f8443727 Mon Sep 17 00:00:00 2001 From: secynic Date: Thu, 17 Sep 2020 12:26:38 -0500 Subject: [PATCH 39/39] Increment version 1.2.0 --- CHANGES.rst | 2 +- LICENSE.txt | 2 +- ipwhois/__init__.py | 4 ++-- ipwhois/asn.py | 2 +- ipwhois/docs/source/conf.py | 6 +++--- ipwhois/exceptions.py | 2 +- ipwhois/hr.py | 2 +- ipwhois/ipwhois.py | 2 +- ipwhois/net.py | 2 +- ipwhois/nir.py | 2 +- ipwhois/rdap.py | 2 +- ipwhois/scripts/docs/generate_examples.py | 2 +- ipwhois/scripts/ipwhois_cli.py | 2 +- ipwhois/scripts/ipwhois_utils_cli.py | 2 +- ipwhois/utils.py | 2 +- ipwhois/whois.py | 2 +- setup.py | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8bcb171d..52519b34 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,7 +1,7 @@ Changelog ========= -1.2.0 (TBD) +1.2.0 (2020-09-17) ------------------ - Removed deprecated functions: asn.IPASN._parse_fields_http, diff --git a/LICENSE.txt b/LICENSE.txt index 0da70ecd..be67aadd 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2013-2019 Philip Hane +Copyright (c) 2013-2020 Philip Hane All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/__init__.py b/ipwhois/__init__.py index 440666f9..2a1402bd 100644 --- a/ipwhois/__init__.py +++ b/ipwhois/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -26,4 +26,4 @@ from .net import Net from .ipwhois import IPWhois -__version__ = '1.1.0' +__version__ = '1.2.0' diff --git a/ipwhois/asn.py b/ipwhois/asn.py index e2cda715..71b56758 100644 --- a/ipwhois/asn.py +++ b/ipwhois/asn.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/docs/source/conf.py b/ipwhois/docs/source/conf.py index 108ae4d9..1039de6d 100644 --- a/ipwhois/docs/source/conf.py +++ b/ipwhois/docs/source/conf.py @@ -60,16 +60,16 @@ # General information about the project. project = 'ipwhois' -copyright = '2013-2019, Philip Hane' +copyright = '2013-2020, Philip Hane' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.1.0' +version = '1.2.0' # The full version, including alpha/beta/rc tags. -release = '1.1.0' +release = '1.2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/ipwhois/exceptions.py b/ipwhois/exceptions.py index 066814a6..88251876 100644 --- a/ipwhois/exceptions.py +++ b/ipwhois/exceptions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/hr.py b/ipwhois/hr.py index b51196e4..31383b7b 100644 --- a/ipwhois/hr.py +++ b/ipwhois/hr.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/ipwhois.py b/ipwhois/ipwhois.py index d5368e8e..946c64b5 100644 --- a/ipwhois/ipwhois.py +++ b/ipwhois/ipwhois.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/net.py b/ipwhois/net.py index ac23c350..c7ce39cd 100644 --- a/ipwhois/net.py +++ b/ipwhois/net.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/nir.py b/ipwhois/nir.py index 231f3795..b3c79efb 100644 --- a/ipwhois/nir.py +++ b/ipwhois/nir.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/rdap.py b/ipwhois/rdap.py index 1e80338a..3a65aef2 100644 --- a/ipwhois/rdap.py +++ b/ipwhois/rdap.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/scripts/docs/generate_examples.py b/ipwhois/scripts/docs/generate_examples.py index 68e21373..c5a6ea2b 100644 --- a/ipwhois/scripts/docs/generate_examples.py +++ b/ipwhois/scripts/docs/generate_examples.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/scripts/ipwhois_cli.py b/ipwhois/scripts/ipwhois_cli.py index 74a08f93..c36b7919 100644 --- a/ipwhois/scripts/ipwhois_cli.py +++ b/ipwhois/scripts/ipwhois_cli.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/scripts/ipwhois_utils_cli.py b/ipwhois/scripts/ipwhois_utils_cli.py index 166ea5d1..d2895d2b 100644 --- a/ipwhois/scripts/ipwhois_utils_cli.py +++ b/ipwhois/scripts/ipwhois_utils_cli.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/utils.py b/ipwhois/utils.py index 920ed0f8..f3c0a76e 100644 --- a/ipwhois/utils.py +++ b/ipwhois/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/ipwhois/whois.py b/ipwhois/whois.py index 99dc07c9..40c131d3 100644 --- a/ipwhois/whois.py +++ b/ipwhois/whois.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2019 Philip Hane +# Copyright (c) 2013-2020 Philip Hane # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/setup.py b/setup.py index e651eaf9..ef22a097 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import io NAME = 'ipwhois' -VERSION = '1.1.0' +VERSION = '1.2.0' AUTHOR = 'Philip Hane' AUTHOR_EMAIL = 'secynic@gmail.com' DESCRIPTION = 'Retrieve and parse whois data for IPv4 and IPv6 addresses.'