diff --git a/musicbrainzngs/compat.py b/musicbrainzngs/compat.py index 36574b5..dae21d5 100644 --- a/musicbrainzngs/compat.py +++ b/musicbrainzngs/compat.py @@ -39,23 +39,12 @@ if is_py2: from StringIO import StringIO - from urllib2 import HTTPPasswordMgr, HTTPDigestAuthHandler, Request,\ - HTTPHandler, build_opener, HTTPError, URLError,\ - build_opener - from httplib import BadStatusLine, HTTPException - from urlparse import urlunparse - from urllib import urlencode bytes = str unicode = unicode basestring = basestring elif is_py3: from io import StringIO - from urllib.request import HTTPPasswordMgr, HTTPDigestAuthHandler, Request,\ - HTTPHandler, build_opener - from urllib.error import HTTPError, URLError - from http.client import HTTPException, BadStatusLine - from urllib.parse import urlunparse, urlencode unicode = str bytes = bytes diff --git a/musicbrainzngs/musicbrainz.py b/musicbrainzngs/musicbrainz.py index 121ef18..2271b49 100644 --- a/musicbrainzngs/musicbrainz.py +++ b/musicbrainzngs/musicbrainz.py @@ -7,12 +7,10 @@ import threading import time import logging -import socket -import hashlib -import locale -import sys import xml.etree.ElementTree as etree from xml.parsers import expat +import requests +from requests.auth import HTTPDigestAuth from musicbrainzngs import mbxml from musicbrainzngs import util @@ -365,126 +363,9 @@ def __call__(self, *args, **kwargs): self.remaining_requests -= 1.0 return self.fun(*args, **kwargs) -# From pymb2 -class _RedirectPasswordMgr(compat.HTTPPasswordMgr): - def __init__(self): - self._realms = { } - - def find_user_password(self, realm, uri): - # ignoring the uri parameter intentionally - try: - return self._realms[realm] - except KeyError: - return (None, None) - - def add_password(self, realm, uri, username, password): - # ignoring the uri parameter intentionally - self._realms[realm] = (username, password) - -class _DigestAuthHandler(compat.HTTPDigestAuthHandler): - def get_authorization (self, req, chal): - qop = chal.get ('qop', None) - if qop and ',' in qop and 'auth' in qop.split (','): - chal['qop'] = 'auth' - - return compat.HTTPDigestAuthHandler.get_authorization (self, req, chal) - - def _encode_utf8(self, msg): - """The MusicBrainz server also accepts UTF-8 encoded passwords.""" - encoding = sys.stdin.encoding or locale.getpreferredencoding() - try: - # This works on Python 2 (msg in bytes) - msg = msg.decode(encoding) - except AttributeError: - # on Python 3 (msg is already in unicode) - pass - return msg.encode("utf-8") - - def get_algorithm_impls(self, algorithm): - # algorithm should be case-insensitive according to RFC2617 - algorithm = algorithm.upper() - # lambdas assume digest modules are imported at the top level - if algorithm == 'MD5': - H = lambda x: hashlib.md5(self._encode_utf8(x)).hexdigest() - elif algorithm == 'SHA': - H = lambda x: hashlib.sha1(self._encode_utf8(x)).hexdigest() - # XXX MD5-sess - KD = lambda s, d: H("%s:%s" % (s, d)) - return H, KD - -class _MusicbrainzHttpRequest(compat.Request): - """ A custom request handler that allows DELETE and PUT""" - def __init__(self, method, url, data=None): - compat.Request.__init__(self, url, data) - allowed_m = ["GET", "POST", "DELETE", "PUT"] - if method not in allowed_m: - raise ValueError("invalid method: %s" % method) - self.method = method - - def get_method(self): - return self.method - # Core (internal) functions for calling the MB API. -def _safe_read(opener, req, body=None, max_retries=8, retry_delay_delta=2.0): - """Open an HTTP request with a given URL opener and (optionally) a - request body. Transient errors lead to retries. Permanent errors - and repeated errors are translated into a small set of handleable - exceptions. Return a bytestring. - """ - last_exc = None - for retry_num in range(max_retries): - if retry_num: # Not the first try: delay an increasing amount. - _log.debug("retrying after delay (#%i)" % retry_num) - time.sleep(retry_num * retry_delay_delta) - - try: - if body: - f = opener.open(req, body) - else: - f = opener.open(req) - return f.read() - - except compat.HTTPError as exc: - if exc.code in (400, 404, 411): - # Bad request, not found, etc. - raise ResponseError(cause=exc) - elif exc.code in (503, 502, 500): - # Rate limiting, internal overloading... - _log.debug("HTTP error %i" % exc.code) - elif exc.code in (401, ): - raise AuthenticationError(cause=exc) - else: - # Other, unknown error. Should handle more cases, but - # retrying for now. - _log.debug("unknown HTTP error %i" % exc.code) - last_exc = exc - except compat.BadStatusLine as exc: - _log.debug("bad status line") - last_exc = exc - except compat.HTTPException as exc: - _log.debug("miscellaneous HTTP exception: %s" % str(exc)) - last_exc = exc - except compat.URLError as exc: - if isinstance(exc.reason, socket.error): - code = exc.reason.errno - if code == 104: # "Connection reset by peer." - continue - raise NetworkError(cause=exc) - except socket.timeout as exc: - _log.debug("socket timeout") - last_exc = exc - except socket.error as exc: - if exc.errno == 104: - continue - raise NetworkError(cause=exc) - except IOError as exc: - raise NetworkError(cause=exc) - - # Out of retries! - raise NetworkError("retried %i times" % max_retries, last_exc) - # Get the XML parsing exceptions to catch. The behavior chnaged with Python 2.7 # and ElementTree 1.3. if hasattr(etree, 'ParseError'): @@ -513,55 +394,40 @@ def _mb_request(path, method='GET', auth_required=False, client_required=False, if client_required: args["client"] = _client - # Encode Unicode arguments using UTF-8. - for key, value in args.items(): - if isinstance(value, compat.unicode): - args[key] = value.encode('utf8') - - # Construct the full URL for the request, including hostname and - # query string. - url = compat.urlunparse(( - 'http', - hostname, - '/ws/2/%s' % path, - '', - compat.urlencode(args), - '' - )) - _log.debug("%s request for %s" % (method, url)) - - # Set up HTTP request handler and URL opener. - httpHandler = compat.HTTPHandler(debuglevel=0) - handlers = [httpHandler] - - # Add credentials if required. - if auth_required: - _log.debug("Auth required for %s" % url) - if not user: - raise UsageError("authorization required; " - "use auth(user, pass) first") - passwordMgr = _RedirectPasswordMgr() - authHandler = _DigestAuthHandler(passwordMgr) - authHandler.add_password("musicbrainz.org", (), user, password) - handlers.append(authHandler) - - opener = compat.build_opener(*handlers) - - # Make request. - req = _MusicbrainzHttpRequest(method, url, data) - req.add_header('User-Agent', _useragent) - _log.debug("requesting with UA %s" % _useragent) + headers = {} if body: - req.add_header('Content-Type', 'application/xml; charset=UTF-8') - elif not data and not req.has_header('Content-Length'): + headers['Content-Type'] = 'application/xml; charset=UTF-8' + else: # Explicitly indicate zero content length if no request data # will be sent (avoids HTTP 411 error). - req.add_header('Content-Length', '0') - resp = _safe_read(opener, req, body) + headers['Content-Length'] = '0' + + req = requests.Request( + method, + 'http://{0}/ws/2/{1}'.format(hostname, path), + params=args, + auth=HTTPDigestAuth(user, password) if auth_required else None, + headers=headers, + data=body, + ) + + # Make request (with retries). + session = requests.Session() + adapter = requests.adapters.HTTPAdapter(max_retries=8) + session.mount('http://', adapter) + session.mount('https://', adapter) + try: + resp = session.send(req.prepare(), allow_redirects=True) + except requests.RequestException as exc: + raise NetworkError(cause=exc) + if resp.status_code != 200: + raise ResponseError( + 'API responded with code {0}'.format(resp.status_code) + ) # Parse the response. try: - return mbxml.parse_message(resp) + return mbxml.parse_message(resp.content) except UnicodeError as exc: raise ResponseError(cause=exc) except Exception as exc: diff --git a/setup.py b/setup.py index 1405492..fe07443 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import sys -from distutils.core import setup -from distutils.core import Command +from setuptools import setup +from setuptools import Command from musicbrainzngs import musicbrainz @@ -53,6 +53,9 @@ def run(self): url="https://github.com/alastair/python-musicbrainz-ngs", packages=['musicbrainzngs'], cmdclass={'test': test }, + install_requires=[ + 'requests>=1.2.1' + ], license='BSD 2-clause', classifiers=[ "Development Status :: 3 - Alpha",