depend on requests (experimental)

musicbrainzngs/compat.py

@@ -39,23 +39,12 @@
 
 if is_py2:
 	from StringIO import StringIO
-	from urllib2 import HTTPPasswordMgr, HTTPDigestAuthHandler, Request,\
-						HTTPHandler, build_opener, HTTPError, URLError,\
-						build_opener
-	from httplib import BadStatusLine, HTTPException
-	from urlparse import urlunparse
-	from urllib import urlencode
 
 	bytes = str
 	unicode = unicode
 	basestring = basestring
 elif is_py3:
 	from io import StringIO
-	from urllib.request import HTTPPasswordMgr, HTTPDigestAuthHandler, Request,\
-								HTTPHandler, build_opener
-	from urllib.error import HTTPError, URLError
-	from http.client import HTTPException, BadStatusLine
-	from urllib.parse import urlunparse, urlencode
 
 	unicode = str
 	bytes = bytes

musicbrainzngs/musicbrainz.py

@@ -7,12 +7,10 @@
 import threading
 import time
 import logging
-import socket
-import hashlib
-import locale
-import sys
 import xml.etree.ElementTree as etree
 from xml.parsers import expat
+import requests
+from requests.auth import HTTPDigestAuth
 
 from musicbrainzngs import mbxml
 from musicbrainzngs import util
@@ -365,126 +363,9 @@ def __call__(self, *args, **kwargs):
                 self.remaining_requests -= 1.0
             return self.fun(*args, **kwargs)
 
-# From pymb2
-class _RedirectPasswordMgr(compat.HTTPPasswordMgr):
-	def __init__(self):
-		self._realms = { }
-
-	def find_user_password(self, realm, uri):
-		# ignoring the uri parameter intentionally
-		try:
-			return self._realms[realm]
-		except KeyError:
-			return (None, None)
-
-	def add_password(self, realm, uri, username, password):
-		# ignoring the uri parameter intentionally
-		self._realms[realm] = (username, password)
-
-class _DigestAuthHandler(compat.HTTPDigestAuthHandler):
-    def get_authorization (self, req, chal):
-        qop = chal.get ('qop', None)
-        if qop and ',' in qop and 'auth' in qop.split (','):
-            chal['qop'] = 'auth'
-
-        return compat.HTTPDigestAuthHandler.get_authorization (self, req, chal)
-
-    def _encode_utf8(self, msg):
-        """The MusicBrainz server also accepts UTF-8 encoded passwords."""
-        encoding = sys.stdin.encoding or locale.getpreferredencoding()
-        try:
-            # This works on Python 2 (msg in bytes)
-            msg = msg.decode(encoding)
-        except AttributeError:
-            # on Python 3 (msg is already in unicode)
-            pass
-        return msg.encode("utf-8")
-
-    def get_algorithm_impls(self, algorithm):
-        # algorithm should be case-insensitive according to RFC2617
-        algorithm = algorithm.upper()
-        # lambdas assume digest modules are imported at the top level
-        if algorithm == 'MD5':
-            H = lambda x: hashlib.md5(self._encode_utf8(x)).hexdigest()
-        elif algorithm == 'SHA':
-            H = lambda x: hashlib.sha1(self._encode_utf8(x)).hexdigest()
-        # XXX MD5-sess
-        KD = lambda s, d: H("%s:%s" % (s, d))
-        return H, KD
-
-class _MusicbrainzHttpRequest(compat.Request):
-	""" A custom request handler that allows DELETE and PUT"""
-	def __init__(self, method, url, data=None):
-		compat.Request.__init__(self, url, data)
-		allowed_m = ["GET", "POST", "DELETE", "PUT"]
-		if method not in allowed_m:
-			raise ValueError("invalid method: %s" % method)
-		self.method = method
-
-	def get_method(self):
-		return self.method
-
 
 # Core (internal) functions for calling the MB API.
 
-def _safe_read(opener, req, body=None, max_retries=8, retry_delay_delta=2.0):
-	"""Open an HTTP request with a given URL opener and (optionally) a
-	request body. Transient errors lead to retries.  Permanent errors
-	and repeated errors are translated into a small set of handleable
-	exceptions. Return a bytestring.
-	"""
-	last_exc = None
-	for retry_num in range(max_retries):
-		if retry_num: # Not the first try: delay an increasing amount.
-			_log.debug("retrying after delay (#%i)" % retry_num)
-			time.sleep(retry_num * retry_delay_delta)
-
-		try:
-			if body:
-				f = opener.open(req, body)
-			else:
-				f = opener.open(req)
-			return f.read()
-
-		except compat.HTTPError as exc:
-			if exc.code in (400, 404, 411):
-				# Bad request, not found, etc.
-				raise ResponseError(cause=exc)
-			elif exc.code in (503, 502, 500):
-				# Rate limiting, internal overloading...
-				_log.debug("HTTP error %i" % exc.code)
-			elif exc.code in (401, ):
-				raise AuthenticationError(cause=exc)
-			else:
-				# Other, unknown error. Should handle more cases, but
-				# retrying for now.
-				_log.debug("unknown HTTP error %i" % exc.code)
-			last_exc = exc
-		except compat.BadStatusLine as exc:
-			_log.debug("bad status line")
-			last_exc = exc
-		except compat.HTTPException as exc:
-			_log.debug("miscellaneous HTTP exception: %s" % str(exc))
-			last_exc = exc
-		except compat.URLError as exc:
-			if isinstance(exc.reason, socket.error):
-				code = exc.reason.errno
-				if code == 104: # "Connection reset by peer."
-					continue
-			raise NetworkError(cause=exc)
-		except socket.timeout as exc:
-			_log.debug("socket timeout")
-			last_exc = exc
-		except socket.error as exc:
-			if exc.errno == 104:
-				continue
-			raise NetworkError(cause=exc)
-		except IOError as exc:
-			raise NetworkError(cause=exc)
-
-	# Out of retries!
-	raise NetworkError("retried %i times" % max_retries, last_exc)
-
 # Get the XML parsing exceptions to catch. The behavior chnaged with Python 2.7
 # and ElementTree 1.3.
 if hasattr(etree, 'ParseError'):
@@ -513,55 +394,40 @@ def _mb_request(path, method='GET', auth_required=False, client_required=False,
 	if client_required:
 		args["client"] = _client
 
-	# Encode Unicode arguments using UTF-8.
-	for key, value in args.items():
-		if isinstance(value, compat.unicode):
-			args[key] = value.encode('utf8')
-
-	# Construct the full URL for the request, including hostname and
-	# query string.
-	url = compat.urlunparse((
-		'http',
-		hostname,
-		'/ws/2/%s' % path,
-		'',
-		compat.urlencode(args),
-		''
-	))
-	_log.debug("%s request for %s" % (method, url))
-
-	# Set up HTTP request handler and URL opener.
-	httpHandler = compat.HTTPHandler(debuglevel=0)
-	handlers = [httpHandler]
-
-	# Add credentials if required.
-	if auth_required:
-		_log.debug("Auth required for %s" % url)
-		if not user:
-			raise UsageError("authorization required; "
-							 "use auth(user, pass) first")
-		passwordMgr = _RedirectPasswordMgr()
-		authHandler = _DigestAuthHandler(passwordMgr)
-		authHandler.add_password("musicbrainz.org", (), user, password)
-		handlers.append(authHandler)
-
-	opener = compat.build_opener(*handlers)
-
-	# Make request.
-	req = _MusicbrainzHttpRequest(method, url, data)
-	req.add_header('User-Agent', _useragent)
-	_log.debug("requesting with UA %s" % _useragent)
+	headers = {}
 	if body:
-		req.add_header('Content-Type', 'application/xml; charset=UTF-8')
-	elif not data and not req.has_header('Content-Length'):
+		headers['Content-Type'] = 'application/xml; charset=UTF-8'
+	else:
 		# Explicitly indicate zero content length if no request data
 		# will be sent (avoids HTTP 411 error).
-		req.add_header('Content-Length', '0')
-	resp = _safe_read(opener, req, body)
+		headers['Content-Length'] = '0'
+
+	req = requests.Request(
+		method,
+		'http://{0}/ws/2/{1}'.format(hostname, path),
+		params=args,
+		auth=HTTPDigestAuth(user, password) if auth_required else None,
+		headers=headers,
+		data=body,
+	)
+
+	# Make request (with retries).
+	session = requests.Session()
+	adapter = requests.adapters.HTTPAdapter(max_retries=8)
+	session.mount('http://', adapter)
+	session.mount('https://', adapter)
+	try:
+		resp = session.send(req.prepare(), allow_redirects=True)
+	except requests.RequestException as exc:
+		raise NetworkError(cause=exc)
+	if resp.status_code != 200:
+		raise ResponseError(
+			'API responded with code {0}'.format(resp.status_code)
+		)
 
 	# Parse the response.
 	try:
-		return mbxml.parse_message(resp)
+		return mbxml.parse_message(resp.content)
 	except UnicodeError as exc:
 		raise ResponseError(cause=exc)
 	except Exception as exc:

setup.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 import sys
 
-from distutils.core import setup
-from distutils.core import Command
+from setuptools import setup
+from setuptools import Command
 
 from musicbrainzngs import musicbrainz
 
@@ -53,6 +53,9 @@ def run(self):
     url="https://github.com/alastair/python-musicbrainz-ngs",
     packages=['musicbrainzngs'],
     cmdclass={'test': test },
+    install_requires=[
+        'requests>=1.2.1'
+    ],
     license='BSD 2-clause',
     classifiers=[
         "Development Status :: 3 - Alpha",