From 716f059fc461dce9d275baf0b00a7f32ba6c24a5 Mon Sep 17 00:00:00 2001
From: Bernard Grymonpon <bernard@openminds.be>
Date: Tue, 24 Apr 2012 16:50:22 +0200
Subject: [PATCH 01/47] break from the for-attempts loop when we complete a
succesfull proxy-attempt
---
mantrid/actions.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 73f6d63..ed3c2d8 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -154,6 +154,7 @@ def send_onwards(data):
try:
size = send_onwards(read_data)
size += SocketMelder(sock, server_sock).run()
+ break
except socket.error, e:
if e.errno != errno.EPIPE:
raise
From 1702dc46006cccfc679fe2cc99f3bcbf9c589486 Mon Sep 17 00:00:00 2001
From: Pawel Niznik <pawelniznik@data.futuresimple.local>
Date: Fri, 15 Jun 2012 15:43:08 +0200
Subject: [PATCH 02/47] Fixing HTTP compatibility - headers[Connection] =
"close\r"
---
mantrid/loadbalancer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index d22d16c..f3d5e94 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -301,7 +301,7 @@ def handle(self, sock, address, internal=False):
host = headers['Host']
except KeyError:
host = "unknown"
- headers['Connection'] = "close"
+ headers['Connection'] = "close\r"
if not internal:
headers['X-Forwarded-For'] = address[0]
headers['X-Forwarded-Protocol'] = ""
From 194cd9eb7ea118f09053f78c86856129d34f257f Mon Sep 17 00:00:00 2001
From: Pawel Niznik <pawelniznik@data.futuresimple.local>
Date: Thu, 21 Jun 2012 15:43:56 +0200
Subject: [PATCH 03/47] Introducing new header Host -> LoadBalanceTo
---
mantrid/loadbalancer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index f3d5e94..65e85d1 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -298,7 +298,7 @@ def handle(self, sock, address, internal=False):
headers = mimetools.Message(rfile, 0)
# Work out the host
try:
- host = headers['Host']
+ host = headers['LoadBalanceTo']
except KeyError:
host = "unknown"
headers['Connection'] = "close\r"
From c62af255d0d3af84fe689aa6543d59cc4e128255 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Wed, 5 Dec 2012 18:02:53 +0100
Subject: [PATCH 04/47] Added configurable load balancing algorithms.
Currently only "random" and "least connections" are supported.
---
mantrid/actions.py | 26 ++++++++++++++++++-----
mantrid/backend.py | 28 +++++++++++++++++++++++++
mantrid/cli.py | 13 +++++++-----
mantrid/client.py | 6 +++---
mantrid/json.py | 46 +++++++++++++++++++++++++++++++++++++++++
mantrid/loadbalancer.py | 19 ++++++++++-------
mantrid/management.py | 11 +++++-----
7 files changed, 123 insertions(+), 26 deletions(-)
create mode 100644 mantrid/backend.py
create mode 100644 mantrid/json.py
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 73f6d63..9c2a8f1 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -3,12 +3,16 @@
"""
import errno
+import operator
import os
import random
+
import eventlet
from eventlet.green import socket
from httplib import responses
-from .socketmeld import SocketMelder
+
+from mantrid.backend import Backend
+from mantrid.socketmeld import SocketMelder
class Action(object):
@@ -128,35 +132,47 @@ class Proxy(Action):
attempts = 1
delay = 1
- def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=None):
+ def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=None, algorithm='random'):
super(Proxy, self).__init__(balancer, host, matched_host)
self.backends = backends
+ self.algorithm = 'random'
+ self.select_backend = self.least_connections if algorithm == 'least_connections' else self.random
assert self.backends
if attempts is not None:
self.attempts = int(attempts)
if delay is not None:
self.delay = float(delay)
+ def random(self):
+ return random.choice(self.backends)
+
+ def least_connections(self):
+ return min(self.backends, key=operator.attrgetter('connections'))
+
def handle(self, sock, read_data, path, headers):
"Sends back a static error page."
for i in range(self.attempts):
try:
- server_sock = eventlet.connect(
- tuple(random.choice(self.backends)),
- )
+ backend = self.select_backend()
+ server_sock = eventlet.connect((backend.host, backend.port))
+ backend.add_connection()
except socket.error:
eventlet.sleep(self.delay)
continue
+
# Function to help track data usage
def send_onwards(data):
server_sock.sendall(data)
return len(data)
+
try:
size = send_onwards(read_data)
size += SocketMelder(sock, server_sock).run()
except socket.error, e:
if e.errno != errno.EPIPE:
raise
+ finally:
+ backend.drop_connection()
class Spin(Action):
diff --git a/mantrid/backend.py b/mantrid/backend.py
new file mode 100644
index 0000000..448ad26
--- /dev/null
+++ b/mantrid/backend.py
@@ -0,0 +1,28 @@
+
+class Backend(object):
+ def __init__(self, address_tuple):
+ self.address_tuple = address_tuple
+ self.active_connections = 0
+
+ @property
+ def address(self):
+ return self.address_tuple
+
+ def add_connection(self):
+ self.active_connections += 1
+
+ def drop_connection(self):
+ self.active_connections -= 1
+
+ @property
+ def connections(self):
+ return self.active_connections
+
+ @property
+ def host(self):
+ return self.address_tuple[0]
+
+ @property
+ def port(self):
+ return self.address_tuple[1]
+
diff --git a/mantrid/cli.py b/mantrid/cli.py
index f730866..9bf62b5 100644
--- a/mantrid/cli.py
+++ b/mantrid/cli.py
@@ -1,5 +1,7 @@
import sys
-from .client import MantridClient
+
+from mantrid.backend import Backend
+from mantrid.client import MantridClient
class MantridCli(object):
@@ -47,11 +49,12 @@ def action_list(self):
print format % ("HOST", "ACTION", "SUBDOMS")
for host, details in sorted(self.client.get_all().items()):
if details[0] in ("proxy", "mirror"):
- action = "%s<%s>" % (
+ action = "%s[%s]<%s>" % (
details[0],
+ details[1]['algorithm'],
",".join(
- "%s:%s" % (host, port)
- for host, port in details[1]['backends']
+ "%s:%s" % (backend.host, backend.port)
+ for backend in details[1]['backends']
)
)
elif details[0] == "static":
@@ -114,7 +117,7 @@ def action_set(self, hostname=None, action=None, subdoms=None, *args):
# Expand some options from text to datastructure
if "backends" in options:
options['backends'] = [
- (lambda x: (x[0], int(x[1])))(bit.split(":", 1))
+ Backend((lambda x: (x[0], int(x[1])))(bit.split(":", 1)))
for bit in options['backends'].split(",")
]
# Set!
diff --git a/mantrid/client.py b/mantrid/client.py
index ea7f869..8e72386 100644
--- a/mantrid/client.py
+++ b/mantrid/client.py
@@ -3,8 +3,8 @@
httplib2 = eventlet.import_patched("httplib2")
except ImportError:
import httplib2
-import json
+import mantrid.json
class MantridClient(object):
"""
@@ -20,10 +20,10 @@ def _request(self, path, method, body=None):
resp, content = h.request(
self.base_url + path,
method,
- body = json.dumps(body),
+ body = mantrid.json.dump(body),
)
if resp['status'] == "200":
- return json.loads(content)
+ return mantrid.json.load(content)
else:
raise IOError(
"Got %s reponse from server (%s)" % (
diff --git a/mantrid/json.py b/mantrid/json.py
new file mode 100644
index 0000000..20e0cbb
--- /dev/null
+++ b/mantrid/json.py
@@ -0,0 +1,46 @@
+"""JSON helper that defaults to secure (loads, dumps) methods and
+supports custom mantrid data types.
+"""
+
+from __future__ import absolute_import
+
+import copy
+import json
+
+import mantrid.backend
+
+
+class MantridEncoder(json.JSONEncoder):
+ """Custom serialization for mantrid types."""
+ def default(self, obj):
+ if isinstance(obj, mantrid.backend.Backend):
+ return {'__backend__': (obj.host, obj.port)}
+ return json.JSONEncoder.default(self, obj)
+
+def load_mantrid(dct):
+ """Custom deserialization for mantrid types."""
+ if '__backend__' in dct:
+ return mantrid.backend.Backend(dct['__backend__'])
+ return dct
+
+
+def dumps(*args, **kwargs):
+ """Securely dump objects to JSON, supporting custom mantrid types."""
+ new_kwargs = copy.copy(kwargs)
+ new_kwargs['cls'] = MantridEncoder
+ return json.dumps(*args, **new_kwargs)
+
+def dump(*args, **kwargs):
+ """Securely dump objects to JSON, supporting custom mantrid types."""
+ return dumps(*args, **kwargs)
+
+def loads(*args, **kwargs):
+ """Securely load objects from JSON, supporting custom mantrid types."""
+ new_kwargs = copy.copy(kwargs)
+ new_kwargs['object_hook'] = load_mantrid
+ return json.loads(*args, **new_kwargs)
+
+def load(*args, **kwargs):
+ """Securely load objects from JSON, supporting custom mantrid types."""
+ return loads(*args, **kwargs)
+
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index f3d5e94..b64ee7c 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -4,17 +4,20 @@
import traceback
import mimetools
import resource
-import json
import os
import sys
import argparse
+
from eventlet import wsgi
from eventlet.green import socket
-from .actions import Unknown, Proxy, Empty, Static, Redirect, NoHosts, Spin
-from .config import SimpleConfig
-from .management import ManagementApp
-from .stats_socket import StatsSocket
-from .greenbody import GreenBody
+
+import mantrid.json
+
+from mantrid.actions import Unknown, Proxy, Empty, Static, Redirect, NoHosts, Spin
+from mantrid.config import SimpleConfig
+from mantrid.management import ManagementApp
+from mantrid.stats_socket import StatsSocket
+from mantrid.greenbody import GreenBody
class Balancer(object):
@@ -103,7 +106,7 @@ def load(self):
if os.path.getsize(self.state_file) <= 1:
raise IOError("File is empty.")
with open(self.state_file) as fh:
- state = json.load(fh)
+ state = mantrid.json.load(fh)
assert isinstance(state, dict)
self.hosts = state['hosts']
self.stats = state['stats']
@@ -117,7 +120,7 @@ def load(self):
def save(self):
"Saves the state to the state file"
with open(self.state_file, "w") as fh:
- json.dump({
+ mantrid.json.dump({
"hosts": self.hosts,
"stats": self.stats,
}, fh)
diff --git a/mantrid/management.py b/mantrid/management.py
index 80336a6..d8b26f1 100644
--- a/mantrid/management.py
+++ b/mantrid/management.py
@@ -1,6 +1,7 @@
-import json
import re
+import mantrid.json
+
class HttpNotFound(Exception):
"Exception raised to pass on a 404 error."
@@ -43,21 +44,21 @@ def handle(self, environ, start_response):
# Handle errors
except HttpNotFound:
start_response('404 Not Found', [('Content-Type', 'application/json')])
- return [json.dumps({"error": "not_found"})]
+ return [mantrid.json.dump({"error": "not_found"})]
except HttpMethodNotAllowed:
start_response('405 Method Not Allowed', [('Content-Type', 'application/json')])
- return [json.dumps({"error": "method_not_allowed"})]
+ return [mantrid.json.dump({"error": "method_not_allowed"})]
# Dispatch to the named method
body = environ['wsgi.input'].read()
if body:
- body = json.loads(body)
+ body = mantrid.json.load(body)
response = handler(
environ['PATH_INFO'].lower(),
body,
)
# Send the response
start_response('200 OK', [('Content-Type', 'application/json')])
- return [json.dumps(response)]
+ return [mantrid.json.dump(response)]
def route(self, path, method):
# Simple routing for paths
From 97103b4ff53ccc48d41007ab52662d56917aeb28 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Thu, 6 Dec 2012 13:37:25 +0100
Subject: [PATCH 05/47] Removed a nonsense comment.
---
mantrid/actions.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 86fb1a8..e3c5574 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -150,7 +150,6 @@ def least_connections(self):
return min(self.backends, key=operator.attrgetter('connections'))
def handle(self, sock, read_data, path, headers):
- "Sends back a static error page."
for i in range(self.attempts):
try:
backend = self.select_backend()
From 27f171de1154d0cfc6a4d487f6aee8d39fcb3892 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Thu, 6 Dec 2012 13:38:46 +0100
Subject: [PATCH 06/47] Reverted my previous "cleanup" changes.
I managed to confuse JSON and YAML APIs there...
---
mantrid/client.py | 4 ++--
mantrid/json.py | 8 ++++++--
mantrid/management.py | 8 ++++----
3 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/mantrid/client.py b/mantrid/client.py
index 8e72386..56f06a4 100644
--- a/mantrid/client.py
+++ b/mantrid/client.py
@@ -20,10 +20,10 @@ def _request(self, path, method, body=None):
resp, content = h.request(
self.base_url + path,
method,
- body = mantrid.json.dump(body),
+ body = mantrid.json.dumps(body),
)
if resp['status'] == "200":
- return mantrid.json.load(content)
+ return mantrid.json.loads(content)
else:
raise IOError(
"Got %s reponse from server (%s)" % (
diff --git a/mantrid/json.py b/mantrid/json.py
index 20e0cbb..0679060 100644
--- a/mantrid/json.py
+++ b/mantrid/json.py
@@ -32,7 +32,9 @@ def dumps(*args, **kwargs):
def dump(*args, **kwargs):
"""Securely dump objects to JSON, supporting custom mantrid types."""
- return dumps(*args, **kwargs)
+ new_kwargs = copy.copy(kwargs)
+ new_kwargs['cls'] = MantridEncoder
+ return json.dump(*args, **new_kwargs)
def loads(*args, **kwargs):
"""Securely load objects from JSON, supporting custom mantrid types."""
@@ -42,5 +44,7 @@ def loads(*args, **kwargs):
def load(*args, **kwargs):
"""Securely load objects from JSON, supporting custom mantrid types."""
- return loads(*args, **kwargs)
+ new_kwargs = copy.copy(kwargs)
+ new_kwargs['object_hook'] = load_mantrid
+ return json.load(*args, **new_kwargs)
diff --git a/mantrid/management.py b/mantrid/management.py
index d8b26f1..05f243c 100644
--- a/mantrid/management.py
+++ b/mantrid/management.py
@@ -44,21 +44,21 @@ def handle(self, environ, start_response):
# Handle errors
except HttpNotFound:
start_response('404 Not Found', [('Content-Type', 'application/json')])
- return [mantrid.json.dump({"error": "not_found"})]
+ return [mantrid.json.dumps({"error": "not_found"})]
except HttpMethodNotAllowed:
start_response('405 Method Not Allowed', [('Content-Type', 'application/json')])
- return [mantrid.json.dump({"error": "method_not_allowed"})]
+ return [mantrid.json.dumps({"error": "method_not_allowed"})]
# Dispatch to the named method
body = environ['wsgi.input'].read()
if body:
- body = mantrid.json.load(body)
+ body = mantrid.json.loads(body)
response = handler(
environ['PATH_INFO'].lower(),
body,
)
# Send the response
start_response('200 OK', [('Content-Type', 'application/json')])
- return [mantrid.json.dump(response)]
+ return [mantrid.json.dumps(response)]
def route(self, path, method):
# Simple routing for paths
From 490310e094409044116aa3eb0c62cd37b1161409 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Thu, 6 Dec 2012 15:59:26 +0100
Subject: [PATCH 07/47] First cut at health-checking. Needed to fix
least-connections balancing.
Without it a host that rejects connections would always
get selected as the best candidate for the incoming request.
Also, this code really needs more abstraction. :-)
---
mantrid/actions.py | 17 ++++++++++++----
mantrid/backend.py | 33 +++++++++++++++++++++++++++++++
mantrid/cli.py | 13 +++++++++++--
mantrid/loadbalancer.py | 43 ++++++++++++++++++++++++++++++++++++++++-
4 files changed, 99 insertions(+), 7 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index e3c5574..c099f14 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -3,6 +3,7 @@
"""
import errno
+import logging
import operator
import os
import random
@@ -132,10 +133,11 @@ class Proxy(Action):
attempts = 1
delay = 1
- def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=None, algorithm='random'):
+ def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=None, algorithm='random', healthcheck=False):
super(Proxy, self).__init__(balancer, host, matched_host)
self.backends = backends
- self.algorithm = 'random'
+ self.algorithm = algorithm
+ self.healthcheck = healthcheck
self.select_backend = self.least_connections if algorithm == 'least_connections' else self.random
assert self.backends
if attempts is not None:
@@ -143,11 +145,15 @@ def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=
if delay is not None:
self.delay = float(delay)
+
+ def valid_backends(self):
+ return [b for b in self.backends if not b.blacklisted or not self.healthcheck]
+
def random(self):
- return random.choice(self.backends)
+ return random.choice(self.valid_backends())
def least_connections(self):
- return min(self.backends, key=operator.attrgetter('connections'))
+ return min(self.valid_backends(), key=operator.attrgetter('connections'))
def handle(self, sock, read_data, path, headers):
for i in range(self.attempts):
@@ -156,6 +162,9 @@ def handle(self, sock, read_data, path, headers):
server_sock = eventlet.connect((backend.host, backend.port))
backend.add_connection()
except socket.error:
+ if self.healthcheck and not backend.blacklisted:
+ logging.warn("Blacklisting backend %s", backend)
+ backend.blacklisted = True
eventlet.sleep(self.delay)
continue
diff --git a/mantrid/backend.py b/mantrid/backend.py
index 448ad26..a714a82 100644
--- a/mantrid/backend.py
+++ b/mantrid/backend.py
@@ -1,8 +1,17 @@
+import eventlet
+import logging
+
+from eventlet.green import socket
class Backend(object):
+
+ health_check_delay_seconds = 1
+
def __init__(self, address_tuple):
self.address_tuple = address_tuple
self.active_connections = 0
+ self.blacklisted = False
+ self.retired = False
@property
def address(self):
@@ -26,3 +35,27 @@ def host(self):
def port(self):
return self.address_tuple[1]
+ def __repr__(self):
+ return "Backend((%s, %s))" % (self.host, self.port)
+
+ def start_health_check(self):
+ eventlet.spawn(self._health_check_loop)
+
+ def _health_check_loop(self):
+ while True:
+ if self.retired:
+ logging.warn("Stopping health-checking of %s", self)
+ break
+
+ logging.info("Checking health of %s", self)
+ try:
+ socket = eventlet.connect((self.host, self.port))
+ logging.info("%s is alive, making sure it is not blacklisted", self)
+ self.blacklisted = False
+ socket.close()
+ except:
+ logging.info("%s seems dead, will check again later", self)
+ pass
+
+ eventlet.sleep(self.health_check_delay_seconds)
+
diff --git a/mantrid/cli.py b/mantrid/cli.py
index 9bf62b5..ac92e9d 100644
--- a/mantrid/cli.py
+++ b/mantrid/cli.py
@@ -49,9 +49,10 @@ def action_list(self):
print format % ("HOST", "ACTION", "SUBDOMS")
for host, details in sorted(self.client.get_all().items()):
if details[0] in ("proxy", "mirror"):
- action = "%s[%s]<%s>" % (
+ action = "%s[algorithm=%s,healthcheck=%s]<%s>" % (
details[0],
- details[1]['algorithm'],
+ details[1].get('algorithm', 'default'),
+ details[1].get('healthcheck', False),
",".join(
"%s:%s" % (backend.host, backend.port)
for backend in details[1]['backends']
@@ -105,6 +106,9 @@ def action_set(self, hostname=None, action=None, subdoms=None, *args):
if action in ("proxy, mirror") and "backends" not in options:
sys.stderr.write("The %s action requires a backends option.\n" % action)
sys.exit(1)
+ if "healthcheck" in options and options["healthcheck"].lower() not in ("true", "false"):
+ sys.stderr.write("The healthcheck option must be one of (true, false)")
+ sys.exit(1)
if action == "static" and "type" not in options:
sys.stderr.write("The %s action requires a type option.\n" % action)
sys.exit(1)
@@ -120,6 +124,8 @@ def action_set(self, hostname=None, action=None, subdoms=None, *args):
Backend((lambda x: (x[0], int(x[1])))(bit.split(":", 1)))
for bit in options['backends'].split(",")
]
+ if "healthcheck" in options:
+ options['healthcheck'] = (options['healthcheck'].lower() == "true")
# Set!
self.client.set(
hostname,
@@ -144,3 +150,6 @@ def action_stats(self, hostname=None):
details.get("bytes_received", 0),
details.get("bytes_sent", 0),
)
+
+if __name__ == "__main__":
+ MantridCli.main()
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 84df54d..456bf84 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -20,6 +20,36 @@
from mantrid.greenbody import GreenBody
+class ManagedHostDict(dict):
+ def __init__(self, *args, **kwargs):
+ super(ManagedHostDict, self).__init__(*args, **kwargs)
+ for host in self:
+ if self[host][1].get('healthcheck', False):
+ self._start_health_check_of(host)
+
+ def __setitem__(self, host, settings):
+ if host in self:
+ self._retire_backends_of(host)
+
+ super(ManagedHostDict, self).__setitem__(host, settings)
+
+ if settings[1].get('healthcheck', False):
+ self._start_health_check_of(host)
+
+ def __delitem__(self, host):
+ if host in self and self[host][1].get('healthcheck', False):
+ self._retire_backends_of(host)
+ super(ManagedHostDict, self).__delitem__(host)
+
+ def _retire_backends_of(self, host):
+ for backend in self[host][1].get("backends", []):
+ backend.retired = True
+
+ def _start_health_check_of(self, host):
+ for backend in self[host][1].get("backends", []):
+ backend.start_health_check()
+
+
class Balancer(object):
"""
Main loadbalancer class.
@@ -54,6 +84,7 @@ def __init__(self, external_addresses, internal_addresses, management_addresses,
self.uid = uid
self.gid = gid
self.static_dir = static_dir
+ self.hosts = ManagedHostDict()
@classmethod
def main(cls):
@@ -114,7 +145,7 @@ def load(self):
self.stats[key]['open_requests'] = 0
except (IOError, OSError):
# There is no state file; start empty.
- self.hosts = {}
+ self.hosts = ManagedHostDict()
self.stats = {}
def save(self):
@@ -352,5 +383,15 @@ def handle(self, sock, address, internal=False):
except:
logging.error(traceback.format_exc())
+ def _set_hosts(self, hosts):
+ self.__dict__['hosts'] = ManagedHostDict(hosts)
+
+ def _get_hosts(self):
+ return self.__dict__['hosts']
+
+ hosts = property(fget=_get_hosts, fset=_set_hosts)
+
+
if __name__ == "__main__":
Balancer.main()
+
From 71f70abbc6c68063bac777a380b75e88c0353dc7 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Thu, 6 Dec 2012 17:55:36 +0100
Subject: [PATCH 08/47] Fixed the problem with save_loop dying for good.
---
mantrid/loadbalancer.py | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 456bf84..fda561a 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -232,11 +232,14 @@ def save_loop(self):
"""
last_hash = hash(repr(self.hosts))
while self.running:
- eventlet.sleep(self.save_interval)
- next_hash = hash(repr(self.hosts))
- if next_hash != last_hash:
- self.save()
- last_hash = next_hash
+ try:
+ eventlet.sleep(self.save_interval)
+ next_hash = hash(repr(self.hosts))
+ if next_hash != last_hash:
+ self.save()
+ last_hash = next_hash
+ except:
+ logging.error("Failed to save state", exc_info=True)
def management_loop(self, address, family):
"""
From 4a47cbccc533806de107eaeb2b6937bc3c5a725c Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Thu, 6 Dec 2012 17:56:48 +0100
Subject: [PATCH 09/47] Defer marking the backend as free until transfers
finished.
---
mantrid/actions.py | 4 +---
mantrid/socketmeld.py | 8 +++++++-
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index c099f14..ab9f15d 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -175,13 +175,11 @@ def send_onwards(data):
try:
size = send_onwards(read_data)
- size += SocketMelder(sock, server_sock).run()
+ size += SocketMelder(sock, server_sock).run(on_finish=backend.drop_connection)
break
except socket.error, e:
if e.errno != errno.EPIPE:
raise
- finally:
- backend.drop_connection()
class Spin(Action):
diff --git a/mantrid/socketmeld.py b/mantrid/socketmeld.py
index 7a51c08..7aa8bea 100644
--- a/mantrid/socketmeld.py
+++ b/mantrid/socketmeld.py
@@ -32,7 +32,13 @@ def piper(self, in_sock, out_sock, out_addr, onkill):
except greenlet.GreenletExit:
return
- def run(self):
+ def run(self, on_finish=lambda: None):
+ try:
+ return self._run()
+ finally:
+ on_finish()
+
+ def _run(self):
self.threads = {
"ctos": eventlet.spawn(self.piper, self.server, self.client, "client", "stoc"),
"stoc": eventlet.spawn(self.piper, self.client, self.server, "server", "ctos"),
From f082cd4c94dc8a0ad2f0fc186d82fbba3c6db26a Mon Sep 17 00:00:00 2001
From: Marcin Sawicki <odcinek@gmail.com>
Date: Thu, 6 Dec 2012 18:03:50 +0100
Subject: [PATCH 10/47] Does not really improve anything. Revert "Defer marking
the backend as free until transfers finished."
This reverts commit 4a47cbccc533806de107eaeb2b6937bc3c5a725c.
---
mantrid/actions.py | 4 +++-
mantrid/socketmeld.py | 8 +-------
2 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index ab9f15d..c099f14 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -175,11 +175,13 @@ def send_onwards(data):
try:
size = send_onwards(read_data)
- size += SocketMelder(sock, server_sock).run(on_finish=backend.drop_connection)
+ size += SocketMelder(sock, server_sock).run()
break
except socket.error, e:
if e.errno != errno.EPIPE:
raise
+ finally:
+ backend.drop_connection()
class Spin(Action):
diff --git a/mantrid/socketmeld.py b/mantrid/socketmeld.py
index 7aa8bea..7a51c08 100644
--- a/mantrid/socketmeld.py
+++ b/mantrid/socketmeld.py
@@ -32,13 +32,7 @@ def piper(self, in_sock, out_sock, out_addr, onkill):
except greenlet.GreenletExit:
return
- def run(self, on_finish=lambda: None):
- try:
- return self._run()
- finally:
- on_finish()
-
- def _run(self):
+ def run(self):
self.threads = {
"ctos": eventlet.spawn(self.piper, self.server, self.client, "client", "stoc"),
"stoc": eventlet.spawn(self.piper, self.client, self.server, "server", "ctos"),
From 412f1f9cb4df38b45525f63d09243d0e55bb799c Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Fri, 7 Dec 2012 12:14:40 +0100
Subject: [PATCH 11/47] Randomize backends in the least_connections algorithm.
---
mantrid/actions.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index c099f14..5e2b4a8 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -145,7 +145,6 @@ def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=
if delay is not None:
self.delay = float(delay)
-
def valid_backends(self):
return [b for b in self.backends if not b.blacklisted or not self.healthcheck]
@@ -153,7 +152,11 @@ def random(self):
return random.choice(self.valid_backends())
def least_connections(self):
- return min(self.valid_backends(), key=operator.attrgetter('connections'))
+ backends = self.valid_backends()
+ min_connections = min(b.connections for b in backends)
+
+ # this is possibly a little bit safer than always returning the first backend
+ return random.choice([b for b in backends if b.connections == min_connections])
def handle(self, sock, read_data, path, headers):
for i in range(self.attempts):
From c91dc861ffe1029d6ddc3b0d03578377bc2499ef Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Fri, 7 Dec 2012 13:51:03 +0100
Subject: [PATCH 12/47] Prevent the management loop from failing due to
exceptions.
---
mantrid/loadbalancer.py | 44 +++++++++++++++++++++++------------------
1 file changed, 25 insertions(+), 19 deletions(-)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index fda561a..2bc0afe 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -245,25 +245,31 @@ def management_loop(self, address, family):
"""
Accepts management requests.
"""
- try:
- sock = eventlet.listen(address, family)
- except socket.error, e:
- logging.critical("Cannot listen on (%s, %s): %s" % (address, family, e))
- return
- # Sleep to ensure we've dropped privileges by the time we start serving
- eventlet.sleep(0.5)
- # Actually serve management
- logging.info("Listening for management on %s" % (address, ))
- management_app = ManagementApp(self)
- try:
- with open("/dev/null", "w") as log_dest:
- wsgi.server(
- sock,
- management_app.handle,
- log = log_dest,
- )
- finally:
- sock.close()
+ while True:
+ try:
+ try:
+ sock = eventlet.listen(address, family)
+ except socket.error, e:
+ logging.critical("Cannot listen on (%s, %s): %s" % (address, family, e))
+ return
+ # Sleep to ensure we've dropped privileges by the time we start serving
+ eventlet.sleep(0.5)
+ # Actually serve management
+ logging.info("Listening for management on %s" % (address, ))
+ management_app = ManagementApp(self)
+ try:
+ with open("/dev/null", "w") as log_dest:
+ wsgi.server(
+ sock,
+ management_app.handle,
+ log = log_dest,
+ )
+ finally:
+ sock.close()
+ except:
+ logging.error("Management loop failed with exception", exc_info=True)
+ # don't let it spin too fast
+ eventlet.sleep(1)
### Client handling ###
From 08ba7c3821c3f895a69a996a7e6d91537c7fb63e Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Mon, 10 Dec 2012 17:09:01 +0100
Subject: [PATCH 13/47] Made least_connections and healthcheck the defaults.
---
mantrid/actions.py | 6 ++++--
mantrid/cli.py | 5 +++--
mantrid/loadbalancer.py | 6 +++---
3 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 5e2b4a8..8a2e4fb 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -132,13 +132,15 @@ class Proxy(Action):
attempts = 1
delay = 1
+ default_healthcheck = True
+ default_algorithm = "least_connections"
- def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=None, algorithm='random', healthcheck=False):
+ def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=None, algorithm=default_algorithm, healthcheck=default_healthcheck):
super(Proxy, self).__init__(balancer, host, matched_host)
self.backends = backends
self.algorithm = algorithm
self.healthcheck = healthcheck
- self.select_backend = self.least_connections if algorithm == 'least_connections' else self.random
+ self.select_backend = self.random if algorithm == 'random' else self.least_connections
assert self.backends
if attempts is not None:
self.attempts = int(attempts)
diff --git a/mantrid/cli.py b/mantrid/cli.py
index ac92e9d..d0535b0 100644
--- a/mantrid/cli.py
+++ b/mantrid/cli.py
@@ -1,5 +1,6 @@
import sys
+from mantrid.actions import Proxy
from mantrid.backend import Backend
from mantrid.client import MantridClient
@@ -51,8 +52,8 @@ def action_list(self):
if details[0] in ("proxy", "mirror"):
action = "%s[algorithm=%s,healthcheck=%s]<%s>" % (
details[0],
- details[1].get('algorithm', 'default'),
- details[1].get('healthcheck', False),
+ details[1].get('algorithm', Proxy.default_algorithm),
+ details[1].get('healthcheck', Proxy.default_healthcheck),
",".join(
"%s:%s" % (backend.host, backend.port)
for backend in details[1]['backends']
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 2bc0afe..4360812 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -24,7 +24,7 @@ class ManagedHostDict(dict):
def __init__(self, *args, **kwargs):
super(ManagedHostDict, self).__init__(*args, **kwargs)
for host in self:
- if self[host][1].get('healthcheck', False):
+ if self[host][1].get('healthcheck', Proxy.default_healthcheck):
self._start_health_check_of(host)
def __setitem__(self, host, settings):
@@ -33,11 +33,11 @@ def __setitem__(self, host, settings):
super(ManagedHostDict, self).__setitem__(host, settings)
- if settings[1].get('healthcheck', False):
+ if settings[1].get('healthcheck', Proxy.default_healthcheck):
self._start_health_check_of(host)
def __delitem__(self, host):
- if host in self and self[host][1].get('healthcheck', False):
+ if host in self and self[host][1].get('healthcheck', Proxy.default_healthcheck):
self._retire_backends_of(host)
super(ManagedHostDict, self).__delitem__(host)
From f49ebd918447a7c1b182fca7eee6b26bc3d63e8b Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Tue, 11 Dec 2012 17:46:07 +0100
Subject: [PATCH 14/47] Adding support for the old state format loading.
---
mantrid/loadbalancer.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 4360812..0e0c1cb 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -131,13 +131,22 @@ def main(cls):
)
balancer.run()
+ def _converted_from_old_format(self, objtree):
+ hosts = objtree['hosts']
+ for host, settings in hosts.items():
+ backends = settings[1]['backends']
+ if backends and not isinstance(backends[0], mantrid.backend.Backend):
+ new_backends = map(mantrid.backend.Backend, backends)
+ settings[1]['backends'] = new_backends
+ return objtree
+
def load(self):
"Loads the state from the state file"
try:
if os.path.getsize(self.state_file) <= 1:
raise IOError("File is empty.")
with open(self.state_file) as fh:
- state = mantrid.json.load(fh)
+ state = self._converted_from_old_format(mantrid.json.load(fh))
assert isinstance(state, dict)
self.hosts = state['hosts']
self.stats = state['stats']
From 927d15dc19a358c9f1bf96e27e5e50b6fe1b39f0 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@futuresimple.com>
Date: Tue, 11 Dec 2012 17:51:09 +0100
Subject: [PATCH 15/47] Changed log level of healthcheck messages.
---
mantrid/backend.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/mantrid/backend.py b/mantrid/backend.py
index a714a82..0048a5a 100644
--- a/mantrid/backend.py
+++ b/mantrid/backend.py
@@ -47,14 +47,14 @@ def _health_check_loop(self):
logging.warn("Stopping health-checking of %s", self)
break
- logging.info("Checking health of %s", self)
+ logging.debug("Checking health of %s", self)
try:
socket = eventlet.connect((self.host, self.port))
- logging.info("%s is alive, making sure it is not blacklisted", self)
+ logging.debug("%s is alive, making sure it is not blacklisted", self)
self.blacklisted = False
socket.close()
except:
- logging.info("%s seems dead, will check again later", self)
+ logging.debug("%s seems dead, will check again later", self)
pass
eventlet.sleep(self.health_check_delay_seconds)
From 3194808e6ddd0fe80526e1a86dc529520326e463 Mon Sep 17 00:00:00 2001
From: Mateusz Zawisza <mateusz.zawisza@gmail.com>
Date: Tue, 8 Oct 2013 13:36:26 +0200
Subject: [PATCH 16/47] improves logging when no backends found
---
mantrid/actions.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 8a2e4fb..ef2129c 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -137,6 +137,7 @@ class Proxy(Action):
def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=None, algorithm=default_algorithm, healthcheck=default_healthcheck):
super(Proxy, self).__init__(balancer, host, matched_host)
+ self.host = host
self.backends = backends
self.algorithm = algorithm
self.healthcheck = healthcheck
@@ -155,6 +156,9 @@ def random(self):
def least_connections(self):
backends = self.valid_backends()
+ if len(backends) == 0:
+ logging.warn("No healthy backends for host: %s!", self.host)
+
min_connections = min(b.connections for b in backends)
# this is possibly a little bit safer than always returning the first backend
From b3029b880abfcd4090fd9b2c8de87b5a514edca9 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 5 Jun 2014 02:21:44 +0200
Subject: [PATCH 17/47] Start healthchecks once connection attempt fails
Previously all backends would have healthcheck threads
started immediately. This was mostly useless, as the healthcheck
logic would never mark the backend down - it could only mark it
as back up once it's been marked down my the load-balancing logic.
This generated substantial amounts of traffic.
Now the healthcheck loop should start when the backend is
blacklisted by the load balancing logic and stop once its found
to be back up.
---
mantrid/backend.py | 35 +++++++++++++++++++++++------------
mantrid/loadbalancer.py | 11 -----------
2 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/mantrid/backend.py b/mantrid/backend.py
index 0048a5a..0bdd70a 100644
--- a/mantrid/backend.py
+++ b/mantrid/backend.py
@@ -10,9 +10,19 @@ class Backend(object):
def __init__(self, address_tuple):
self.address_tuple = address_tuple
self.active_connections = 0
- self.blacklisted = False
+ self._blacklisted = False
self.retired = False
+ @property
+ def blacklisted(self):
+ return self._blacklisted
+
+ @blacklisted.setter
+ def blacklisted(self, value):
+ if value:
+ self.start_health_check()
+ self._blacklisted = value
+
@property
def address(self):
return self.address_tuple
@@ -43,19 +53,20 @@ def start_health_check(self):
def _health_check_loop(self):
while True:
- if self.retired:
+ if self.retired or not self.blacklisted:
logging.warn("Stopping health-checking of %s", self)
break
- logging.debug("Checking health of %s", self)
- try:
- socket = eventlet.connect((self.host, self.port))
- logging.debug("%s is alive, making sure it is not blacklisted", self)
- self.blacklisted = False
- socket.close()
- except:
- logging.debug("%s seems dead, will check again later", self)
- pass
-
+ self._check_health()
eventlet.sleep(self.health_check_delay_seconds)
+ def _check_health(self):
+ logging.debug("Checking health of %s", self)
+ try:
+ socket = eventlet.connect((self.host, self.port))
+ logging.debug("%s is alive, making sure it is not blacklisted", self)
+ self.blacklisted = False
+ socket.close()
+ except:
+ logging.debug("%s seems dead, will check again later", self)
+
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 0e0c1cb..0d24f78 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -23,9 +23,6 @@
class ManagedHostDict(dict):
def __init__(self, *args, **kwargs):
super(ManagedHostDict, self).__init__(*args, **kwargs)
- for host in self:
- if self[host][1].get('healthcheck', Proxy.default_healthcheck):
- self._start_health_check_of(host)
def __setitem__(self, host, settings):
if host in self:
@@ -33,9 +30,6 @@ def __setitem__(self, host, settings):
super(ManagedHostDict, self).__setitem__(host, settings)
- if settings[1].get('healthcheck', Proxy.default_healthcheck):
- self._start_health_check_of(host)
-
def __delitem__(self, host):
if host in self and self[host][1].get('healthcheck', Proxy.default_healthcheck):
self._retire_backends_of(host)
@@ -45,11 +39,6 @@ def _retire_backends_of(self, host):
for backend in self[host][1].get("backends", []):
backend.retired = True
- def _start_health_check_of(self, host):
- for backend in self[host][1].get("backends", []):
- backend.start_health_check()
-
-
class Balancer(object):
"""
Main loadbalancer class.
From 7cb11bc22cf59966f12a134196703e846213c291 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 5 Jun 2014 02:24:38 +0200
Subject: [PATCH 18/47] Change the default number of connection attempts to 2
Also decreased the delay between consecutive attempts to 0.5s.
Fixed sleep logic so that the connection handler doesn't sleep
after the last connection attempt and returns immediately.
---
mantrid/actions.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index ef2129c..ac57b26 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -130,8 +130,8 @@ def handle(self, sock, read_data, path, headers):
class Proxy(Action):
"Proxies them through to a server. What loadbalancers do."
- attempts = 1
- delay = 1
+ attempts = 2
+ delay = 0.5
default_healthcheck = True
default_algorithm = "least_connections"
@@ -165,7 +165,7 @@ def least_connections(self):
return random.choice([b for b in backends if b.connections == min_connections])
def handle(self, sock, read_data, path, headers):
- for i in range(self.attempts):
+ for attempt in range(self.attempts):
try:
backend = self.select_backend()
server_sock = eventlet.connect((backend.host, backend.port))
@@ -174,7 +174,10 @@ def handle(self, sock, read_data, path, headers):
if self.healthcheck and not backend.blacklisted:
logging.warn("Blacklisting backend %s", backend)
backend.blacklisted = True
- eventlet.sleep(self.delay)
+
+ if attempt < self.attempts - 1:
+ eventlet.sleep(self.delay)
+
continue
# Function to help track data usage
From 2ae34aa2bdf0c936f70d26a8e3bba311dbace27e Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 5 Jun 2014 17:38:35 +0200
Subject: [PATCH 19/47] Revert "Change the default number of connection
attempts to 2"
This reverts commit 7cb11bc22cf59966f12a134196703e846213c291.
---
mantrid/actions.py | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index ac57b26..ef2129c 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -130,8 +130,8 @@ def handle(self, sock, read_data, path, headers):
class Proxy(Action):
"Proxies them through to a server. What loadbalancers do."
- attempts = 2
- delay = 0.5
+ attempts = 1
+ delay = 1
default_healthcheck = True
default_algorithm = "least_connections"
@@ -165,7 +165,7 @@ def least_connections(self):
return random.choice([b for b in backends if b.connections == min_connections])
def handle(self, sock, read_data, path, headers):
- for attempt in range(self.attempts):
+ for i in range(self.attempts):
try:
backend = self.select_backend()
server_sock = eventlet.connect((backend.host, backend.port))
@@ -174,10 +174,7 @@ def handle(self, sock, read_data, path, headers):
if self.healthcheck and not backend.blacklisted:
logging.warn("Blacklisting backend %s", backend)
backend.blacklisted = True
-
- if attempt < self.attempts - 1:
- eventlet.sleep(self.delay)
-
+ eventlet.sleep(self.delay)
continue
# Function to help track data usage
From 3a3b4d533568380b4ea203d0a425b9f345a2d69b Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 5 Jun 2014 19:02:39 +0200
Subject: [PATCH 20/47] Bump version.
---
mantrid/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/__init__.py b/mantrid/__init__.py
index 976498a..92192ee 100644
--- a/mantrid/__init__.py
+++ b/mantrid/__init__.py
@@ -1 +1 @@
-__version__ = "1.0.3"
+__version__ = "1.0.4"
From 70c166d58418048710de16c4108156b1a57ba7d2 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 5 Jun 2014 19:22:52 +0200
Subject: [PATCH 21/47] CHANGELOG update to test some pip installation
silyness.
---
CHANGELOG | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGELOG b/CHANGELOG
index 6338704..5c8f5ba 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,8 @@
+1.0.4
+-----
+
+ * Don't start healtchecks for every backend unless down
+
1.0.3
-----
From 2608e01171aff16a91a478cc5d7834be77b2f709 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 5 Jun 2014 02:21:44 +0200
Subject: [PATCH 22/47] Start healthchecks once connection attempt fails
Previously all backends would have healthcheck threads
started immediately. This was mostly useless, as the healthcheck
logic would never mark the backend down - it could only mark it
as back up once it's been marked down my the load-balancing logic.
This generated substantial amounts of traffic.
Now the healthcheck loop should start when the backend is
blacklisted by the load balancing logic and stop once its found
to be back up.
---
mantrid/backend.py | 35 +++++++++++++++++++++++------------
mantrid/loadbalancer.py | 11 -----------
2 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/mantrid/backend.py b/mantrid/backend.py
index 0048a5a..0bdd70a 100644
--- a/mantrid/backend.py
+++ b/mantrid/backend.py
@@ -10,9 +10,19 @@ class Backend(object):
def __init__(self, address_tuple):
self.address_tuple = address_tuple
self.active_connections = 0
- self.blacklisted = False
+ self._blacklisted = False
self.retired = False
+ @property
+ def blacklisted(self):
+ return self._blacklisted
+
+ @blacklisted.setter
+ def blacklisted(self, value):
+ if value:
+ self.start_health_check()
+ self._blacklisted = value
+
@property
def address(self):
return self.address_tuple
@@ -43,19 +53,20 @@ def start_health_check(self):
def _health_check_loop(self):
while True:
- if self.retired:
+ if self.retired or not self.blacklisted:
logging.warn("Stopping health-checking of %s", self)
break
- logging.debug("Checking health of %s", self)
- try:
- socket = eventlet.connect((self.host, self.port))
- logging.debug("%s is alive, making sure it is not blacklisted", self)
- self.blacklisted = False
- socket.close()
- except:
- logging.debug("%s seems dead, will check again later", self)
- pass
-
+ self._check_health()
eventlet.sleep(self.health_check_delay_seconds)
+ def _check_health(self):
+ logging.debug("Checking health of %s", self)
+ try:
+ socket = eventlet.connect((self.host, self.port))
+ logging.debug("%s is alive, making sure it is not blacklisted", self)
+ self.blacklisted = False
+ socket.close()
+ except:
+ logging.debug("%s seems dead, will check again later", self)
+
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 0e0c1cb..0d24f78 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -23,9 +23,6 @@
class ManagedHostDict(dict):
def __init__(self, *args, **kwargs):
super(ManagedHostDict, self).__init__(*args, **kwargs)
- for host in self:
- if self[host][1].get('healthcheck', Proxy.default_healthcheck):
- self._start_health_check_of(host)
def __setitem__(self, host, settings):
if host in self:
@@ -33,9 +30,6 @@ def __setitem__(self, host, settings):
super(ManagedHostDict, self).__setitem__(host, settings)
- if settings[1].get('healthcheck', Proxy.default_healthcheck):
- self._start_health_check_of(host)
-
def __delitem__(self, host):
if host in self and self[host][1].get('healthcheck', Proxy.default_healthcheck):
self._retire_backends_of(host)
@@ -45,11 +39,6 @@ def _retire_backends_of(self, host):
for backend in self[host][1].get("backends", []):
backend.retired = True
- def _start_health_check_of(self, host):
- for backend in self[host][1].get("backends", []):
- backend.start_health_check()
-
-
class Balancer(object):
"""
Main loadbalancer class.
From 8d2884a4362750f40519a45a641a0b6935c6914b Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 5 Jun 2014 19:02:39 +0200
Subject: [PATCH 23/47] Bump version.
---
mantrid/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/__init__.py b/mantrid/__init__.py
index 976498a..92192ee 100644
--- a/mantrid/__init__.py
+++ b/mantrid/__init__.py
@@ -1 +1 @@
-__version__ = "1.0.3"
+__version__ = "1.0.4"
From d342726d4a7096f893291ac1c57f4082f8c0e402 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 5 Jun 2014 19:22:52 +0200
Subject: [PATCH 24/47] CHANGELOG update to test some pip installation
silyness.
---
CHANGELOG | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGELOG b/CHANGELOG
index 6338704..5c8f5ba 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,8 @@
+1.0.4
+-----
+
+ * Don't start healtchecks for every backend unless down
+
1.0.3
-----
From c6843c3e09fd8d310ea6f773f60f0e35fb641e65 Mon Sep 17 00:00:00 2001
From: Mateusz Zawisza <mateusz.zawisza@gmail.com>
Date: Mon, 7 Jul 2014 12:47:34 +0200
Subject: [PATCH 25/47] sets eventlet version to always be 0.14.0
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 652a161..001c87a 100644
--- a/setup.py
+++ b/setup.py
@@ -40,6 +40,6 @@
install_requires = [
"httplib2",
"argparse",
- "eventlet>=0.9.16",
+ "eventlet==0.14.0",
],
)
From 0da650599129ef8dfd08d0e2c9bf7e3ca959eca0 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 10 Jul 2014 14:46:58 +0200
Subject: [PATCH 26/47] Add connection and request timeout handling
---
mantrid/actions.py | 27 +++++++++++++++++++------
mantrid/backend.py | 13 +++++++++---
mantrid/socketmeld.py | 46 +++++++++++++++++++++++++++++--------------
3 files changed, 62 insertions(+), 24 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index ef2129c..1bb601c 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -10,6 +10,7 @@
import eventlet
from eventlet.green import socket
+from eventlet.timeout import Timeout
from httplib import responses
from mantrid.backend import Backend
@@ -134,6 +135,7 @@ class Proxy(Action):
delay = 1
default_healthcheck = True
default_algorithm = "least_connections"
+ connection_timeout_seconds = 2
def __init__(self, balancer, host, matched_host, backends, attempts=None, delay=None, algorithm=default_algorithm, healthcheck=default_healthcheck):
super(Proxy, self).__init__(balancer, host, matched_host)
@@ -166,14 +168,22 @@ def least_connections(self):
def handle(self, sock, read_data, path, headers):
for i in range(self.attempts):
+ backend = self.select_backend()
try:
- backend = self.select_backend()
- server_sock = eventlet.connect((backend.host, backend.port))
- backend.add_connection()
+ timeout = Timeout(self.connection_timeout_seconds)
+ try:
+ server_sock = eventlet.connect((backend.host, backend.port))
+ backend.add_connection()
+ finally:
+ timeout.cancel()
except socket.error:
- if self.healthcheck and not backend.blacklisted:
- logging.warn("Blacklisting backend %s", backend)
- backend.blacklisted = True
+ logging.exception("Socket error on connect() to %s", backend)
+ self.blacklist(backend)
+ eventlet.sleep(self.delay)
+ continue
+ except:
+ logging.exception("Timeout on connect() to %s", backend)
+ self.blacklist(backend)
eventlet.sleep(self.delay)
continue
@@ -192,6 +202,11 @@ def send_onwards(data):
finally:
backend.drop_connection()
+ def blacklist(self, backend):
+ if self.healthcheck and not backend.blacklisted:
+ logging.warn("Blacklisting backend %s", backend)
+ backend.blacklisted = True
+
class Spin(Action):
"""
diff --git a/mantrid/backend.py b/mantrid/backend.py
index 0bdd70a..8566d29 100644
--- a/mantrid/backend.py
+++ b/mantrid/backend.py
@@ -2,10 +2,12 @@
import logging
from eventlet.green import socket
+from eventlet.timeout import Timeout
class Backend(object):
- health_check_delay_seconds = 1
+ healthcheck_delay_seconds = 1
+ healthcheck_timeout_seconds = 1
def __init__(self, address_tuple):
self.address_tuple = address_tuple
@@ -58,12 +60,17 @@ def _health_check_loop(self):
break
self._check_health()
- eventlet.sleep(self.health_check_delay_seconds)
+ eventlet.sleep(self.healthcheck_delay_seconds)
def _check_health(self):
logging.debug("Checking health of %s", self)
try:
- socket = eventlet.connect((self.host, self.port))
+ timeout = Timeout(self.healthcheck_timeout_seconds)
+ try:
+ socket = eventlet.connect((self.host, self.port))
+ finally:
+ timeout.cancel()
+
logging.debug("%s is alive, making sure it is not blacklisted", self)
self.blacklisted = False
socket.close()
diff --git a/mantrid/socketmeld.py b/mantrid/socketmeld.py
index 7a51c08..bfa6a55 100644
--- a/mantrid/socketmeld.py
+++ b/mantrid/socketmeld.py
@@ -1,6 +1,7 @@
import eventlet
import greenlet
from eventlet.green import socket
+from eventlet.timeout import Timeout
class SocketMelder(object):
@@ -8,6 +9,8 @@ class SocketMelder(object):
Takes two sockets and directly connects them together.
"""
+ transmission_timeout_seconds = 30
+
def __init__(self, client, server):
self.client = client
self.server = server
@@ -16,19 +19,23 @@ def __init__(self, client, server):
def piper(self, in_sock, out_sock, out_addr, onkill):
"Worker thread for data reading"
try:
- while True:
- written = in_sock.recv(32768)
- if not written:
+ timeout = Timeout(self.transmission_timeout_seconds)
+ try:
+ while True:
+ written = in_sock.recv(32768)
+ if not written:
+ try:
+ out_sock.shutdown(socket.SHUT_WR)
+ except socket.error:
+ self.threads[onkill].kill()
+ break
try:
- out_sock.shutdown(socket.SHUT_WR)
+ out_sock.sendall(written)
except socket.error:
- self.threads[onkill].kill()
- break
- try:
- out_sock.sendall(written)
- except socket.error:
- pass
- self.data_handled += len(written)
+ pass
+ self.data_handled += len(written)
+ finally:
+ timeout.cancel()
except greenlet.GreenletExit:
return
@@ -39,12 +46,21 @@ def run(self):
}
try:
self.threads['stoc'].wait()
- except (greenlet.GreenletExit, socket.error):
+ except (greenlet.GreenletExit, socket.error, Timeout):
pass
try:
self.threads['ctos'].wait()
- except (greenlet.GreenletExit, socket.error):
+ except (greenlet.GreenletExit, socket.error, Timeout):
pass
- self.server.close()
- self.client.close()
+
+ try:
+ self.server.close()
+ except:
+ logging.exception("Exception caught closing server socket")
+
+ try:
+ self.client.close()
+ except:
+ logging.exception("Exception caught closing client socket")
+
return self.data_handled
From 327ed5ad3f768e38630113310edee553927aab95 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 10 Jul 2014 15:23:46 +0200
Subject: [PATCH 27/47] Version bump
---
mantrid/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/__init__.py b/mantrid/__init__.py
index 92192ee..68cdeee 100644
--- a/mantrid/__init__.py
+++ b/mantrid/__init__.py
@@ -1 +1 @@
-__version__ = "1.0.4"
+__version__ = "1.0.5"
From 61e33e07f6e93486c6d0a1656f84699b9e7f5ec0 Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Thu, 10 Jul 2014 23:24:58 +0200
Subject: [PATCH 28/47] Better logging, fixed problem with missing import
---
mantrid/actions.py | 9 +++++----
mantrid/backend.py | 3 ++-
mantrid/socketmeld.py | 21 ++++++++++++++++-----
3 files changed, 23 insertions(+), 10 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 1bb601c..afc6bc4 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -173,16 +173,17 @@ def handle(self, sock, read_data, path, headers):
timeout = Timeout(self.connection_timeout_seconds)
try:
server_sock = eventlet.connect((backend.host, backend.port))
- backend.add_connection()
finally:
timeout.cancel()
+
+ backend.add_connection()
except socket.error:
- logging.exception("Socket error on connect() to %s", backend)
+ logging.exception("Socket error on connect() to %s of %s", backend, self.host)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
except:
- logging.exception("Timeout on connect() to %s", backend)
+ logging.warn("Timeout on connect() to %s of %s", backend, self.host)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
@@ -194,7 +195,7 @@ def send_onwards(data):
try:
size = send_onwards(read_data)
- size += SocketMelder(sock, server_sock).run()
+ size += SocketMelder(sock, server_sock, backend, self.host).run()
break
except socket.error, e:
if e.errno != errno.EPIPE:
diff --git a/mantrid/backend.py b/mantrid/backend.py
index 8566d29..1e78b7a 100644
--- a/mantrid/backend.py
+++ b/mantrid/backend.py
@@ -56,7 +56,8 @@ def start_health_check(self):
def _health_check_loop(self):
while True:
if self.retired or not self.blacklisted:
- logging.warn("Stopping health-checking of %s", self)
+ reason = "removing backend" if self.retired else "available"
+ logging.warn("Stopping health-checking of %s: %s", self, reason)
break
self._check_health()
diff --git a/mantrid/socketmeld.py b/mantrid/socketmeld.py
index bfa6a55..2b8248e 100644
--- a/mantrid/socketmeld.py
+++ b/mantrid/socketmeld.py
@@ -1,5 +1,8 @@
+import logging
+
import eventlet
import greenlet
+
from eventlet.green import socket
from eventlet.timeout import Timeout
@@ -11,9 +14,11 @@ class SocketMelder(object):
transmission_timeout_seconds = 30
- def __init__(self, client, server):
+ def __init__(self, client, server, backend, host):
self.client = client
self.server = server
+ self.backend = backend
+ self.host = host
self.data_handled = 0
def piper(self, in_sock, out_sock, out_addr, onkill):
@@ -38,29 +43,35 @@ def piper(self, in_sock, out_sock, out_addr, onkill):
timeout.cancel()
except greenlet.GreenletExit:
return
+ except Timeout:
+ logging.warn("Timeout serving request to backend %s of %s", self.backend, self.host)
+ return
def run(self):
+ # Two pipers == repeated logging of timeouts
self.threads = {
"ctos": eventlet.spawn(self.piper, self.server, self.client, "client", "stoc"),
"stoc": eventlet.spawn(self.piper, self.client, self.server, "server", "ctos"),
}
+
try:
self.threads['stoc'].wait()
- except (greenlet.GreenletExit, socket.error, Timeout):
+ except (greenlet.GreenletExit, socket.error):
pass
+
try:
self.threads['ctos'].wait()
- except (greenlet.GreenletExit, socket.error, Timeout):
+ except (greenlet.GreenletExit, socket.error):
pass
try:
self.server.close()
except:
- logging.exception("Exception caught closing server socket")
+ logging.exception("Exception caught closing server socket, backend %s of %s", self.backend, self.host)
try:
self.client.close()
except:
- logging.exception("Exception caught closing client socket")
+ logging.exception("Exception caught closing client socket, backend: %s of %s", self.backend, self.host)
return self.data_handled
From fd0ac553d6574d2f2f5da66e0dbbbf8334db92bc Mon Sep 17 00:00:00 2001
From: Karol Nowak <karol@getbase.com>
Date: Fri, 11 Jul 2014 00:05:56 +0200
Subject: [PATCH 29/47] Retry connection (not request!) if it fails
---
mantrid/actions.py | 33 ++++++++++++++++++---------------
1 file changed, 18 insertions(+), 15 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index afc6bc4..f8bc4b6 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -131,7 +131,7 @@ def handle(self, sock, read_data, path, headers):
class Proxy(Action):
"Proxies them through to a server. What loadbalancers do."
- attempts = 1
+ attempts = 2
delay = 1
default_healthcheck = True
default_algorithm = "least_connections"
@@ -167,7 +167,10 @@ def least_connections(self):
return random.choice([b for b in backends if b.connections == min_connections])
def handle(self, sock, read_data, path, headers):
- for i in range(self.attempts):
+ for attempt in range(self.attempts):
+ if attempt > 0:
+ logging.warn("Retrying connection for host %s", self.host)
+
backend = self.select_backend()
try:
timeout = Timeout(self.connection_timeout_seconds)
@@ -177,6 +180,7 @@ def handle(self, sock, read_data, path, headers):
timeout.cancel()
backend.add_connection()
+ break
except socket.error:
logging.exception("Socket error on connect() to %s of %s", backend, self.host)
self.blacklist(backend)
@@ -188,20 +192,19 @@ def handle(self, sock, read_data, path, headers):
eventlet.sleep(self.delay)
continue
- # Function to help track data usage
- def send_onwards(data):
- server_sock.sendall(data)
- return len(data)
+ # Function to help track data usage
+ def send_onwards(data):
+ server_sock.sendall(data)
+ return len(data)
- try:
- size = send_onwards(read_data)
- size += SocketMelder(sock, server_sock, backend, self.host).run()
- break
- except socket.error, e:
- if e.errno != errno.EPIPE:
- raise
- finally:
- backend.drop_connection()
+ try:
+ size = send_onwards(read_data)
+ size += SocketMelder(sock, server_sock, backend, self.host).run()
+ except socket.error, e:
+ if e.errno != errno.EPIPE:
+ raise
+ finally:
+ backend.drop_connection()
def blacklist(self, backend):
if self.healthcheck and not backend.blacklisted:
From fa189c42acd73952421834792d2369ffc1adf074 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miros=C5=82aw=20Naga=C5=9B?= <mgolden@smoczus.pl>
Date: Thu, 15 Jan 2015 11:01:55 +0100
Subject: [PATCH 30/47] Add support for X-Loadbalance-To header
---
mantrid/loadbalancer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 0d24f78..1e745a5 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -339,7 +339,7 @@ def handle(self, sock, address, internal=False):
headers = mimetools.Message(rfile, 0)
# Work out the host
try:
- host = headers['LoadBalanceTo']
+ host = headers['X-Loadbalance-To'] if 'X-Loadbalance-To' in headers else headers['LoadBalanceTo']
except KeyError:
host = "unknown"
headers['Connection'] = "close\r"
From 785dbb88c249e5b402a73cdf13c023cb1ff76244 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Pawe=C5=82=20G=C5=82azik?= <zytek@nuxi.pl>
Date: Mon, 16 Feb 2015 13:34:05 +0100
Subject: [PATCH 31/47] Version 1.0.6
---
mantrid/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/__init__.py b/mantrid/__init__.py
index 68cdeee..382021f 100644
--- a/mantrid/__init__.py
+++ b/mantrid/__init__.py
@@ -1 +1 @@
-__version__ = "1.0.5"
+__version__ = "1.0.6"
From 63d7426f960b32c142daa0e967b5b96f97446f44 Mon Sep 17 00:00:00 2001
From: Lukasz Siudut <lsiudut@gmail.com>
Date: Thu, 19 Feb 2015 10:33:31 +0100
Subject: [PATCH 32/47] Check whether there are healthy backend available
Currently when there are no backend available in pool Mantrid is
throwing general exception. We want to handle it in more civilized
way so nginx can catch a problem and fallback to other grid
clusters.
This fix is needed for az-separation, as when setting up new cluster
it will not have any backends set in mantrid. As workers connect
only to nginx->mantrid correlated to their grid, we don't want them
to throw errors and bother anyone, so let's just let nginx to
connect to other, healthy grid cluster.
---
mantrid/actions.py | 10 ++++++++--
mantrid/loadbalancer.py | 9 ++++++++-
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index f8bc4b6..6763221 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -16,6 +16,9 @@
from mantrid.backend import Backend
from mantrid.socketmeld import SocketMelder
+class NoHealthyBackends(Exception):
+ "Poll of usable backends is empty"
+ pass
class Action(object):
"Base action. Doesn't do anything."
@@ -160,8 +163,11 @@ def least_connections(self):
backends = self.valid_backends()
if len(backends) == 0:
logging.warn("No healthy backends for host: %s!", self.host)
-
- min_connections = min(b.connections for b in backends)
+
+ try:
+ min_connections = min(b.connections for b in backends)
+ except ValueError:
+ raise NoHealthyBackends()
# this is possibly a little bit safer than always returning the first backend
return random.choice([b for b in backends if b.connections == min_connections])
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 1e745a5..c9f150f 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -13,7 +13,7 @@
import mantrid.json
-from mantrid.actions import Unknown, Proxy, Empty, Static, Redirect, NoHosts, Spin
+from mantrid.actions import NoHealthyBackends, Unknown, Proxy, Empty, Static, Redirect, NoHosts, Spin
from mantrid.config import SimpleConfig
from mantrid.management import ManagementApp
from mantrid.stats_socket import StatsSocket
@@ -376,6 +376,13 @@ def handle(self, sock, address, internal=False):
except socket.error, e:
if e.errno not in (errno.EPIPE, errno.ETIMEDOUT, errno.ECONNRESET):
logging.error(traceback.format_exc())
+ except NoHealthyBackends, e:
+ logging.error("No healthy bakckends available!")
+ try:
+ sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
+ except socket.error, e:
+ if e.errno != errno.EPIPE:
+ raise
except:
logging.error(traceback.format_exc())
try:
From fd31d039a8a0c352c66b360c4256d0250bd50048 Mon Sep 17 00:00:00 2001
From: Lukasz Siudut <lsiudut@gmail.com>
Date: Thu, 19 Feb 2015 12:26:20 +0100
Subject: [PATCH 33/47] Changed to HTTP/503, as nginx doesn't support custom
code
---
examples/default.conf | 4 ++--
mantrid/loadbalancer.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/examples/default.conf b/examples/default.conf
index c95c551..5eb5855 100644
--- a/examples/default.conf
+++ b/examples/default.conf
@@ -1,7 +1,7 @@
# This configuration file is equivalent to the default Mantrid configuration.
-# Bind to all address on port 80 for untrusted connections
-bind = *:80
+# Bind to all address on port 8080 for untrusted connections
+bind = *:8080
# Don't bind to any addresses for trusted connections
# bind_internal = [::1]:81
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index c9f150f..a3cbc92 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -379,7 +379,7 @@ def handle(self, sock, address, internal=False):
except NoHealthyBackends, e:
logging.error("No healthy bakckends available!")
try:
- sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
+ sock.sendall("HTTP/1.0 503 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
except socket.error, e:
if e.errno != errno.EPIPE:
raise
From 219f2dbce5d7f23a7c4b4706eafe26eef21bacee Mon Sep 17 00:00:00 2001
From: Lukasz Siudut <lsiudut@gmail.com>
Date: Thu, 19 Feb 2015 14:07:03 +0100
Subject: [PATCH 34/47] Revert "Changed to HTTP/503, as nginx doesn't support
custom code"
This reverts commit fd31d039a8a0c352c66b360c4256d0250bd50048.
Apparantely applications returns 503 from time to time. In this case
we really don't want nginx to retry. Therefore we're returning to
idea of 597 and gonna patch nginx upstream module to support it, what
seems to be pretty straightforward.
---
examples/default.conf | 4 ++--
mantrid/loadbalancer.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/examples/default.conf b/examples/default.conf
index 5eb5855..c95c551 100644
--- a/examples/default.conf
+++ b/examples/default.conf
@@ -1,7 +1,7 @@
# This configuration file is equivalent to the default Mantrid configuration.
-# Bind to all address on port 8080 for untrusted connections
-bind = *:8080
+# Bind to all address on port 80 for untrusted connections
+bind = *:80
# Don't bind to any addresses for trusted connections
# bind_internal = [::1]:81
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index a3cbc92..c9f150f 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -379,7 +379,7 @@ def handle(self, sock, address, internal=False):
except NoHealthyBackends, e:
logging.error("No healthy bakckends available!")
try:
- sock.sendall("HTTP/1.0 503 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
+ sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
except socket.error, e:
if e.errno != errno.EPIPE:
raise
From dfefe00246cd8892170c02ca96bc88f53dad70f9 Mon Sep 17 00:00:00 2001
From: Lukasz Siudut <lsiudut@gmail.com>
Date: Thu, 19 Feb 2015 14:15:17 +0100
Subject: [PATCH 35/47] Logging cleanups
---
mantrid/actions.py | 2 --
mantrid/loadbalancer.py | 2 +-
2 files changed, 1 insertion(+), 3 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 6763221..62ae84e 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -161,8 +161,6 @@ def random(self):
def least_connections(self):
backends = self.valid_backends()
- if len(backends) == 0:
- logging.warn("No healthy backends for host: %s!", self.host)
try:
min_connections = min(b.connections for b in backends)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index c9f150f..f3644f2 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -377,7 +377,7 @@ def handle(self, sock, address, internal=False):
if e.errno not in (errno.EPIPE, errno.ETIMEDOUT, errno.ECONNRESET):
logging.error(traceback.format_exc())
except NoHealthyBackends, e:
- logging.error("No healthy bakckends available!")
+ logging.error("No healthy bakckends available for host '%s'" % host)
try:
sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
except socket.error, e:
From a2f143a41831837599f2d380c351aa775ad83ea6 Mon Sep 17 00:00:00 2001
From: Lukasz Siudut <lsiudut@gmail.com>
Date: Fri, 27 Feb 2015 00:28:46 +0100
Subject: [PATCH 36/47] Added support for aliases
Alias can be set for another hostname that is already configured.
It theoretically should help for problems with least_connections
alghoritm.
---
mantrid/actions.py | 17 +++++++++++++++++
mantrid/cli.py | 8 ++++++++
mantrid/loadbalancer.py | 12 +++++++-----
3 files changed, 32 insertions(+), 5 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 62ae84e..b99ed29 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -244,3 +244,20 @@ def handle(self, sock, read_data, path, headers):
# OK, nothing happened, so give up.
action = Static(self.balancer, self.host, self.matched_host, type="timeout")
return action.handle(sock, read_data, path, headers)
+
+class Alias(Action):
+ """
+ Alias for another backend
+ """
+ def __init__(self, balancer, host, matched_host, hostname, **kwargs):
+ self.host = host
+ self.balancer = balancer
+ self.matched_host = matched_host
+ self.hostname = hostname
+
+ action, kwargs, allow_subs = self.balancer.hosts[self.hostname]
+ action_class = self.balancer.action_mapping[action]
+ self.aliased = action_class(balancer = self.balancer, host = self.host, matched_host = self.matched_host, **kwargs)
+
+ def handle(self, **kwargs):
+ return self.aliased.handle(**kwargs)
diff --git a/mantrid/cli.py b/mantrid/cli.py
index d0535b0..bfdc451 100644
--- a/mantrid/cli.py
+++ b/mantrid/cli.py
@@ -74,6 +74,11 @@ def action_list(self):
details[0],
details[1]['code'],
)
+ elif details[0] == "alias":
+ action = "%s<%s>" % (
+ details[0],
+ details[1]['hostname'],
+ )
else:
action = details[0]
print format % (host, action, details[2])
@@ -107,6 +112,9 @@ def action_set(self, hostname=None, action=None, subdoms=None, *args):
if action in ("proxy, mirror") and "backends" not in options:
sys.stderr.write("The %s action requires a backends option.\n" % action)
sys.exit(1)
+ if action == "alias" and "hostname" not in options:
+ sys.stderr.write("The %s action requires hostname option.\n" % action)
+ sys.exit(1)
if "healthcheck" in options and options["healthcheck"].lower() not in ("true", "false"):
sys.stderr.write("The healthcheck option must be one of (true, false)")
sys.exit(1)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index f3644f2..a13e061 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -13,7 +13,7 @@
import mantrid.json
-from mantrid.actions import NoHealthyBackends, Unknown, Proxy, Empty, Static, Redirect, NoHosts, Spin
+from mantrid.actions import NoHealthyBackends, Unknown, Proxy, Empty, Static, Redirect, NoHosts, Spin, Alias
from mantrid.config import SimpleConfig
from mantrid.management import ManagementApp
from mantrid.stats_socket import StatsSocket
@@ -51,6 +51,7 @@ class Balancer(object):
"empty": Empty,
"static": Static,
"redirect": Redirect,
+ "alias": Alias,
"unknown": Unknown,
"spin": Spin,
"no_hosts": NoHosts,
@@ -123,10 +124,11 @@ def main(cls):
def _converted_from_old_format(self, objtree):
hosts = objtree['hosts']
for host, settings in hosts.items():
- backends = settings[1]['backends']
- if backends and not isinstance(backends[0], mantrid.backend.Backend):
- new_backends = map(mantrid.backend.Backend, backends)
- settings[1]['backends'] = new_backends
+ if settings[0] == "proxy":
+ backends = settings[1]['backends']
+ if backends and not isinstance(backends[0], mantrid.backend.Backend):
+ new_backends = map(mantrid.backend.Backend, backends)
+ settings[1]['backends'] = new_backends
return objtree
def load(self):
From cef7db5c8484c8ec5c63d12eb5727653537976e0 Mon Sep 17 00:00:00 2001
From: Lukasz Siudut <lsiudut@gmail.com>
Date: Fri, 27 Feb 2015 00:58:12 +0100
Subject: [PATCH 37/47] Rename not-used parameter to avoid overwriting
---
mantrid/actions.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index b99ed29..551a93c 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -249,7 +249,7 @@ class Alias(Action):
"""
Alias for another backend
"""
- def __init__(self, balancer, host, matched_host, hostname, **kwargs):
+ def __init__(self, balancer, host, matched_host, hostname, **_kwargs):
self.host = host
self.balancer = balancer
self.matched_host = matched_host
From 91198f38544b35b9cea4e040e94aaed550a796b5 Mon Sep 17 00:00:00 2001
From: Lukasz Siudut <lsiudut@gmail.com>
Date: Tue, 3 Mar 2015 15:23:19 +0100
Subject: [PATCH 38/47] Log X-Request-Id if present
API Proxy is going to set X-Request-Id. Let's log it to improve
visibility across backend requests.
---
mantrid/actions.py | 7 ++++---
mantrid/loadbalancer.py | 7 +++++--
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 551a93c..852d538 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -171,9 +171,10 @@ def least_connections(self):
return random.choice([b for b in backends if b.connections == min_connections])
def handle(self, sock, read_data, path, headers):
+ request_id = headers.get("X-Request-Id", "-")
for attempt in range(self.attempts):
if attempt > 0:
- logging.warn("Retrying connection for host %s", self.host)
+ logging.warn("Retrying connection for host %s, request-id: %s", self.host, request_id)
backend = self.select_backend()
try:
@@ -186,12 +187,12 @@ def handle(self, sock, read_data, path, headers):
backend.add_connection()
break
except socket.error:
- logging.exception("Socket error on connect() to %s of %s", backend, self.host)
+ logging.exception("Proxy socket error on connect() to %s of %s, request-id: %s", backend, self.host, request_id)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
except:
- logging.warn("Timeout on connect() to %s of %s", backend, self.host)
+ logging.warn("Proxy timeout on connect() to %s of %s, request-id: %s", backend, self.host, request_id)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index a13e061..3492581 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -344,6 +344,7 @@ def handle(self, sock, address, internal=False):
host = headers['X-Loadbalance-To'] if 'X-Loadbalance-To' in headers else headers['LoadBalanceTo']
except KeyError:
host = "unknown"
+ request_id = headers.get("X-Request-Id", "-")
headers['Connection'] = "close\r"
if not internal:
headers['X-Forwarded-For'] = address[0]
@@ -377,15 +378,17 @@ def handle(self, sock, address, internal=False):
stats_dict['bytes_received'] = stats_dict.get('bytes_received', 0) + sock.bytes_received
except socket.error, e:
if e.errno not in (errno.EPIPE, errno.ETIMEDOUT, errno.ECONNRESET):
+ logging.error("Loadbalancer socket error, error: %s, request-id: %s", request_id, e)
logging.error(traceback.format_exc())
except NoHealthyBackends, e:
- logging.error("No healthy bakckends available for host '%s'" % host)
+ logging.error("No healthy bakckends available for host '%s', request-id: %s", host, request_id)
try:
sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
except socket.error, e:
if e.errno != errno.EPIPE:
raise
- except:
+ except Exception, e:
+ logging.error("Internal Server Error, request id: %s, error: %s", request_id, e)
logging.error(traceback.format_exc())
try:
sock.sendall("HTTP/1.0 500 Internal Server Error\r\n\r\nThere has been an internal error in the load balancer.")
From 3ee670343daece59a745480b0909811f6c7e5f6d Mon Sep 17 00:00:00 2001
From: Lukasz Siudut <lsiudut@gmail.com>
Date: Wed, 4 Mar 2015 13:18:26 +0100
Subject: [PATCH 39/47] Changed format for Bogdan request
---
mantrid/actions.py | 6 +++---
mantrid/loadbalancer.py | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 852d538..d5af33b 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -174,7 +174,7 @@ def handle(self, sock, read_data, path, headers):
request_id = headers.get("X-Request-Id", "-")
for attempt in range(self.attempts):
if attempt > 0:
- logging.warn("Retrying connection for host %s, request-id: %s", self.host, request_id)
+ logging.warn("[%s] Retrying connection for host %s", request_id, self.host)
backend = self.select_backend()
try:
@@ -187,12 +187,12 @@ def handle(self, sock, read_data, path, headers):
backend.add_connection()
break
except socket.error:
- logging.exception("Proxy socket error on connect() to %s of %s, request-id: %s", backend, self.host, request_id)
+ logging.exception("[%s] Proxy socket error on connect() to %s of %s", request_id, backend, self.host)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
except:
- logging.warn("Proxy timeout on connect() to %s of %s, request-id: %s", backend, self.host, request_id)
+ logging.warn("[%s] Proxy timeout on connect() to %s of %s", request_id, backend, self.host)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 3492581..8a630eb 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -378,17 +378,17 @@ def handle(self, sock, address, internal=False):
stats_dict['bytes_received'] = stats_dict.get('bytes_received', 0) + sock.bytes_received
except socket.error, e:
if e.errno not in (errno.EPIPE, errno.ETIMEDOUT, errno.ECONNRESET):
- logging.error("Loadbalancer socket error, error: %s, request-id: %s", request_id, e)
+ logging.error("[%s] Loadbalancer socket error, error: %s", request_id, e)
logging.error(traceback.format_exc())
except NoHealthyBackends, e:
- logging.error("No healthy bakckends available for host '%s', request-id: %s", host, request_id)
+ logging.error("[%s] No healthy bakckends available for host '%s'", request_id, host)
try:
sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
except socket.error, e:
if e.errno != errno.EPIPE:
raise
except Exception, e:
- logging.error("Internal Server Error, request id: %s, error: %s", request_id, e)
+ logging.error("[%s] Internal Server Error, error: %s", request_id, e)
logging.error(traceback.format_exc())
try:
sock.sendall("HTTP/1.0 500 Internal Server Error\r\n\r\nThere has been an internal error in the load balancer.")
From d77f4d8bd00db09c08bbe6aa7e1d4b3ad50166a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miros=C5=82aw=20Naga=C5=9B?= <mgolden@smoczus.pl>
Date: Tue, 10 Mar 2015 14:59:08 +0100
Subject: [PATCH 40/47] Get rid of stack traces in logs
---
mantrid/actions.py | 2 +-
mantrid/loadbalancer.py | 11 ++++-------
mantrid/socketmeld.py | 4 ++--
3 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index d5af33b..465fc38 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -187,7 +187,7 @@ def handle(self, sock, read_data, path, headers):
backend.add_connection()
break
except socket.error:
- logging.exception("[%s] Proxy socket error on connect() to %s of %s", request_id, backend, self.host)
+ logging.error("[%s] Proxy socket error on connect() to %s of %s", request_id, backend, self.host)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 8a630eb..2d94256 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -1,7 +1,6 @@
import eventlet
import errno
import logging
-import traceback
import mimetools
import resource
import os
@@ -218,8 +217,8 @@ def run(self):
pool.wait()
except (KeyboardInterrupt, StopIteration, SystemExit):
pass
- except:
- logging.error(traceback.format_exc())
+ except Exception, e:
+ logging.error("Unhandled Exception %s" % e)
# We're done
self.running = False
logging.info("Exiting")
@@ -379,7 +378,6 @@ def handle(self, sock, address, internal=False):
except socket.error, e:
if e.errno not in (errno.EPIPE, errno.ETIMEDOUT, errno.ECONNRESET):
logging.error("[%s] Loadbalancer socket error, error: %s", request_id, e)
- logging.error(traceback.format_exc())
except NoHealthyBackends, e:
logging.error("[%s] No healthy bakckends available for host '%s'", request_id, host)
try:
@@ -389,7 +387,6 @@ def handle(self, sock, address, internal=False):
raise
except Exception, e:
logging.error("[%s] Internal Server Error, error: %s", request_id, e)
- logging.error(traceback.format_exc())
try:
sock.sendall("HTTP/1.0 500 Internal Server Error\r\n\r\nThere has been an internal error in the load balancer.")
except socket.error, e:
@@ -399,8 +396,8 @@ def handle(self, sock, address, internal=False):
try:
sock.close()
rfile.close()
- except:
- logging.error(traceback.format_exc())
+ except Exception, e:
+ logging.error("Unhandled Exception %s" % e)
def _set_hosts(self, hosts):
self.__dict__['hosts'] = ManagedHostDict(hosts)
diff --git a/mantrid/socketmeld.py b/mantrid/socketmeld.py
index 2b8248e..a210bbb 100644
--- a/mantrid/socketmeld.py
+++ b/mantrid/socketmeld.py
@@ -67,11 +67,11 @@ def run(self):
try:
self.server.close()
except:
- logging.exception("Exception caught closing server socket, backend %s of %s", self.backend, self.host)
+ logging.error("Exception caught closing server socket, backend %s of %s", self.backend, self.host)
try:
self.client.close()
except:
- logging.exception("Exception caught closing client socket, backend: %s of %s", self.backend, self.host)
+ logging.error("Exception caught closing client socket, backend: %s of %s", self.backend, self.host)
return self.data_handled
From d164080efd7d57f355b43469c8271a89ea1746f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miros=C5=82aw=20Naga=C5=9B?= <mgolden@smoczus.pl>
Date: Tue, 10 Mar 2015 15:01:11 +0100
Subject: [PATCH 41/47] Normalize mantrid logs
---
mantrid/actions.py | 8 ++++----
mantrid/loadbalancer.py | 10 +++++-----
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 465fc38..2e53d58 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -174,7 +174,7 @@ def handle(self, sock, read_data, path, headers):
request_id = headers.get("X-Request-Id", "-")
for attempt in range(self.attempts):
if attempt > 0:
- logging.warn("[%s] Retrying connection for host %s", request_id, self.host)
+ logging.warn("Retrying connection for host %s\" - \"%s", self.host, request_id)
backend = self.select_backend()
try:
@@ -187,12 +187,12 @@ def handle(self, sock, read_data, path, headers):
backend.add_connection()
break
except socket.error:
- logging.error("[%s] Proxy socket error on connect() to %s of %s", request_id, backend, self.host)
+ logging.error("Proxy socket error on connect() to %s of %s\" - \"%s", backend, self.host, request_id)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
except:
- logging.warn("[%s] Proxy timeout on connect() to %s of %s", request_id, backend, self.host)
+ logging.warn("Proxy timeout on connect() to %s of %s\" - \"%s", backend, self.host, request_id)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
@@ -213,7 +213,7 @@ def send_onwards(data):
def blacklist(self, backend):
if self.healthcheck and not backend.blacklisted:
- logging.warn("Blacklisting backend %s", backend)
+ logging.warn("Blacklisting backend %s of %s", backend, self.host)
backend.blacklisted = True
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 2d94256..71db13e 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -88,7 +88,7 @@ def main(cls):
# Output to stderr, always
sh = logging.StreamHandler()
sh.setFormatter(logging.Formatter(
- fmt = "%(asctime)s - %(levelname)8s: %(message)s",
+ fmt = "\"%(asctime)s\" - \"%(levelname)8s\" - \"%(message)s\"",
datefmt="%Y-%m-%d %H:%M:%S",
))
sh.setLevel(logging.DEBUG)
@@ -377,16 +377,16 @@ def handle(self, sock, address, internal=False):
stats_dict['bytes_received'] = stats_dict.get('bytes_received', 0) + sock.bytes_received
except socket.error, e:
if e.errno not in (errno.EPIPE, errno.ETIMEDOUT, errno.ECONNRESET):
- logging.error("[%s] Loadbalancer socket error, error: %s", request_id, e)
+ logging.error("Loadbalancer socket error, error: %s\" - \"%s", e, request_id)
except NoHealthyBackends, e:
- logging.error("[%s] No healthy bakckends available for host '%s'", request_id, host)
+ logging.error("No healthy backends available for host '%s'\" - \"%s", host, request_id)
try:
- sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy bakckends available.")
+ sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy backends available.")
except socket.error, e:
if e.errno != errno.EPIPE:
raise
except Exception, e:
- logging.error("[%s] Internal Server Error, error: %s", request_id, e)
+ logging.error("Internal Server Error, error: %s\" - \"%s", e, request_id)
try:
sock.sendall("HTTP/1.0 500 Internal Server Error\r\n\r\nThere has been an internal error in the load balancer.")
except socket.error, e:
From 8feb2e9d23bef6ac7dfef5f474c84ab044cb3cc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miros=C5=82aw=20Naga=C5=9B?= <mgolden@smoczus.pl>
Date: Tue, 10 Mar 2015 15:07:33 +0100
Subject: [PATCH 42/47] No need for severity string to be of fixed size
---
mantrid/loadbalancer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 71db13e..6293dea 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -88,7 +88,7 @@ def main(cls):
# Output to stderr, always
sh = logging.StreamHandler()
sh.setFormatter(logging.Formatter(
- fmt = "\"%(asctime)s\" - \"%(levelname)8s\" - \"%(message)s\"",
+ fmt = "\"%(asctime)s\" - \"%(levelname)s\" - \"%(message)s\"",
datefmt="%Y-%m-%d %H:%M:%S",
))
sh.setLevel(logging.DEBUG)
From 97180b8f929396a97a9e0bae916c547e4cfcd5f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miros=C5=82aw=20Naga=C5=9B?= <mgolden@smoczus.pl>
Date: Tue, 10 Mar 2015 16:17:43 +0100
Subject: [PATCH 43/47] Fix logs formatting that include request_id string
---
mantrid/actions.py | 6 +++---
mantrid/loadbalancer.py | 8 ++++----
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/mantrid/actions.py b/mantrid/actions.py
index 2e53d58..17808cb 100644
--- a/mantrid/actions.py
+++ b/mantrid/actions.py
@@ -174,7 +174,7 @@ def handle(self, sock, read_data, path, headers):
request_id = headers.get("X-Request-Id", "-")
for attempt in range(self.attempts):
if attempt > 0:
- logging.warn("Retrying connection for host %s\" - \"%s", self.host, request_id)
+ logging.warn("[%s] Retrying connection for host %s", request_id, self.host)
backend = self.select_backend()
try:
@@ -187,12 +187,12 @@ def handle(self, sock, read_data, path, headers):
backend.add_connection()
break
except socket.error:
- logging.error("Proxy socket error on connect() to %s of %s\" - \"%s", backend, self.host, request_id)
+ logging.error("[%s] Proxy socket error on connect() to %s of %s", request_id, backend, self.host)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
except:
- logging.warn("Proxy timeout on connect() to %s of %s\" - \"%s", backend, self.host, request_id)
+ logging.warn("[%s] Proxy timeout on connect() to %s of %s", request_id, backend, self.host)
self.blacklist(backend)
eventlet.sleep(self.delay)
continue
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 6293dea..0bb6ee1 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -88,7 +88,7 @@ def main(cls):
# Output to stderr, always
sh = logging.StreamHandler()
sh.setFormatter(logging.Formatter(
- fmt = "\"%(asctime)s\" - \"%(levelname)s\" - \"%(message)s\"",
+ fmt = "%(asctime)s - %(levelname)s - %(message)s";,
datefmt="%Y-%m-%d %H:%M:%S",
))
sh.setLevel(logging.DEBUG)
@@ -377,16 +377,16 @@ def handle(self, sock, address, internal=False):
stats_dict['bytes_received'] = stats_dict.get('bytes_received', 0) + sock.bytes_received
except socket.error, e:
if e.errno not in (errno.EPIPE, errno.ETIMEDOUT, errno.ECONNRESET):
- logging.error("Loadbalancer socket error, error: %s\" - \"%s", e, request_id)
+ logging.error("[%s] Loadbalancer socket error, error: %s", request_id, e)
except NoHealthyBackends, e:
- logging.error("No healthy backends available for host '%s'\" - \"%s", host, request_id)
+ logging.error("[%s] No healthy backends available for host '%s'", request_id, host)
try:
sock.sendall("HTTP/1.0 597 No Healthy Backends\r\n\r\nNo healthy backends available.")
except socket.error, e:
if e.errno != errno.EPIPE:
raise
except Exception, e:
- logging.error("Internal Server Error, error: %s\" - \"%s", e, request_id)
+ logging.error("[%s] Internal Server Error, error: %s", request_id, e)
try:
sock.sendall("HTTP/1.0 500 Internal Server Error\r\n\r\nThere has been an internal error in the load balancer.")
except socket.error, e:
From d632238f1a3a2e54e872eb56de182c551cf50586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miros=C5=82aw=20Naga=C5=9B?= <mgolden@smoczus.pl>
Date: Tue, 10 Mar 2015 16:19:35 +0100
Subject: [PATCH 44/47] Fix typo
---
mantrid/loadbalancer.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/loadbalancer.py b/mantrid/loadbalancer.py
index 0bb6ee1..236e3d2 100644
--- a/mantrid/loadbalancer.py
+++ b/mantrid/loadbalancer.py
@@ -88,7 +88,7 @@ def main(cls):
# Output to stderr, always
sh = logging.StreamHandler()
sh.setFormatter(logging.Formatter(
- fmt = "%(asctime)s - %(levelname)s - %(message)s";,
+ fmt = "%(asctime)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
))
sh.setLevel(logging.DEBUG)
From ab7f0ac784418719f1c242ecdfa0d9049bbe829b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miros=C5=82aw=20Naga=C5=9B?= <mgolden@smoczus.pl>
Date: Tue, 10 Mar 2015 17:10:01 +0100
Subject: [PATCH 45/47] Bump version
---
mantrid/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/__init__.py b/mantrid/__init__.py
index 382021f..9e604c0 100644
--- a/mantrid/__init__.py
+++ b/mantrid/__init__.py
@@ -1 +1 @@
-__version__ = "1.0.6"
+__version__ = "1.0.7"
From a303a235f0242a80fb304ef2a169a16896e166bc Mon Sep 17 00:00:00 2001
From: Marcin Matlaszek <mmatlaszek@gmail.com>
Date: Thu, 26 Mar 2015 14:24:15 +0100
Subject: [PATCH 46/47] Return 594 status code on backend timeout.
---
mantrid/__init__.py | 2 +-
mantrid/socketmeld.py | 8 +++++++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/mantrid/__init__.py b/mantrid/__init__.py
index 9e604c0..e13bd59 100644
--- a/mantrid/__init__.py
+++ b/mantrid/__init__.py
@@ -1 +1 @@
-__version__ = "1.0.7"
+__version__ = "1.0.8"
diff --git a/mantrid/socketmeld.py b/mantrid/socketmeld.py
index a210bbb..36042e3 100644
--- a/mantrid/socketmeld.py
+++ b/mantrid/socketmeld.py
@@ -6,7 +6,6 @@
from eventlet.green import socket
from eventlet.timeout import Timeout
-
class SocketMelder(object):
"""
Takes two sockets and directly connects them together.
@@ -44,6 +43,13 @@ def piper(self, in_sock, out_sock, out_addr, onkill):
except greenlet.GreenletExit:
return
except Timeout:
+ # This one prevents only from closing connection without any data nor status code returned
+ # from mantrid when no data was received from backend.
+ # When it happens, nginx reports 'upstream prematurely closed connection' and returns 500,
+ # and want to have our custom error page to know when it happens.
+
+ if onkill == "stoc" and self.data_handled == 0:
+ out_sock.sendall("HTTP/1.0 594 Backend timeout\r\n\r\n")
logging.warn("Timeout serving request to backend %s of %s", self.backend, self.host)
return
From af7d170f6505afe32651af628a9cf98083e5b446 Mon Sep 17 00:00:00 2001
From: Marcin Matlaszek <mmatlaszek@gmail.com>
Date: Mon, 30 Mar 2015 15:06:30 +0200
Subject: [PATCH 47/47] Add closing connection after sending status
---
mantrid/socketmeld.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mantrid/socketmeld.py b/mantrid/socketmeld.py
index 36042e3..b752ad1 100644
--- a/mantrid/socketmeld.py
+++ b/mantrid/socketmeld.py
@@ -49,7 +49,7 @@ def piper(self, in_sock, out_sock, out_addr, onkill):
# and want to have our custom error page to know when it happens.
if onkill == "stoc" and self.data_handled == 0:
- out_sock.sendall("HTTP/1.0 594 Backend timeout\r\n\r\n")
+ out_sock.sendall("HTTP/1.0 594 Backend timeout\r\nConnection: close\r\nContent-length: 0\r\n\r\n")
logging.warn("Timeout serving request to backend %s of %s", self.backend, self.host)
return