Merge pull request #35 from byjg/issues

Issues

.vscode/launch.json

@@ -4,6 +4,22 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "PyTest Current File",
+            "type": "python",
+            "request": "launch",
+            "module": "pytest",
+            "justMyCode": true,
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src",
+            "args": [
+                "-vv",
+                "${file}"
+            ],
+            "env": {
+                "PYTHONPATH": "${cwd}/src"
+            }
+        },
         {
             "name": "Python: Current File",
             "type": "python",

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "cSpell.words": [
+        "certonly",
+        "letsencrypt"
+    ]
+}

docs/docker-environment.md

@@ -1,19 +1,20 @@
 # Docker environment variables
 
-| Environment Variable          | Description                                                                                     | Default          |
-|-------------------------------|-------------------------------------------------------------------------------------------------|------------------|
-| EASYHAPROXY_DISCOVER          | How the services will be discovered to create `haproxy.cfg`:  `static`, `docker`, `swarm` or `kubernetes`     | **required**     |
-| EASYHAPROXY_LABEL_PREFIX      | (Optional) The key will search for matching resources.                                          | `easyhaproxy`    |
-| EASYHAPROXY_LETSENCRYPT_EMAIL | (Optional) The email will be used to request the certificate to Letsencrypt                     | *empty*          |
-| EASYHAPROXY_SSL_MODE          | (Optional) `strict` supports only the most recent TLS version; `default` good SSL integration with recent browsers; `loose` supports all old SSL protocols for old browsers (not recommended).  | `default`|
-| EASYHAPROXY_REFRESH_CONF      | (Optional) Check configuration every N seconds.                                                 | 10               |
-| EASYHAPROXY_LOG_LEVEL         | (Optional) The log level for EasyHAproxy messages. Available: TRACE,DEBUG,INFO,WARN,ERROR,FATAL | DEBUG            |
-| CERTBOT_LOG_LEVEL             | (Optional) The log level for Certbot messages. Available: TRACE,DEBUG,INFO,WARN,ERROR,FATAL     | DEBUG            |
-| HAPROXY_LOG_LEVEL             | (Optional) The log level for HAProxy messages. Available: TRACE,DEBUG,INFO,WARN,ERROR,FATAL     | DEBUG            |
-| HAPROXY_USERNAME              | (Optional) The HAProxy username to the statistics.                                              |  `admin`         |
-| HAPROXY_PASSWORD              | (Optional) The HAProxy password to the statistics. If not set, statistics will be available with no password  | *empty* |
-| HAPROXY_STATS_PORT            | (Optional) The HAProxy port to the statistics. If set to `false`, disable statistics            | `1936`           |
-| HAPROXY_CUSTOMERRORS          | (Optional) If HAProxy will use custom HTML errors. true/false.                                  | `false`          |
+| Environment Variable            | Description                                                                                     | Default          |
+|---------------------------------|-------------------------------------------------------------------------------------------------|------------------|
+| EASYHAPROXY_DISCOVER            | How the services will be discovered to create `haproxy.cfg`:  `static`, `docker`, `swarm` or `kubernetes`     | **required**     |
+| EASYHAPROXY_LABEL_PREFIX        | (Optional) The key will search for matching resources.                                          | `easyhaproxy`    |
+| EASYHAPROXY_LETSENCRYPT_EMAIL   | (Optional) The email will be used to request the certificate to Letsencrypt                     | *empty*          |
+| EASYHAPROXY_LETSENCRYPT_SERVER  | (Optional) Can be `staging` or 'schema://domain.tld'. If set, will try to connect to the Letsencrypt test server  | *empty*            |
+| EASYHAPROXY_SSL_MODE            | (Optional) `strict` supports only the most recent TLS version; `default` good SSL integration with recent browsers; `loose` supports all old SSL protocols for old browsers (not recommended).  | `default`|
+| EASYHAPROXY_REFRESH_CONF        | (Optional) Check configuration every N seconds.                                                 | 10               |
+| EASYHAPROXY_LOG_LEVEL           | (Optional) The log level for EasyHAproxy messages. Available: TRACE,DEBUG,INFO,WARN,ERROR,FATAL | DEBUG            |
+| CERTBOT_LOG_LEVEL               | (Optional) The log level for Certbot messages. Available: TRACE,DEBUG,INFO,WARN,ERROR,FATAL     | DEBUG            |
+| HAPROXY_LOG_LEVEL               | (Optional) The log level for HAProxy messages. Available: TRACE,DEBUG,INFO,WARN,ERROR,FATAL     | DEBUG            |
+| HAPROXY_USERNAME                | (Optional) The HAProxy username to the statistics.                                              |  `admin`         |
+| HAPROXY_PASSWORD                | (Optional) The HAProxy password to the statistics. If not set, statistics will be available with no password  | *empty* |
+| HAPROXY_STATS_PORT              | (Optional) The HAProxy port to the statistics. If set to `false`, disable statistics            | `1936`           |
+| HAPROXY_CUSTOMERRORS            | (Optional) If HAProxy will use custom HTML errors. true/false.                                  | `false`          |
 
 
 

docs/docker.md

@@ -5,6 +5,7 @@
 This method will use a docker standalone installation to discover the containers and configure the HAProxy.
 
 The only requirement is that containers and EasyHAProxy must be in the same docker network.
+If not, EasyHAProxy will connect the container with the EasyHAProxy network.
 
 e.g.:
 

docs/swarm.md

@@ -3,9 +3,10 @@
 ## Setup Docker EasyHAProxy
 
 This method will use a docker swarm installation to discover the containers and configure the HAProxy.
-The advantage of this method is that you can discover containers in other nodes from the cluster. 
+The advantage of this method is that you can discover containers in other nodes from the cluster.
 
 The only requirement is that containers and EasyHAProxy must be in the same docker swarm network.
+If not, EasyHAProxy will connect the service with the EasyHAProxy service network.
 
 e.g.:
 

examples/docker/docker-compose.yml

@@ -13,7 +13,7 @@
 # location: https://host1.local/
 #
 # Test SSL:
-# openssl s_client -showcerts -connect 127.0.0.1:443 --servername host1.local
+# openssl s_client -showcerts -connect 127.0.0.1:443 -servername host1.local
 
 version: "3"
 

examples/swarm/docker-compose.yml

@@ -13,7 +13,7 @@
 # location: https://host1.local/
 #
 # Test SSL:
-# openssl s_client -showcerts -connect 127.0.0.1:443 --servername host1.local
+# openssl s_client -showcerts -connect 127.0.0.1:443 -servername host1.local
 
 version: "3"
 

src/easymapping/__init__.py

@@ -49,7 +49,7 @@ class HaproxyConfigGenerator:
     def __init__(self, mapping):
         self.mapping = mapping
         self.mapping.setdefault("ssl_mode", 'default')
-        self.mapping.setdefault("letsencrypt", {"email": ""})
+        self.mapping.setdefault("letsencrypt", {"email": "", "staging": False})
         self.mapping["ssl_mode"] = self.mapping["ssl_mode"].lower()
         self.label = DockerLabelHandler(mapping['lookup_label'] if 'lookup_label' in mapping else "easyhaproxy")
         self.letsencrypt_hosts = []

src/functions/__init__.py

@@ -174,9 +174,18 @@ def sleep(self):
 
 
 class Certbot:
-    def __init__(self, certs, email):
+    def __init__(self, certs, email, test_server):
         self.certs = certs
         self.email = email
+        self.test_server = self.set_test_server(test_server)
+
+    def set_test_server(self, test_server):
+        if test_server.lower() == "staging":
+            return "--staging"
+        elif test_server.lower().startswith("http"):
+            return "--server " + test_server
+        else:
+            return ""
 
     def check_certificates(self, hosts):
         if self.email == "" or len(hosts) == 0:
@@ -201,7 +210,7 @@ def check_certificates(self, hosts):
                         Functions.log(Functions.CERTBOT_LOG, Functions.DEBUG, "Renew certificate for %s" % (host))
                         renew_certs.append(host_arg)
 
-            certbot_certonly = ('/usr/bin/certbot certonly '
+            certbot_certonly = ('/usr/bin/certbot certonly {test_server}'
                                 '    --standalone'
                                 '    --preferred-challenges http'
                                 '    --http-01-port 2080'
@@ -210,7 +219,9 @@ def check_certificates(self, hosts):
                                 '    --no-eff-email'
                                 '    --non-interactive'
                                 '    --max-log-backups=0'
-                                '    %s --email %s' % (' '.join(request_certs), self.email)
+                                '    {certs} --email {email}'.format(certs = ' '.join(request_certs),
+                                                           email = self.email,
+                                                           test_server = self.test_server)
                             )
 
             ret_reload = False
@@ -235,6 +246,8 @@ def merge_certificate(self, cert, key, filename):
 
     def find_live_certificates(self):
         letsencrypt_certs = "/etc/letsencrypt/live/"
+        if not os.path.exists(letsencrypt_certs):
+            return
         for item in os.listdir(letsencrypt_certs):
             path = os.path.join(letsencrypt_certs, item)
             if os.path.isdir(path):

src/main.py

@@ -22,7 +22,7 @@ def start():
     haproxy.haproxy("start")
     haproxy.sleep()
 
-    certbot = Certbot(Consts.certs_letsencrypt, os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL"))
+    certbot = Certbot(Consts.certs_letsencrypt, os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL"), os.getenv("EASYHAPROXY_LETSENCRYPT_SERVER", "").lower())
 
     while True:
         if old_haproxy is not None:
@@ -70,4 +70,4 @@ def main():
     start()
 
 if __name__ == '__main__':
-    main()
+    main()

src/processor/__init__.py

@@ -6,6 +6,7 @@
 import json
 import base64
 import docker
+import socket
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
 
@@ -27,7 +28,8 @@ def read():
         env_vars["lookup_label"] = os.getenv("EASYHAPROXY_LABEL_PREFIX") if os.getenv("EASYHAPROXY_LABEL_PREFIX") else "easyhaproxy"
         if (os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL")):
             env_vars["letsencrypt"] = {
-                "email": os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL")
+                "email": os.getenv("EASYHAPROXY_LETSENCRYPT_EMAIL"),
+                "server": os.getenv("EASYHAPROXY_LETSENCRYPT_SERVER", "false").lower() in ["true", "1", "yes"]
             }
 
         return env_vars
@@ -126,9 +128,25 @@ def __init__(self, filename = None):
         super().__init__()
 
     def inspect_network(self):
+        try:
+            ha_proxy_network_name = next(iter(self.client.containers.get(socket.gethostname()).attrs["NetworkSettings"]["Networks"]))
+        except:
+            # HAProxy is not running in a container, get first container network
+            if len(self.client.containers.list()) == 0:
+                return
+            ha_proxy_network_name = next(iter(self.client.containers.get(self.client.containers.list()[0].name).attrs["NetworkSettings"]["Networks"]))
+
+        ha_proxy_network = self.client.networks.get(ha_proxy_network_name)
+
         self.parsed_object = {}
         for container in self.client.containers.list():
-            self.parsed_object[container.name] = container.labels
+            # Issue 32 - Docker container cannot connect to containers in different network.
+            if ha_proxy_network_name not in container.attrs["NetworkSettings"]["Networks"].keys():
+                ha_proxy_network.connect(container.name)
+                container = self.client.containers.get(container.name)  # refresh object
+
+            ip_address = container.attrs["NetworkSettings"]["Networks"][ha_proxy_network_name]["IPAddress"]
+            self.parsed_object[ip_address] = container.labels
 
 
 class Swarm(ProcessorInterface):
@@ -137,9 +155,28 @@ def __init__(self, filename = None):
         super().__init__()
 
     def inspect_network(self):
+        ha_proxy_service_name = self.client.containers.get(socket.gethostname()).name.split('.')[0]
+        for endpoint in self.client.services.get(ha_proxy_service_name).attrs['Endpoint']["VirtualIPs"]:                
+            ha_proxy_network_id = endpoint["NetworkID"]
+            if self.client.networks.get(ha_proxy_network_id).name != 'ingress':
+                break
+
         self.parsed_object = {}
-        for container in self.client.services.list():
-            self.parsed_object[container.attrs["Spec"]["Name"]] = container.attrs["Spec"]["Labels"]
+        for service in self.client.services.list():
+            ip_address = None
+            network_list = []
+            for endpoint in service.attrs["Endpoint"]["VirtualIPs"]:
+                if ha_proxy_network_id == endpoint["NetworkID"]:
+                    ip_address = endpoint["Addr"].split("/")[0]
+                    break
+                network_list.append(endpoint["NetworkID"])
+
+            if ip_address is None:
+                network_list.append(ha_proxy_network_id)
+                service.update(networks = network_list)
+                continue # skip to the next service to give time to update the network
+
+            self.parsed_object[ip_address] = service.attrs["Spec"]["Labels"]
 
 
 class Kubernetes(ProcessorInterface):
@@ -162,6 +199,8 @@ def inspect_network(self):
 
         self.parsed_object = {}
         for ingress in ret.items:
+            if 'kubernetes.io/ingress.class' not in ingress.metadata.annotations:
+                continue
             if ingress.metadata.annotations['kubernetes.io/ingress.class'] != "easyhaproxy-ingress":
                 continue
 

src/tests/test_containerenv.py

@@ -95,8 +95,24 @@ def test_container_env_stats_password():
             "lookup_label": "easyhaproxy",
             "letsencrypt": {
                 "email": "[email protected]",
+                "server": False
             }
         } == ContainerEnv.read()
     finally:
         os.environ['EASYHAPROXY_LETSENCRYPT_EMAIL'] = ''
 
+def test_container_env_letsencrypt():
+    os.environ['EASYHAPROXY_LETSENCRYPT_EMAIL'] = '[email protected]'
+    os.environ['EASYHAPROXY_LETSENCRYPT_SERVER'] = 'true'
+    try:
+        assert {
+            "customerrors": False,
+            "ssl_mode": "default",
+            "lookup_label": "easyhaproxy",
+            "letsencrypt": {
+                "email": "[email protected]",
+                "server": True
+            }
+        } == ContainerEnv.read()
+    finally:
+        os.environ['EASYHAPROXY_LETSENCRYPT_EMAIL'] = ''