Merge pull request #62 from docker/staging

1.4.1

Author: Feng Honglin

README.md

@@ -87,7 +87,7 @@ Similar to using legacy links, here list some differences that you need to notic
 - DO not overwrite `HOSTNAME` environment variable in `dockercloud/haproxy container`.
 - As it is the case on Docker Cloud, auto reconfiguration is supported when the linked services scales or/and the linked container starts/stops.
 
-##### example of docker-compose.yml running in linux:
+##### example of docker-compose.yml running on Linux or Docker for Mac (beta):
 
 	version: '2'
 	services:
@@ -102,7 +102,7 @@ Similar to using legacy links, here list some differences that you need to notic
 	    ports:
 	      - 80:80
 
-##### example of docker-compose.yml running in Mac OS
+##### example of docker-compose.yml running on Mac OS
 
 	version: '2'
 	services:
@@ -185,6 +185,7 @@ Settings in this part is immutable, you have to redeploy HAProxy service to make
 |MONITOR_URI| |the exact URI which we want to intercept to return HAProxy's health status instead of forwarding the request.See: http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-monitor-uri. Possible value: `/ping`|
 |OPTION|redispatch|comma-separated list of HAProxy `option` entries to the `default` section.|
 |RSYSLOG_DESTINATION|127.0.0.1|the rsyslog destination to where HAProxy logs are sent|
+|SKIP_FORWARDED_PROTO||If set to any value, HAProxy will not add an X-Forwarded- headers. This can be used when combining HAProxy with another load balancer|
 |SSL_BIND_CIPHERS| |explicitly set which SSL ciphers will be used for the SSL server. This sets the HAProxy `ssl-default-bind-ciphers` configuration setting.|
 |SSL_BIND_OPTIONS|no-sslv3|explicitly set which SSL bind options will be used for the SSL server. This sets the HAProxy `ssl-default-bind-options` configuration setting. The default will allow only TLSv1.0+ to be used on the SSL server.|
 |STATS_AUTH|stats:stats|username and password required to access the Haproxy stats.|

haproxy/config.py

@@ -53,6 +53,7 @@ def parse_extra_frontend_settings(envvars):
 MONITOR_URI = os.getenv("MONITOR_URI")
 OPTION = os.getenv("OPTION", "redispatch, httplog, dontlognull, forwardfor")
 RSYSLOG_DESTINATION = os.getenv("RSYSLOG_DESTINATION", "127.0.0.1")
+SKIP_FORWARDED_PROTO = os.getenv("SKIP_FORWARDED_PROTO")
 SSL_BIND_CIPHERS = os.getenv("SSL_BIND_CIPHERS")
 SSL_BIND_OPTIONS = os.getenv("SSL_BIND_OPTIONS")
 STATS_AUTH = os.getenv("STATS_AUTH", "stats:stats")

haproxy/eventhandler.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import time
 
 import dockercloud
 from compose.cli.docker_client import docker_client
@@ -52,12 +53,23 @@ def on_user_reload(signum, frame):
         run_haproxy("User reload")
 
 
+def on_cloud_error(e):
+    if isinstance(e, KeyboardInterrupt):
+        exit(0)
+
+
 def listen_dockercloud_events():
     events = dockercloud.Events()
     events.on_open(on_websocket_open)
     events.on_close(on_websocket_close)
     events.on_message(on_cloud_event)
-    events.run_forever()
+    events.on_error(on_cloud_error)
+    while True:
+        try:
+            events.run_forever()
+        except dockercloud.AuthError as e:
+            logger.info("Auth error: %s, retry in 1 hour" % e)
+            time.sleep(3600)
 
 
 def listen_docker_events():

haproxy/haproxycfg.py

@@ -64,11 +64,14 @@ def _initialize(link_mode):
     @staticmethod
     def _init_cloud_links():
         haproxy_container = fetch_remote_obj(HAPROXY_CONTAINER_URI)
-        links = CloudLinkHelper.get_cloud_links(haproxy_container)
-        Haproxy.cls_linked_services = CloudLinkHelper.get_linked_services(links)
-        logger.info("Linked service: %s", ", ".join(CloudLinkHelper.get_service_links_str(links)))
-        logger.info("Linked container: %s", ", ".join(CloudLinkHelper.get_container_links_str(links)))
-        return links
+        if haproxy_container:
+            links = CloudLinkHelper.get_cloud_links(haproxy_container)
+            Haproxy.cls_linked_services = CloudLinkHelper.get_linked_services(links)
+            logger.info("Linked service: %s", ", ".join(CloudLinkHelper.get_service_links_str(links)))
+            logger.info("Linked container: %s", ", ".join(CloudLinkHelper.get_container_links_str(links)))
+            return links
+        else:
+            return {}
 
     @staticmethod
     def _init_new_links():

haproxy/helper/frontend_helper.py

@@ -1,6 +1,6 @@
 from collections import OrderedDict
 
-from haproxy.config import EXTRA_BIND_SETTINGS, EXTRA_FRONTEND_SETTINGS, MONITOR_URI, MONITOR_PORT, MAXCONN
+from haproxy.config import EXTRA_BIND_SETTINGS, EXTRA_FRONTEND_SETTINGS, MONITOR_URI, MONITOR_PORT, MAXCONN, SKIP_FORWARDED_PROTO
 
 
 def check_require_default_route(routes, routes_added):
@@ -95,11 +95,12 @@ def config_common_part(port, ssl_bind_string, vhosts):
     bind_string, ssl = get_bind_string(port, ssl_bind_string, vhosts)
     frontend_section.append("bind :%s" % bind_string)
 
-    # add x-forwarded-porto header
-    if ssl:
-        frontend_section.append("reqadd X-Forwarded-Proto:\ https")
-    else:
-        frontend_section.append("reqadd X-Forwarded-Proto:\ http")
+    # add x-forwarded-porto header if not skipped
+    if not SKIP_FORWARDED_PROTO:
+        if ssl:
+            frontend_section.append("reqadd X-Forwarded-Proto:\ https")
+        else:
+            frontend_section.append("reqadd X-Forwarded-Proto:\ http")
 
     # add maxconn
     frontend_section.append("maxconn %s" % MAXCONN)
@@ -135,8 +136,13 @@ def get_bind_string(port, ssl_bind_string, vhosts):
 def config_default_frontend(ssl_bind_string):
     cfg = OrderedDict()
     monitor_uri_configured = False
-    frontend = [("bind :80 %s" % EXTRA_BIND_SETTINGS.get('80', "")).strip(),
-                "reqadd X-Forwarded-Proto:\ http", "maxconn %s" % MAXCONN]
+    frontend = [("bind :80 %s" % EXTRA_BIND_SETTINGS.get('80', "")).strip()]
+
+    # add x-forwarded-porto header if not skipped
+    if not SKIP_FORWARDED_PROTO:
+        frontend.append("reqadd X-Forwarded-Proto:\ http")
+
+    frontend.append("maxconn %s" % MAXCONN)
 
     if MONITOR_URI and MONITOR_PORT == '80':
         frontend.append("monitor-uri %s" % MONITOR_URI)
@@ -149,8 +155,13 @@ def config_default_frontend(ssl_bind_string):
     cfg["frontend default_port_80"] = frontend
 
     if ssl_bind_string:
-        ssl_frontend = [("bind :443 %s %s" % (ssl_bind_string, EXTRA_BIND_SETTINGS.get('443', ""))).strip(),
-                        "reqadd X-Forwarded-Proto:\ https", "maxconn %s" % MAXCONN]
+        ssl_frontend = [("bind :443 %s %s" % (ssl_bind_string, EXTRA_BIND_SETTINGS.get('443', ""))).strip()]
+
+        # add x-forwarded-porto header if not skipped
+        if not SKIP_FORWARDED_PROTO:
+            ssl_frontend.append("reqadd X-Forwarded-Proto:\ https")
+
+        ssl_frontend.append("maxconn %s" % MAXCONN)
 
         if MONITOR_URI and (MONITOR_PORT == '443'):
             ssl_frontend.append("monitor-uri %s" % MONITOR_URI)

haproxy/utils.py

@@ -7,17 +7,27 @@
 
 logger = logging.getLogger("haproxy")
 
+invalid_auth_headers = set()
+
 
 def fetch_remote_obj(uri):
     if not uri:
         return None
 
+    auth_header = str(dockercloud.auth.get_auth_header())
     while True:
         try:
+            if auth_header in invalid_auth_headers:
+                logger.info("Using know invalid credentials")
+                return None
             obj = dockercloud.Utils.fetch_by_resource_uri(uri)
             return obj
+        except dockercloud.AuthError as e:
+            invalid_auth_headers.add(auth_header)
+            logger.info(e)
+            return None
         except Exception as e:
-            logger.error(e)
+            logger.info(e)
             time.sleep(config.API_RETRY)
 
 

requirements.txt

@@ -11,5 +11,5 @@ requests==2.7.0
 six==1.9.0
 websocket-client==0.37.0
 docker-compose==1.6.0
-python-dockercloud==1.0.4
+python-dockercloud==1.0.5
 gevent==1.1.1

tests/unit/helper/test_frontend_helper.py

@@ -17,13 +17,15 @@ def setUp(self):
         frontend_helper.EXTRA_BIND_SETTINGS = {"8888": "name http", "4443": "accept-proxy"}
         frontend_helper.MAXCONN = "55555"
         frontend_helper.EXTRA_FRONTEND_SETTINGS = {}
+        frontend_helper.SKIP_FORWARDED_PROTO = None
 
     def tearDown(self):
         frontend_helper.MONITOR_PORT = self.monitor_port
         frontend_helper.MONITOR_URI = self.monitor_uri
         frontend_helper.EXTRA_BIND_SETTINGS = self.extra_bind_settings
         frontend_helper.MAXCONN = self.maxconn
         frontend_helper.EXTRA_FRONTEND_SETTINGS = {}
+        frontend_helper.SKIP_FORWARDED_PROTO = None
 
     def test_config_frontend_with_virtual_host_without_monitoring_uri_added(self):
         vhosts = [{'service_alias': 'web-a', 'path': '', 'host': 'a.com', 'scheme': 'http', 'port': '80'}]
@@ -185,6 +187,27 @@ def test_config_common_part_with_extra_frontend_sttings(self, mock_get_bind_stri
         self.assertFalse(monitor_uri_configured)
         frontend_helper.EXTRA_FRONTEND_SETTINGS = {}
 
+    @mock.patch("haproxy.helper.frontend_helper.get_bind_string")
+    def test_config_common_part_without_forwarded_headers(self, mock_get_bind_string):
+        mock_get_bind_string.return_value = ("80", False)
+        frontend_helper.SKIP_FORWARDED_PROTO = 'true'
+        frontend_section, monitor_uri_configured = config_common_part("80", "ssl crt /certs/", [])
+        self.assertEqual(["bind :80",
+                          "maxconn 55555",
+                          "acl is_websocket hdr(Upgrade) -i WebSocket"],
+                         frontend_section)
+
+    @mock.patch("haproxy.helper.frontend_helper.get_bind_string")
+    def test_config_common_part_without_forwarded_headers_with_ssl(self, mock_get_bind_string):
+        mock_get_bind_string.return_value = ("9999 ssl crt /certs/", True)
+        frontend_helper.SKIP_FORWARDED_PROTO = 'true'
+        frontend_section, monitor_uri_configured = config_common_part("9999", "ssl crt /certs/", [])
+        self.assertEqual(["bind :9999 ssl crt /certs/",
+                          'maxconn 55555',
+                          "monitor-uri %s" % frontend_helper.MONITOR_URI,
+                          "acl is_websocket hdr(Upgrade) -i WebSocket"],
+                         frontend_section)
+
     def test_get_bind_string(self):
         vhosts = [{'service_alias': 'web-a', 'path': '', 'host': 'a.com', 'scheme': 'http', 'port': '80'},
                   {'service_alias': 'web-b', 'path': '', 'host': 'a.com', 'scheme': 'http', 'port': '8888'},
@@ -277,6 +300,21 @@ def test_config_default_front(self):
                                         'default_backend default_service'])]), cfg)
         self.assertTrue(monitor_uri_configured)
 
+        frontend_helper.SKIP_FORWARDED_PROTO = 'true'
+        cfg, monitor_uri_configured = config_default_frontend("ssl crt /certs/")
+        self.assertEqual(OrderedDict([('frontend default_port_80',
+                                       ['bind :80',
+                                        'maxconn 55555',
+                                        'reqadd header1 value1',
+                                        'default_backend default_service']),
+                                      ('frontend default_port_443',
+                                       ['bind :443 ssl crt /certs/ accept-proxy',
+                                        'maxconn 55555',
+                                        'monitor-uri /ping',
+                                        'reqadd header2 value2',
+                                        'default_backend default_service'])]), cfg)
+        self.assertTrue(monitor_uri_configured)
+
     def test_config_common_part_with_monitor_uri(self):
         self.assertEqual(OrderedDict(), config_monitor_frontend(True))
         self.assertEqual(OrderedDict([('frontend monitor', ['bind :9999', 'monitor-uri /ping'])]),