Merge pull request #192 from docker/staging

1.6.6

Author: Feng Honglin

.gitignore

@@ -40,3 +40,4 @@ nosetests.xml
 .pydevproject
 .idea
 .DS_Store
+.vagrant/

README.md

@@ -199,6 +199,7 @@ Settings in this part is immutable, you have to redeploy HAProxy service to make
 |Environment Variable|Default|Description|
 |:-----:|:-----:|:----------|
 |ADDITIONAL_BACKEND_\<NAME\>| |add an additional backend with the name set in <NAME>. Possible values include:`balance source, server 127.0.0.1:8080`|
+|ADDITIONAL_BACKEND_FILE_\<NAME\>| add an additional backend with the name set in <NAME> and value of the contents of specified file.
 |ADDITIONAL_SERVICES| |list of additional services to balance (es: `prj1:web,prj2:sql`). Discovery will be based on `com.docker.compose.[project|service]` container labels. This environment variable only works on compose v2, and the referenced services must be on a network resolvable and accessible to this containers.|
 |BALANCE|roundrobin|load balancing algorithm to use. Possible values include: `roundrobin`, `static-rr`, `source`, `leastconn`. See:[HAProxy:balance](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-balance)|
 |CA_CERT_FILE| |the path of a ca-cert file. This allows you to mount your ca-cert file directly from a volume instead of from envvar. If set, `CA_CERT` envvar will be ignored. Possible value: `/cacerts/cert0.pem`|
@@ -207,8 +208,11 @@ Settings in this part is immutable, you have to redeploy HAProxy service to make
 |DEFAULT_SSL_CERT| |Default ssl cert, a pem file content with private key followed by public certificate, '\n'(two chars) as the line separator. should be formatted as one line - see [SSL Termination](#ssl-termination)|
 |EXTRA_BIND_SETTINGS| |comma-separated string(`<port>:<setting>`) of extra settings, and each part will be appended to the related port bind section in the configuration file. To escape comma, use `\,`. Possible value: `443:accept-proxy, 80:name http`|
 |EXTRA_DEFAULT_SETTINGS| |comma-separated string of extra settings, and each part will be appended to DEFAULT section in the configuration file. To escape comma, use `\,`|
+|EXTRA_DEFAULT_SETTINGS_FILE|File whose contents will be included in the DEFAULT section of the configuration file.|
 |EXTRA_FRONTEND_SETTINGS_\<PORT\>| |comma-separated string of extra settings, and each part will be appended frontend section with the port number specified in the name of the envvar. To escape comma, use `\,`. E.g. `EXTRA_FRONTEND_SETTINGS_80=balance source, maxconn 2000`|
+|EXTRA_DEFAULT_SETTINGS_FILE_\<PORT\>|File whose contents will be appended to the frontend section with the port number specified in the filename.|
 |EXTRA_GLOBAL_SETTINGS| |comma-separated string of extra settings, and each part will be appended to GLOBAL section in the configuration file. To escape comma, use `\,`. Possible value: `tune.ssl.cachesize 20000, tune.ssl.default-dh-param 2048`|
+|EXTRA_GLOBAL_SETTINGS_FILE|File whose contents will be included in the GLOBAL section of the configuration file.|
 |EXTRA_ROUTE_SETTINGS| |a string which is append to the each backend route after the health check, can be over written in the linked services. Possible value: "send-proxy"|
 |EXTRA_SSL_CERTS| |list of extra certificate names separated by comma, eg. `CERT1, CERT2, CERT3`. You also need to specify each certificate as separate env variables like so: `CERT1="<cert-body1>"`, `CERT2="<cert-body2>"`, `CERT3="<cert-body3>"`|
 |FORCE_DEFAULT_BACKEND| True | set the default_service as a default backend. This is useful when you have more than one backend and you don't want your default_service as a default backend    
@@ -238,7 +242,6 @@ Settings here can overwrite the settings in HAProxy, which are only applied to t
 
 |Environment Variable|Description|
 |:-----:|:----------|
-|APPSESSION|sticky session option, possible value `JSESSIONID len 52 timeout 3h`. See:[HAProxy:appsession](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-appsession)|
 |BALANCE|load balancing algorithm to use. Possible values include: `roundrobin`, `static-rr`, `source`, `leastconn`. See:[HAProxy:balance](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-balance)|
 |COOKIE|sticky session option. Possible value `SRV insert indirect nocache`. See:[HAProxy:cookie](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-cookie)|
 |DEFAULT_SSL_CERT|similar to SSL_CERT, but stores the pem file at `/certs/cert0.pem` as the default ssl certs. If multiple `DEFAULT_SSL_CERT` are specified in linked services and HAProxy, the behavior is undefined|
@@ -363,10 +366,9 @@ Copy the output and set it as the value of `SSL_CERT` or `DEFAULT_SSL_CERT`.
 There are three method to setup affinity and sticky session:
 
 1. set `BALANCE=source` in your application service. When setting `source` method of balance, HAProxy will hash the client IP address and make sure that the same IP always goes to the same server.
-2. set `APPSESSION=<value>`. use application session to determine which server a client should connect to. Possible value of `<value>` could be `JSESSIONID len 52 timeout 3h`
 2. set `COOKIE=<value>`. use application cookie to determine which server a client should connect to. Possible value of `<value>` could be `SRV insert indirect nocache`
 
-Check [HAProxy:appsession](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-appsession) and [HAProxy:cookie](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-cookie) for more information.
+Check [HAProxy:cookie](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-cookie) for more information.
 
 
 ## TCP load balancing

haproxy/__init__.py

@@ -1 +1 @@
-__version__ = "1.6.5"
+__version__ = "1.6.6"

haproxy/config.py

@@ -1,6 +1,9 @@
+import logging
 import os
 import re
 
+logger = logging.getLogger("haproxy")
+
 
 class RunningMode():
     LegacyMode, ComposeMode, SwarmMode, CloudMode = range(4)
@@ -21,11 +24,24 @@ def parse_extra_frontend_settings(envvars):
     settings_dict = {}
     if isinstance(envvars, os._Environ) or isinstance(envvars, dict):
         frontend_settings_pattern = re.compile(r"^EXTRA_FRONTEND_SETTINGS_(\d{1,5})$")
+        frontend_settings_file_pattern = re.compile(r"^EXTRA_FRONTEND_SETTINGS_FILE_(\d{1,5})$")
         for k, v in envvars.iteritems():
+            settings = []
             match = frontend_settings_pattern.match(k)
+            file_match = frontend_settings_file_pattern.match(k)
             if match:
                 port = match.group(1)
-                settings = [x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())]
+                settings.extend([x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())])
+            elif file_match:
+                port = file_match.group(1)
+                try:
+                    with open(v) as file:
+                        for line in file:
+                            settings.append(line.strip())
+                except Exception as e:
+                    logger.info("Error reading %s at '%s', error %s" % (k, v, e))
+
+            if len(settings) > 0:
                 if port in settings_dict:
                     settings_dict[port].extend(settings)
                 else:
@@ -36,12 +52,26 @@ def parse_extra_frontend_settings(envvars):
 def parse_additional_backend_settings(envvars):
     settings_dict = {}
     if isinstance(envvars, os._Environ) or isinstance(envvars, dict):
-        frontend_settings_pattern = re.compile(r"^ADDITIONAL_BACKEND_(\w{1,9})$")
+        additional_backend_pattern = re.compile(r"^ADDITIONAL_BACKEND_(\w+)$")
+        additional_backend_file_pattern = re.compile(r"^ADDITIONAL_BACKEND_FILE_(\w+)$")
         for k, v in envvars.iteritems():
-            match = frontend_settings_pattern.match(k)
-            if match:
+            settings = []
+            match = additional_backend_pattern.match(k)
+            file_match = additional_backend_file_pattern.match(k)
+
+            if file_match:
+                server = file_match.group(1)
+                try:
+                    with open(v) as f:
+                        for line in f:
+                            settings.append(line.strip())
+                except Exception as e:
+                    logger.info("Error reading %s at '%s', error %s" % (k, v, e))
+            elif match:
                 server = match.group(1)
-                settings = [x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())]
+                settings.extend([x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())])
+
+            if len(settings) > 0:
                 if server in settings_dict:
                     settings_dict[server].extend(settings)
                 else:
@@ -61,8 +91,10 @@ def parse_additional_backend_settings(envvars):
 DEFAULT_SSL_CERT = os.getenv("DEFAULT_SSL_CERT") or os.getenv("SSL_CERT")
 EXTRA_BIND_SETTINGS = parse_extra_bind_settings(os.getenv("EXTRA_BIND_SETTINGS"))
 EXTRA_DEFAULT_SETTINGS = os.getenv("EXTRA_DEFAULT_SETTINGS")
+EXTRA_DEFAULT_SETTINGS_FILE = os.getenv("EXTRA_DEFAULT_SETTINGS_FILE")
 EXTRA_FRONTEND_SETTINGS = parse_extra_frontend_settings(os.environ)
 EXTRA_GLOBAL_SETTINGS = os.getenv("EXTRA_GLOBAL_SETTINGS")
+EXTRA_GLOBAL_SETTINGS_FILE = os.getenv("EXTRA_GLOBAL_SETTINGS_FILE")
 EXTRA_SSL_CERT = os.getenv("EXTRA_SSL_CERTS")
 EXTRA_ROUTE_SETTINGS = os.getenv("EXTRA_ROUTE_SETTINGS", "")
 FORCE_DEFAULT_BACKEND = os.getenv("FORCE_DEFAULT_BACKEND", "True")

haproxy/haproxycfg.py

@@ -1,5 +1,4 @@
 import copy
-import logging
 import time
 from collections import OrderedDict
 
@@ -256,7 +255,16 @@ def _config_global_section():
         statements.extend(ConfigHelper.config_ssl_bind_options(SSL_BIND_OPTIONS))
         statements.extend(ConfigHelper.config_ssl_bind_ciphers(SSL_BIND_CIPHERS))
         statements.extend(ConfigHelper.config_extra_settings(EXTRA_GLOBAL_SETTINGS))
+        if EXTRA_GLOBAL_SETTINGS_FILE:
+            try:
+                with open(EXTRA_GLOBAL_SETTINGS_FILE) as file:
+                    for line in file:
+                        statements.append(line.strip())
+            except Exception as e:
+                logger.info(
+                    "Error reading EXTRA_GLOBAL_SETTINGS_FILE at '%s', error %s" % (EXTRA_GLOBAL_SETTINGS_FILE, e))
         cfg["global"] = statements
+
         return cfg
 
     @staticmethod
@@ -285,7 +293,14 @@ def _config_defaults_section():
         statements.extend(ConfigHelper.config_option(OPTION))
         statements.extend(ConfigHelper.config_timeout(TIMEOUT))
         statements.extend(ConfigHelper.config_extra_settings(EXTRA_DEFAULT_SETTINGS))
-
+        if EXTRA_DEFAULT_SETTINGS_FILE:
+            try:
+                with open(EXTRA_DEFAULT_SETTINGS_FILE) as file:
+                    for line in file:
+                        statements.append(line.strip())
+            except Exception as e:
+                logger.info(
+                    "Error reading EXTRA_DEFAULT_SETTINGS_FILE at '%s', error %s" % (EXTRA_DEFAULT_SETTINGS_FILE, e))
         cfg["defaults"] = statements
         return cfg
 
@@ -358,11 +373,10 @@ def _config_frontend_sections(self):
                 port_str = "frontend port_%s" % port
                 if port_str in cfg:
                     del cfg[port_str]
-
         else:
             self.require_default_route = FrontendHelper.check_require_default_route(self.specs.get_routes(),
                                                                                     self.routes_added)
-            if self.require_default_route:
+            if self.require_default_route or EXTRA_FRONTEND_SETTINGS:
                 cfg, monitor_uri_configured = FrontendHelper.config_default_frontend(ssl_bind_string)
             else:
                 cfg = OrderedDict()

haproxy/helper/backend_helper.py

@@ -100,11 +100,6 @@ def get_sticky_setting(details, service_alias):
     setting = []
     is_sticky = False
 
-    appsession = get_service_attribute(details, "appsession", service_alias)
-    if appsession:
-        setting.append("appsession %s" % appsession)
-        is_sticky = True
-
     cookie = get_service_attribute(details, "cookie", service_alias)
     if cookie:
         setting.append("cookie %s" % cookie)

haproxy/helper/frontend_helper.py

@@ -137,48 +137,50 @@ def get_bind_string(port, ssl_bind_string, vhosts):
 def config_default_frontend(ssl_bind_string):
     cfg = OrderedDict()
     monitor_uri_configured = False
-    frontend = [("bind :80 %s" % EXTRA_BIND_SETTINGS.get('80', "")).strip()]
-
-    # add x-forwarded-porto header if not skipped
-    if not SKIP_FORWARDED_PROTO:
-        frontend.append("reqadd X-Forwarded-Proto:\ http")
-
-    frontend.append("maxconn %s" % MAXCONN)
-
-    if MONITOR_URI and MONITOR_PORT == '80':
-        frontend.append("monitor-uri %s" % MONITOR_URI)
-        monitor_uri_configured = True
-
-    if "80" in EXTRA_FRONTEND_SETTINGS:
-        frontend.extend(EXTRA_FRONTEND_SETTINGS["80"])
+    monitor_uri_configured_80 = False
+    monitor_uri_configured_443 = False
+    monitor_uri_configured_port = False
+    if EXTRA_FRONTEND_SETTINGS:
+        is_monitor_uri_configured = False
+        for port in EXTRA_FRONTEND_SETTINGS.keys():
+            frontend, is_monitor_uri_configured_port = config_default_frontend_by_port(port, ssl_bind_string)
+            cfg["frontend default_port_%s" % port] = frontend
+            is_monitor_uri_configured = is_monitor_uri_configured or is_monitor_uri_configured_port
+        monitor_uri_configured_port = monitor_uri_configured_port or is_monitor_uri_configured
+    else:
+        frontend, monitor_uri_configured_80 = config_default_frontend_by_port("80", "")
+        cfg["frontend default_port_80"] = frontend
+        if ssl_bind_string:
+            frontend_ssl, monitor_uri_configured_443 = config_default_frontend_by_port("443", ssl_bind_string)
+            cfg["frontend default_port_443"] = frontend_ssl
 
-    if "True" in FORCE_DEFAULT_BACKEND:
-        frontend.append("default_backend default_service")
+    return cfg, monitor_uri_configured or monitor_uri_configured_80 or monitor_uri_configured_443 or monitor_uri_configured_port
 
-    cfg["frontend default_port_80"] = frontend
 
+def config_default_frontend_by_port(port, ssl_bind_string):
+    monitor_uri_configured = False
     if ssl_bind_string:
-        ssl_frontend = [("bind :443 %s %s" % (ssl_bind_string, EXTRA_BIND_SETTINGS.get('443', ""))).strip()]
-
+        frontend = [("bind :%s %s %s" % (port, ssl_bind_string, EXTRA_BIND_SETTINGS.get(port, ""))).strip()]
         # add x-forwarded-porto header if not skipped
         if not SKIP_FORWARDED_PROTO:
-            ssl_frontend.append("reqadd X-Forwarded-Proto:\ https")
-
-        ssl_frontend.append("maxconn %s" % MAXCONN)
-
-        if MONITOR_URI and (MONITOR_PORT == '443'):
-            ssl_frontend.append("monitor-uri %s" % MONITOR_URI)
-            monitor_uri_configured = True
+            frontend.append("reqadd X-Forwarded-Proto:\ https")
+    else:
+        frontend = [("bind :%s %s" % (port, EXTRA_BIND_SETTINGS.get(port, ""))).strip()]
+        if not SKIP_FORWARDED_PROTO:
+            frontend.append("reqadd X-Forwarded-Proto:\ http")
 
-        if "443" in EXTRA_FRONTEND_SETTINGS:
-            ssl_frontend.extend(EXTRA_FRONTEND_SETTINGS["443"])
+    frontend.append("maxconn %s" % MAXCONN)
 
-        if "True" in FORCE_DEFAULT_BACKEND:
-            ssl_frontend.append("default_backend default_service")
+    if MONITOR_URI and MONITOR_PORT == port:
+        frontend.append("monitor-uri %s" % MONITOR_URI)
+        monitor_uri_configured = True
 
-        cfg["frontend default_port_443"] = ssl_frontend
+    if port in EXTRA_FRONTEND_SETTINGS:
+        frontend.extend(EXTRA_FRONTEND_SETTINGS[port])
 
-    return cfg, monitor_uri_configured
+    if "True" in FORCE_DEFAULT_BACKEND:
+        frontend.append("default_backend default_service")
+    return frontend, monitor_uri_configured
 
 
 def config_monitor_frontend(monitor_uri_configured):

haproxy/helper/swarm_mode_link_helper.py

@@ -45,7 +45,7 @@ def get_task_links(tasks, services, haproxy_service_id, haproxy_nets):
             task_slot = "%d" % task.get("Slot", 0)
             task_service_id = task.get("ServiceID", "")
             task_service_name = services_id_name.get(task_service_id, "")
-            task_labels = services_id_labels.get(task_service_id)
+            task_labels = services_id_labels.get(task_service_id, {})
 
             if task_labels.get(LABEL_SWARM_MODE_DEACTIVATE, "").lower() == "true":
                 continue

haproxy/main.py

@@ -23,6 +23,7 @@
 from config import RunningMode
 
 dockercloud.user_agent = "dockercloud-haproxy/%s" % __version__
+dockercloud.api_timeout = 15
 
 logger = logging.getLogger("haproxy")
 

haproxy/parser/base_parser.py

@@ -133,10 +133,6 @@ def parse_virtual_host(value):
     def parse_force_ssl(value):
         return value
 
-    @staticmethod
-    def parse_appsession(value):
-        return value
-
     @staticmethod
     def parse_balance(value):
         return value