Add extra options as files (#184)

* Added config for:
EXTRA_DEFAULT_SETTINGS_FILE
EXTRA_GLOBAL_SETTINGS_FILE
EXTRA_FRONTEND_SETTINGS_FILE_XXX
ADDITIONAL_BACKEND_SETTINGS_FILE_XXX

* correct typo in filename

* update readme

* correct error in backend

Author: markvr

.gitignore

@@ -40,3 +40,4 @@ nosetests.xml
 .pydevproject
 .idea
 .DS_Store
+.vagrant/

README.md

@@ -199,6 +199,7 @@ Settings in this part is immutable, you have to redeploy HAProxy service to make
 |Environment Variable|Default|Description|
 |:-----:|:-----:|:----------|
 |ADDITIONAL_BACKEND_\<NAME\>| |add an additional backend with the name set in <NAME>. Possible values include:`balance source, server 127.0.0.1:8080`|
+|ADDITIONAL_BACKEND_FILE_\<NAME\>| add an additional backend with the name set in <NAME> and value of the contents of specified file.
 |ADDITIONAL_SERVICES| |list of additional services to balance (es: `prj1:web,prj2:sql`). Discovery will be based on `com.docker.compose.[project|service]` container labels. This environment variable only works on compose v2, and the referenced services must be on a network resolvable and accessible to this containers.|
 |BALANCE|roundrobin|load balancing algorithm to use. Possible values include: `roundrobin`, `static-rr`, `source`, `leastconn`. See:[HAProxy:balance](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-balance)|
 |CA_CERT_FILE| |the path of a ca-cert file. This allows you to mount your ca-cert file directly from a volume instead of from envvar. If set, `CA_CERT` envvar will be ignored. Possible value: `/cacerts/cert0.pem`|
@@ -207,8 +208,11 @@ Settings in this part is immutable, you have to redeploy HAProxy service to make
 |DEFAULT_SSL_CERT| |Default ssl cert, a pem file content with private key followed by public certificate, '\n'(two chars) as the line separator. should be formatted as one line - see [SSL Termination](#ssl-termination)|
 |EXTRA_BIND_SETTINGS| |comma-separated string(`<port>:<setting>`) of extra settings, and each part will be appended to the related port bind section in the configuration file. To escape comma, use `\,`. Possible value: `443:accept-proxy, 80:name http`|
 |EXTRA_DEFAULT_SETTINGS| |comma-separated string of extra settings, and each part will be appended to DEFAULT section in the configuration file. To escape comma, use `\,`|
+|EXTRA_DEFAULT_SETTINGS_FILE|File whose contents will be included in the DEFAULT section of the configuration file.|
 |EXTRA_FRONTEND_SETTINGS_\<PORT\>| |comma-separated string of extra settings, and each part will be appended frontend section with the port number specified in the name of the envvar. To escape comma, use `\,`. E.g. `EXTRA_FRONTEND_SETTINGS_80=balance source, maxconn 2000`|
+|EXTRA_DEFAULT_SETTINGS_FILE_\<PORT\>|File whose contents will be appended to the frontend section with the port number specified in the filename.|
 |EXTRA_GLOBAL_SETTINGS| |comma-separated string of extra settings, and each part will be appended to GLOBAL section in the configuration file. To escape comma, use `\,`. Possible value: `tune.ssl.cachesize 20000, tune.ssl.default-dh-param 2048`|
+|EXTRA_GLOBAL_SETTINGS_FILE|File whose contents will be included in the GLOBAL section of the configuration file.|
 |EXTRA_ROUTE_SETTINGS| |a string which is append to the each backend route after the health check, can be over written in the linked services. Possible value: "send-proxy"|
 |EXTRA_SSL_CERTS| |list of extra certificate names separated by comma, eg. `CERT1, CERT2, CERT3`. You also need to specify each certificate as separate env variables like so: `CERT1="<cert-body1>"`, `CERT2="<cert-body2>"`, `CERT3="<cert-body3>"`|
 |FORCE_DEFAULT_BACKEND| True | set the default_service as a default backend. This is useful when you have more than one backend and you don't want your default_service as a default backend    

haproxy/config.py

@@ -1,6 +1,8 @@
 import os
 import re
+import logging
 
+logger = logging.getLogger("haproxy")
 
 class RunningMode():
     LegacyMode, ComposeMode, SwarmMode, CloudMode = range(4)
@@ -21,11 +23,24 @@ def parse_extra_frontend_settings(envvars):
     settings_dict = {}
     if isinstance(envvars, os._Environ) or isinstance(envvars, dict):
         frontend_settings_pattern = re.compile(r"^EXTRA_FRONTEND_SETTINGS_(\d{1,5})$")
+        frontend_settings_file_pattern = re.compile(r"^EXTRA_FRONTEND_SETTINGS_FILE_(\d{1,5})$")
         for k, v in envvars.iteritems():
+            settings = []
             match = frontend_settings_pattern.match(k)
+            file_match = frontend_settings_file_pattern.match(k)
             if match:
                 port = match.group(1)
-                settings = [x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())]
+                settings.extend([x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())])
+            elif file_match:
+                port = file_match.group(1)
+                try:
+                    with open(v) as file:
+                        for line in file:
+                          settings.append(line.strip())
+                except Exception as e:
+                    logger.info("Error reading %s at '%s', error %s" % (k, v, e))
+
+            if len(settings) > 0:
                 if port in settings_dict:
                     settings_dict[port].extend(settings)
                 else:
@@ -36,12 +51,25 @@ def parse_extra_frontend_settings(envvars):
 def parse_additional_backend_settings(envvars):
     settings_dict = {}
     if isinstance(envvars, os._Environ) or isinstance(envvars, dict):
-        frontend_settings_pattern = re.compile(r"^ADDITIONAL_BACKEND_(\w{1,9})$")
+        additional_backend_pattern = re.compile(r"^ADDITIONAL_BACKEND_(\w{1,9})$")
+        additional_backend_file_pattern = re.compile(r"^ADDITIONAL_BACKEND_FILE_(\w{1,9})$")
         for k, v in envvars.iteritems():
-            match = frontend_settings_pattern.match(k)
+            settings = []
+            match = additional_backend_pattern.match(k)
+            file_match = additional_backend_file_pattern.match(k)
             if match:
                 server = match.group(1)
-                settings = [x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())]
+                settings.extend([x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())])
+            elif file_match:
+                server = file_match.group(1)
+                try:
+                    with open(v) as file:
+                        for line in file:
+                            settings.append(line.strip())
+                except Exception as e:
+                  logger.info("Error reading %s at '%s', error %s" % (k, v, e))
+
+            if len(settings) > 0:
                 if server in settings_dict:
                     settings_dict[server].extend(settings)
                 else:
@@ -61,8 +89,10 @@ def parse_additional_backend_settings(envvars):
 DEFAULT_SSL_CERT = os.getenv("DEFAULT_SSL_CERT") or os.getenv("SSL_CERT")
 EXTRA_BIND_SETTINGS = parse_extra_bind_settings(os.getenv("EXTRA_BIND_SETTINGS"))
 EXTRA_DEFAULT_SETTINGS = os.getenv("EXTRA_DEFAULT_SETTINGS")
+EXTRA_DEFAULT_SETTINGS_FILE = os.getenv("EXTRA_DEFAULT_SETTINGS_FILE")
 EXTRA_FRONTEND_SETTINGS = parse_extra_frontend_settings(os.environ)
 EXTRA_GLOBAL_SETTINGS = os.getenv("EXTRA_GLOBAL_SETTINGS")
+EXTRA_GLOBAL_SETTINGS_FILE = os.getenv("EXTRA_GLOBAL_SETTINGS_FILE")
 EXTRA_SSL_CERT = os.getenv("EXTRA_SSL_CERTS")
 EXTRA_ROUTE_SETTINGS = os.getenv("EXTRA_ROUTE_SETTINGS", "")
 FORCE_DEFAULT_BACKEND = os.getenv("FORCE_DEFAULT_BACKEND", "True")

haproxy/haproxycfg.py

@@ -256,7 +256,15 @@ def _config_global_section():
         statements.extend(ConfigHelper.config_ssl_bind_options(SSL_BIND_OPTIONS))
         statements.extend(ConfigHelper.config_ssl_bind_ciphers(SSL_BIND_CIPHERS))
         statements.extend(ConfigHelper.config_extra_settings(EXTRA_GLOBAL_SETTINGS))
+        if EXTRA_GLOBAL_SETTINGS_FILE:
+            try:
+                with open(EXTRA_GLOBAL_SETTINGS_FILE) as file:
+                    for line in file:
+                        statements.append(line.strip())
+            except Exception as e:
+                logger.info("Error reading EXTRA_GLOBAL_SETTINGS_FILE at '%s', error %s" % (EXTRA_GLOBAL_SETTINGS_FILE, e))
         cfg["global"] = statements
+      
         return cfg
 
     @staticmethod
@@ -285,7 +293,13 @@ def _config_defaults_section():
         statements.extend(ConfigHelper.config_option(OPTION))
         statements.extend(ConfigHelper.config_timeout(TIMEOUT))
         statements.extend(ConfigHelper.config_extra_settings(EXTRA_DEFAULT_SETTINGS))
-
+        if EXTRA_DEFAULT_SETTINGS_FILE:
+            try:
+                with open(EXTRA_DEFAULT_SETTINGS_FILE) as file:
+                    for line in file:
+                        statements.append(line.strip())
+            except Exception as e:
+                logger.info("Error reading EXTRA_DEFAULT_SETTINGS_FILE at '%s', error %s" % (EXTRA_DEFAULT_SETTINGS_FILE, e))
         cfg["defaults"] = statements
         return cfg
 

tests/unit/test_conifg.py tests/unit/test_config.pytests/unit/test_conifg.py renamed to tests/unit/test_config.py

@@ -1,5 +1,6 @@
 import unittest
-
+import tempfile
+import time
 from haproxy.config import parse_extra_bind_settings, parse_extra_frontend_settings
 
 
@@ -18,14 +19,23 @@ class ParseExtraFrontendSettings(unittest.TestCase):
     def test_parse_extra_frontend_settings(self):
         self.assertEqual({}, parse_extra_frontend_settings(""))
         self.assertEqual({}, parse_extra_frontend_settings({}))
-
+        tf = tempfile.NamedTemporaryFile();
+        tf.write("reqadd file_header value99")
+        tf.flush()
+        
         envvars = {"EXTRA_FRONTEND_SETTINGS_443": " reqadd header1 value1, reqadd header2 va\,lue2,"
                                                   "  reqadd header3 value3 ",
+                   "EXTRA_FRONTEND_SETTINGS_FILE_443": tf.name,
                    "EXTRA_FRONTEND_SETTINGS_80": "reqadd header4",
                    "EXTRA_FRONTEND_SETTINGS_8080": "",
                    "EXTRA_FRONTEND_SETTINGS_ABC": "reqadd header5",
                    "EXTRA_FRONTEND_SETTINGS_": "reqadd header6"}
-        settings = {"443": ["reqadd header1 value1", "reqadd header2 va,lue2", "reqadd header3 value3"],
+        settings = {"443": ["reqadd file_header value99","reqadd header1 value1", "reqadd header2 va,lue2", "reqadd header3 value3"],
                     "80": ["reqadd header4"],
                     "8080": [""]}
         self.assertEqual(settings, parse_extra_frontend_settings(envvars))
+        
+
+
+
+