Merge pull request #44 from docker/staging

ver 1.4

Author: Feng Honglin

Dockerfile

@@ -1,36 +1,16 @@
 FROM alpine:edge
 MAINTAINER Feng Honglin <[email protected]>
 
-# Install tini, haproxy, pip and the dockercloud-haproxy python package:
-RUN apk --no-cache add \
-    tini \
-    haproxy \
-    py-pip \
-  && apk --no-cache add --virtual deps git \
-  && pip install --upgrade \
-    pip \
-  && apk del deps \
-  # Clean up obsolete files:
-  && rm -rf \
-    # Clean up any temporary files:
-    /tmp/* \
-    # Clean up the pip cache:
-    /root/.cache \
-    # Remove any compiled python files (compile on demand):
-    `find / -regex '.*\.py[co]'`
+COPY . /haproxy-src
 
-COPY reload.sh /reload.sh
-COPY . haproxy-src/
-RUN cd /haproxy-src/ && \
-  pip install . \
-  # Clean up obsolete files:
-  && rm -rf \
-    # Clean up any temporary files:
-    /tmp/* \
-    # Clean up the pip cache:
-    /root/.cache \
-    # Remove any compiled python files (compile on demand):
-    `find / -regex '.*\.py[co]'`
+RUN apk update && \
+    apk --no-cache add tini haproxy py-pip build-base python-dev ca-certificates && \
+    cp /haproxy-src/reload.sh /reload.sh && \
+    cd /haproxy-src && \
+    pip install -r requirements.txt && \
+    pip install . && \
+    apk del build-base python-dev && \
+    rm -rf "/tmp/*" "/root/.cache" `find / -regex '.*\.py[co]'`
 
 ENV RSYSLOG_DESTINATION=127.0.0.1 \
     MODE=http \
@@ -42,7 +22,7 @@ ENV RSYSLOG_DESTINATION=127.0.0.1 \
     STATS_AUTH="stats:stats" \
     SSL_BIND_OPTIONS=no-sslv3 \
     SSL_BIND_CIPHERS="ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:DHE-DSS-AES128-SHA:DES-CBC3-SHA" \
-    HEALTH_CHECK="check"
+    HEALTH_CHECK="check inter 2000 rise 2 fall 3"
 
 EXPOSE 80 443 1936
 ENTRYPOINT ["tini", "--"]

README.md

@@ -132,25 +132,26 @@ Settings in this part is immutable, you have to redeploy HAProxy service to make
 
 |Environment Variable|Default|Description|
 |:-----:|:-----:|:----------|
-|ADDITIONAL_SERVICES||list of additional services to balance (es: `prj1:web,prj2:sql`). Discovery will be based on `com.docker.compose.[project|service]` container labels. This environment variable only works on compose v2, and the referenced services must be on a network accessible to this containers.|
+|ADDITIONAL_SERVICES| |list of additional services to balance (es: `prj1:web,prj2:sql`). Discovery will be based on `com.docker.compose.[project|service]` container labels. This environment variable only works on compose v2, and the referenced services must be on a network accessible to this containers.|
 |BALANCE|roundrobin|load balancing algorithm to use. Possible values include: `roundrobin`, `static-rr`, `source`, `leastconn`. See:[HAProxy:balance](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-balance)|
-|CA_CERT_FILE||the path of a ca-cert file. This allows you to mount your ca-cert file directly from a volume instead of from envvar. If set, `CA_CERT` envvar will be ignored. Possible value: `/cacerts/cert0.pem`|
-|CA_CERT|<empty>|CA cert for haproxy to verify the client. Use the same format as `DEFAULT_SSL_CERT`|
-|CERT_FOLDER||the path of certificates. This allows you to mount your certificate files directly from a volume instead of from envvars. If set, `DEFAULT_SSL_CERT` and `SSL_CERT` from linked services are ignored. Possible value:`/certs/`|
-|DEFAULT_SSL_CERT||Default ssl cert, a pem file content with private key followed by public certificate, '\n'(two chars) as the line separator. should be formatted as one line - see [SSL Termination](#ssl-termination)|
-|EXTRA_BIND_SETTINGS|<empty>|comma-separated string(`<port>:<setting>`) of extra settings, and each part will be appended to the related port bind section in the configuration file. To escape comma, use `\,`. Possible value: `443:accept-proxy, 80:name http`|
-|EXTRA_DEFAULT_SETTINGS|<empty>|comma-separated string of extra settings, and each part will be appended to DEFAULT section in the configuration file. To escape comma, use `\,`|
-|EXTRA_GLOBAL_SETTINGS|<empty>|comma-separated string of extra settings, and each part will be appended to GLOBAL section in the configuration file. To escape comma, use `\,`. Possible value: `tune.ssl.cachesize 20000, tune.ssl.default-dh-param 2048`|
-|EXTRA_SSL_CERTS||list of extra certificate names separated by comma, eg. `CERT1, CERT2, CERT3`. You also need to specify each certificate as separate env variables like so: `CERT1="<cert-body1>"`, `CERT2="<cert-body2>"`, `CERT3="<cert-body3>"`|
+|CA_CERT_FILE| |the path of a ca-cert file. This allows you to mount your ca-cert file directly from a volume instead of from envvar. If set, `CA_CERT` envvar will be ignored. Possible value: `/cacerts/cert0.pem`|
+|CA_CERT| |CA cert for haproxy to verify the client. Use the same format as `DEFAULT_SSL_CERT`|
+|CERT_FOLDER| |the path of certificates. This allows you to mount your certificate files directly from a volume instead of from envvars. If set, `DEFAULT_SSL_CERT` and `SSL_CERT` from linked services are ignored. Possible value:`/certs/`|
+|DEFAULT_SSL_CERT| |Default ssl cert, a pem file content with private key followed by public certificate, '\n'(two chars) as the line separator. should be formatted as one line - see [SSL Termination](#ssl-termination)|
+|EXTRA_BIND_SETTINGS| |comma-separated string(`<port>:<setting>`) of extra settings, and each part will be appended to the related port bind section in the configuration file. To escape comma, use `\,`. Possible value: `443:accept-proxy, 80:name http`|
+|EXTRA_DEFAULT_SETTINGS| |comma-separated string of extra settings, and each part will be appended to DEFAULT section in the configuration file. To escape comma, use `\,`|
+|EXTRA_FRONTEND_SETTINGS_\<PORT\>| |comma-separated string of extra settings, and each part will be appended frontend section with the port number specified in the name of the envvar. To escape comma, use `\,`. E.g. `EXTRA_FRONTEND_SETTINGS_80=balance source, maxconn 2000`|
+|EXTRA_GLOBAL_SETTINGS| |comma-separated string of extra settings, and each part will be appended to GLOBAL section in the configuration file. To escape comma, use `\,`. Possible value: `tune.ssl.cachesize 20000, tune.ssl.default-dh-param 2048`|
+|EXTRA_SSL_CERTS| |list of extra certificate names separated by comma, eg. `CERT1, CERT2, CERT3`. You also need to specify each certificate as separate env variables like so: `CERT1="<cert-body1>"`, `CERT2="<cert-body2>"`, `CERT3="<cert-body3>"`|
 |HEALTH_CHECK|check|set health check on each backend route, possible value: "check inter 2000 rise 2 fall 3". See:[HAProxy:check](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5.2-check)|
-|HTTP_BASIC_AUTH|<empty>|a comma-separated list of credentials(`<user>:<pass>`) for HTTP basic auth, which applies to all the backend routes. To escape comma, use `\,`. *Attention:* DO NOT rely on this for authentication in production|
+|HTTP_BASIC_AUTH| |a comma-separated list of credentials(`<user>:<pass>`) for HTTP basic auth, which applies to all the backend routes. To escape comma, use `\,`. *Attention:* DO NOT rely on this for authentication in production|
 |MAXCONN|4096|sets the maximum per-process number of concurrent connections.|
 |MODE|http|mode of load balancing for HAProxy. Possible values include: `http`, `tcp`, `health`|
-|MONITOR_PORT|<empty>|the port number where monitor_uri should be added to. Use together with `MONTIOR_URI`. Possible value: `80`|
-|MONITOR_URI|<empty>|the exact URI which we want to intercept to return HAProxy's health status instead of forwarding the request.See: http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-monitor-uri. Possible value: `/ping`|
+|MONITOR_PORT| |the port number where monitor_uri should be added to. Use together with `MONTIOR_URI`. Possible value: `80`|
+|MONITOR_URI| |the exact URI which we want to intercept to return HAProxy's health status instead of forwarding the request.See: http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-monitor-uri. Possible value: `/ping`|
 |OPTION|redispatch|comma-separated list of HAProxy `option` entries to the `default` section.|
 |RSYSLOG_DESTINATION|127.0.0.1|the rsyslog destination to where HAProxy logs are sent|
-|SSL_BIND_CIPHERS||explicitly set which SSL ciphers will be used for the SSL server. This sets the HAProxy `ssl-default-bind-ciphers` configuration setting.|
+|SSL_BIND_CIPHERS| |explicitly set which SSL ciphers will be used for the SSL server. This sets the HAProxy `ssl-default-bind-ciphers` configuration setting.|
 |SSL_BIND_OPTIONS|no-sslv3|explicitly set which SSL bind options will be used for the SSL server. This sets the HAProxy `ssl-default-bind-options` configuration setting. The default will allow only TLSv1.0+ to be used on the SSL server.|
 |STATS_AUTH|stats:stats|username and password required to access the Haproxy stats.|
 |STATS_PORT|1936|port for the HAProxy stats section. If this port is published, stats can be accessed at `http://<host-ip>:<STATS_PORT>/`
@@ -160,7 +161,7 @@ Settings in this part is immutable, you have to redeploy HAProxy service to make
 
 Settings here can overwrite the settings in HAProxy, which are only applied to the linked services. If run in Docker Cloud, when the service redeploys, joins or leaves HAProxy service, HAProxy service will automatically update itself to apply the changes
 
-|env var|description|
+|Environment Variable|Description|
 |:-----:|:----------|
 |APPSESSION|sticky session option, possible value `JSESSIONID len 52 timeout 3h`. See:[HAProxy:appsession](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-appsession)|
 |BALANCE|load balancing algorithm to use. Possible values include: `roundrobin`, `static-rr`, `source`, `leastconn`. See:[HAProxy:balance](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-balance)|
@@ -185,7 +186,7 @@ Check [the HAProxy configuration manual](http://cbonte.github.io/haproxy-dconv/c
 
 Both virtual host and virtual path can be specified in environment variable `VIRTUAL_HOST`, which is a set of comma separated urls with the format of `[scheme://]domain[:port][/path]`.
 
-|item|default|description|
+|Item|Default|Description|
 |:---:|:-----:|:---------|
 |scheme|http|possible values: `http`, `https`, `wss`|
 |domain||virtual host. `*` can be used as the wildcard|
@@ -194,7 +195,7 @@ Both virtual host and virtual path can be specified in environment variable `VIR
 
 #### examples of matching
 
-|virtual host|match|not match|
+|Virtual host|Match|Not match|
 |:-----------|:----|:--------|
 |http://example.com|example.com|www.example.com|
 |example.com|example.com|www.example.com|

haproxy/__init__.py

@@ -1 +1 @@
-__version__ = "1.3"
+__version__ = "1.4"

haproxy/config.py

@@ -13,34 +13,53 @@ def parse_extra_bind_settings(extra_bind_settings):
     return bind_dict
 
 
+def parse_extra_frontend_settings(envvars):
+    settings_dict = {}
+    if isinstance(envvars, os._Environ) or isinstance(envvars, dict):
+        frontend_settings_pattern = re.compile(r"^EXTRA_FRONTEND_SETTINGS_(\d{1,5})$")
+        for k, v in envvars.iteritems():
+            match = frontend_settings_pattern.match(k)
+            if match:
+                port = match.group(1)
+                settings = [x.strip().replace("\,", ",") for x in re.split(r'(?<!\\),', v.strip())]
+                if port in settings_dict:
+                    settings_dict[port].extend(settings)
+                else:
+                    settings_dict[port] = settings
+    return settings_dict
+
+
 # envvar
+ADDITIONAL_SERVICES = os.getenv("ADDITIONAL_SERVICES")
+API_AUTH = os.getenv("DOCKERCLOUD_AUTH")
+BALANCE = os.getenv("BALANCE", "roundrobin")
+CA_CERT_FILE = os.getenv("CA_CERT_FILE")
+CERT_FOLDER = os.getenv("CERT_FOLDER")
+DEBUG = os.getenv("DEBUG", False)
+DEFAULT_CA_CERT = os.getenv("CA_CERT")
 DEFAULT_SSL_CERT = os.getenv("DEFAULT_SSL_CERT") or os.getenv("SSL_CERT")
+EXTRA_BIND_SETTINGS = parse_extra_bind_settings(os.getenv("EXTRA_BIND_SETTINGS"))
+EXTRA_DEFAULT_SETTINGS = os.getenv("EXTRA_DEFAULT_SETTINGS")
+EXTRA_FRONTEND_SETTINGS = parse_extra_frontend_settings(os.environ)
+EXTRA_GLOBAL_SETTINGS = os.getenv("EXTRA_GLOBAL_SETTINGS")
 EXTRA_SSL_CERT = os.getenv("EXTRA_SSL_CERTS")
-DEFAULT_CA_CERT = os.getenv("CA_CERT")
+HAPROXY_CONTAINER_URI = os.getenv("DOCKERCLOUD_CONTAINER_API_URI")
+HAPROXY_SERVICE_URI = os.getenv("DOCKERCLOUD_SERVICE_API_URI")
+HEALTH_CHECK = os.getenv("HEALTH_CHECK", "check inter 2000 rise 2 fall 3")
+HTTP_BASIC_AUTH = os.getenv("HTTP_BASIC_AUTH")
 MAXCONN = os.getenv("MAXCONN", "4096")
 MODE = os.getenv("MODE", "http")
+MONITOR_PORT = os.getenv("MONITOR_PORT")
+MONITOR_URI = os.getenv("MONITOR_URI")
 OPTION = os.getenv("OPTION", "redispatch, httplog, dontlognull, forwardfor")
 RSYSLOG_DESTINATION = os.getenv("RSYSLOG_DESTINATION", "127.0.0.1")
 SSL_BIND_CIPHERS = os.getenv("SSL_BIND_CIPHERS")
 SSL_BIND_OPTIONS = os.getenv("SSL_BIND_OPTIONS")
 STATS_AUTH = os.getenv("STATS_AUTH", "stats:stats")
 STATS_PORT = os.getenv("STATS_PORT", "1936")
 TIMEOUT = os.getenv("TIMEOUT", "connect 5000, client 50000, server 50000")
-HEALTH_CHECK = os.getenv("HEALTH_CHECK", "check inter 2000 rise 2 fall 3")
-EXTRA_GLOBAL_SETTINGS = os.getenv("EXTRA_GLOBAL_SETTINGS")
-EXTRA_DEFAULT_SETTINGS = os.getenv("EXTRA_DEFAULT_SETTINGS")
-EXTRA_BIND_SETTINGS = parse_extra_bind_settings(os.getenv("EXTRA_BIND_SETTINGS"))
-HTTP_BASIC_AUTH = os.getenv("HTTP_BASIC_AUTH")
-MONITOR_URI = os.getenv("MONITOR_URI")
-MONITOR_PORT = os.getenv("MONITOR_PORT")
-BALANCE = os.getenv("BALANCE", "roundrobin")
-HAPROXY_CONTAINER_URI = os.getenv("DOCKERCLOUD_CONTAINER_API_URI")
-HAPROXY_SERVICE_URI = os.getenv("DOCKERCLOUD_SERVICE_API_URI")
-API_AUTH = os.getenv("DOCKERCLOUD_AUTH")
-CA_CERT_FILE = os.getenv("CA_CERT_FILE")
-CERT_FOLDER = os.getenv("CERT_FOLDER")
-ADDITIONAL_SERVICES = os.environ.get("ADDITIONAL_SERVICES")
-DEBUG = os.getenv("DEBUG", False)
+
+# global
 LINK_MODE = ""
 
 # const

haproxy/haproxycfg.py

@@ -309,13 +309,12 @@ def _config_backend_sections(self):
         for service_alias in services_aliases:
             backend = BackendHelper.get_backend_section(details, routes, vhosts, service_alias, self.routes_added)
 
-            if BackendHelper.check_backend_has_routes(backend):
-                if not service_alias:
-                    if self.require_default_route:
-                        cfg["backend default_service"] = backend
+            if not service_alias:
+                if self.require_default_route:
+                    cfg["backend default_service"] = backend
+            else:
+                if get_service_attribute(details, "virtual_host", service_alias):
+                    cfg["backend SERVICE_%s" % service_alias] = backend
                 else:
-                    if get_service_attribute(details, "virtual_host", service_alias):
-                        cfg["backend SERVICE_%s" % service_alias] = backend
-                    else:
-                        cfg["backend default_service"] = backend
+                    cfg["backend default_service"] = backend
         return cfg

haproxy/helper/backend_helper.py

@@ -155,11 +155,4 @@ def get_basic_auth_setting(basic_auth):
     if basic_auth:
         setting.append("acl need_auth http_auth(haproxy_userlist)")
         setting.append("http-request auth realm haproxy_basic_auth if !need_auth")
-    return setting
-
-
-def check_backend_has_routes(backend):
-    for b in backend:
-        if b.startswith("server"):
-            return True
-    return False
+    return setting

haproxy/helper/cloud_link_helper.py

@@ -1,10 +1,12 @@
-from multiprocessing.pool import ThreadPool
+from gevent.pool import Pool
 
 from haproxy.config import SERVICE_NAME_MATCH
 from haproxy.utils import get_uuid_from_resource_uri, fetch_remote_obj
 
 LINKED_CONTAINER_CACHE = {}
 
+pool = Pool(size=5)
+
 
 def get_cloud_links(haproxy_container):
     links = _init_links(haproxy_container.linked_to_container)
@@ -41,7 +43,7 @@ def _get_new_added_link_uri(container_object_cache, links):
 
 
 def _get_container_object_from_uri(container_uris):
-    pool = ThreadPool(processes=10)
+    global pool
     container_objects = pool.map(fetch_remote_obj, container_uris)
     return container_objects
 

haproxy/helper/frontend_helper.py

@@ -1,6 +1,6 @@
 from collections import OrderedDict
 
-from haproxy.config import EXTRA_BIND_SETTINGS, MONITOR_URI, MONITOR_PORT, MAXCONN
+from haproxy.config import EXTRA_BIND_SETTINGS, EXTRA_FRONTEND_SETTINGS, MONITOR_URI, MONITOR_PORT, MAXCONN
 
 
 def check_require_default_route(routes, routes_added):
@@ -94,16 +94,27 @@ def config_common_part(port, ssl_bind_string, vhosts):
     frontend_section = []
     bind_string, ssl = get_bind_string(port, ssl_bind_string, vhosts)
     frontend_section.append("bind :%s" % bind_string)
+
+    # add x-forwarded-porto header
     if ssl:
         frontend_section.append("reqadd X-Forwarded-Proto:\ https")
+    else:
+        frontend_section.append("reqadd X-Forwarded-Proto:\ http")
 
-    # add websocket acl rule
-    frontend_section.append("acl is_websocket hdr(Upgrade) -i WebSocket")
+    # add maxconn
+    frontend_section.append("maxconn %s" % MAXCONN)
 
     # add monitor uri
     if port == MONITOR_PORT and MONITOR_URI:
         frontend_section.append("monitor-uri %s" % MONITOR_URI)
         monitor_uri_configured = True
+
+    # add extra frontend setting
+    if EXTRA_FRONTEND_SETTINGS and port in EXTRA_FRONTEND_SETTINGS:
+        frontend_section.extend(EXTRA_FRONTEND_SETTINGS[port])
+
+    # add websocket acl rule
+    frontend_section.append("acl is_websocket hdr(Upgrade) -i WebSocket")
     return frontend_section, monitor_uri_configured
 
 
@@ -124,19 +135,32 @@ def get_bind_string(port, ssl_bind_string, vhosts):
 def config_default_frontend(ssl_bind_string):
     cfg = OrderedDict()
     monitor_uri_configured = False
-    frontend = [("bind :80 %s" % EXTRA_BIND_SETTINGS.get('80', "")).strip()]
-    if ssl_bind_string:
-        frontend.append(
-            ("bind :443 %s %s" % (ssl_bind_string, EXTRA_BIND_SETTINGS.get('443', ""))).strip())
-        frontend.append("reqadd X-Forwarded-Proto:\ https")
+    frontend = [("bind :80 %s" % EXTRA_BIND_SETTINGS.get('80', "")).strip(),
+                "reqadd X-Forwarded-Proto:\ http", "maxconn %s" % MAXCONN]
 
-    if MONITOR_URI and (MONITOR_PORT == '80' or MONITOR_PORT == '443'):
+    if MONITOR_URI and MONITOR_PORT == '80':
         frontend.append("monitor-uri %s" % MONITOR_URI)
         monitor_uri_configured = True
 
-    frontend.append("maxconn %s" % MAXCONN)
+    if "80" in EXTRA_FRONTEND_SETTINGS:
+        frontend.extend(EXTRA_FRONTEND_SETTINGS["80"])
+
     frontend.append("default_backend default_service")
-    cfg["frontend default_frontend"] = frontend
+    cfg["frontend default_port_80"] = frontend
+
+    if ssl_bind_string:
+        ssl_frontend = [("bind :443 %s %s" % (ssl_bind_string, EXTRA_BIND_SETTINGS.get('443', ""))).strip(),
+                        "reqadd X-Forwarded-Proto:\ https", "maxconn %s" % MAXCONN]
+
+        if MONITOR_URI and (MONITOR_PORT == '443'):
+            ssl_frontend.append("monitor-uri %s" % MONITOR_URI)
+            monitor_uri_configured = True
+
+        if "443" in EXTRA_FRONTEND_SETTINGS:
+            ssl_frontend.extend(EXTRA_FRONTEND_SETTINGS["443"])
+
+        ssl_frontend.append("default_backend default_service")
+        cfg["frontend default_port_443"] = ssl_frontend
 
     return cfg, monitor_uri_configured
 

haproxy/main.py

@@ -13,6 +13,10 @@
 from haproxycfg import run_haproxy
 from utils import save_to_file
 
+
+from gevent import monkey
+monkey.patch_all()
+
 dockercloud.user_agent = "dockercloud-haproxy/%s" % __version__
 
 logger = logging.getLogger("haproxy")