use uwsgi instead of paste

.gitignore

@@ -24,6 +24,7 @@ parts
 bin
 local
 /src/*_theme
+/lib
 lib64
 eggs
 develop-eggs

CHANGES.txt

@@ -4,6 +4,25 @@ adhocracy Changelog
 1.2beta2dev
 -----------
 
+- Buildout configuration parts changed:
+
+  * adhocracy.host is now domains.wsgi
+  * port.main is now ports.wsgi
+  * domains.main is no adhocracy.domain
+
+- link python lib and inclue directories, uwsgi needs this.
+
+  Upgrade manual installation:
+  ____________________________
+
+  * cd python
+  * bin/buildout 
+  * cd ..
+  * bin/buildout
+
+- run adhocarcy with uwsgi instead of paster to improve performance, 
+  for development keep with paster [joka]
+
 - added varnish reverse proxy [joka]
 
 - Readd bin/test to run tests (wrapper for py.test) [joka]

INSTALLATION.rst

@@ -70,13 +70,13 @@ Check out Adhocracy:
 
     $ git clone https://github.com/liqd/adhocracy
     $ cd adhocracy
-    
+
     $ git submodule init
     $ git submodule update
 
 ## Setup an isolated python environment to run Adhocracy
 
-To install Adhocracy you need python (2.6|2.7) with PIL (python imaging) but 
+To install Adhocracy you need python (2.6|2.7) with PIL (python imaging) but
 no other system-packages.
 
 Compile python and PIL with the included python buildout::
@@ -92,7 +92,7 @@ Run buildout:
 
 ::
 
-    $ bin/python bootstrap.py 
+    $ bin/python bootstrap.py
     $ bin/buildout
 
 Start Adhocracy and dependent servers:
@@ -109,6 +109,7 @@ have to setup the Adhocracy database manually:
 
     $ bin/paster setup-app etc/adhocracy.ini --name=content
 
+
 Run Adhocracy
 -------------
 
@@ -143,6 +144,14 @@ Start the Adhocracy server in foreground mode:
     $ bin/supervisorctl stop adhocracy
     $ bin/paster serve etc/adhocracy.ini
 
+To run adhocarcy with uwsgi instead of paster:
+
+::
+    $ bin/uwsgi --ini-paste etc/adhocracy.ini
+
+To run the ipdb python debuger with uwsgi use "IPDB()" instead of "ipdb.set_trace()".
+
+
 Buildout configuration
 ----------------------
 

build.sh

@@ -274,7 +274,7 @@ if [ '!' -e python/buildout.python/src ]; then
 fi
 
 # Install local python if necessary
-if [ '!' -x bin/python ]; then
+if [ '!' -x bin/python ] || [ '!' -e lib ]; then
     if [ '!' -f python/bin/buildout ]; then
         (cd python && python bootstrap.py)
     fi

buildout.cfg

@@ -11,6 +11,7 @@ extends =
     buildouts/base.cfg
     buildouts/lxml.cfg
     buildouts/adhocracy.cfg
+    buildouts/uwsgi.cfg
 #    buildouts/adhocracy_theming.cfg
     buildouts/stylesheets.cfg
     buildouts/memcache.cfg

buildouts/adhocracy.cfg

@@ -41,26 +41,25 @@ parts +=
 ##############################################################################
 
 [domains]
-main = adhocracy.lan
+wsgi = 127.0.0.1
 
 [ports]
-main = 5001
+wsgi = 5001
 supervisor = 5010
 
 ##############################################################################
 # Adhocracy settings
 ##############################################################################
 
 [adhocracy]
+domain = adhocracy.lan
 #start adhocracy in debug mode
 debug = False
 #TODO use geo features (You also need to include postgres_with_postgis instead of postgres)
 geo = False
 # theme adhocracy with diazo and merge a wordpress blog.
 themed = False
-
 protocol = http
-host = 127.0.0.1
 #Session secret, replace this with a randomly generated hash value or use "autogenerated" to let
 #buildout do the job
 secret = autogenerated
@@ -152,18 +151,19 @@ command =
 
 # python with all required eggs
 [adhocpy]
-recipe = zc.recipe.egg
+recipe = minitage.recipe.scripts 
 eggs =  ${buildout:eggs}
 extra-paths = ./
 interpreter = adhocpy
 scripts = adhocpy
+env-file = ${buildout:bin-directory}/adhocpy-env
 
 ##############################################################################
 # Set up supervisor to run it all
 ##############################################################################
 
 [supervisor]
-environment = LD_LIBRARY_PATH="${buildout:directory}/python/python-2.7/lib/"
+environment = LD_LIBRARY_PATH="${buildout:directory}/lib/"
 programs +=
     ${buildout:adhocracy_worker-supervisor}
     ${buildout:adhocracy-supervisor}

buildouts/developtools.cfg

@@ -37,17 +37,16 @@ eggs += ZopeSkel
 
 [adhocracy]
 debug = True
+# generate internal urls with port 5001 to be able to work without a
+# proxy (e.g. apache with ReverseProxy, see /etc/vhost.ini.in)
+# on a port != 80
+force_port = True
 smtp_port = 8825
 relative_urls = True
 sqlalchemy.url = sqlite:///${buildout:directory}/var/development.db
 
 [ports]
 mailserver_port = ${adhocracy:smtp_port}
-[domains]
-# generate internal urls with port 5001 to be able to work without a
-# proxy (e.g. apache with ReverseProxy, see /etc/vhost.ini.in)
-# on a port != 80
-main.force_port = True
 
 [supervisor]
 programs -=

buildouts/uwsgi.cfg

@@ -0,0 +1,118 @@
+##############################################################################
+# Buildout to install uwsgi https://github.com/unbit/uwsgi
+#
+# requires:
+#
+#   - bin/adhocracpy-env to set the PYTHONPATH variable
+#
+##############################################################################
+
+[buildout]
+
+parts +=
+    uwsgi
+    uwsgi_bin
+
+adhocracy-supervisor =
+    45 adhocracy (environment=${supervisor:environment} stopsignal=INT redirect_stderr=true stdout_logfile=var/log/adhocracy.log stderr_logfile=NONE) ${buildout:bin-directory}/uwsgi [--ini-paste ${buildout:directory}/etc/adhocracy.ini]
+
+##############################################################################
+# System settings
+##############################################################################
+
+[domains]
+wsgi = 127.0.0.1
+
+[ports]
+wsgi = 5001
+
+[urls]
+uwsgi = http://projects.unbit.it/downloads/uwsgi-1.9.13.tar.gz
+
+###############################
+# Install and configure uwsgi #
+###############################
+
+# compile options
+[uwsgi_build_conf]
+recipe = collective.recipe.template
+output = ${buildout:directory}/etc/uwsgi_buildconf.ini
+input = ${buildout:directory}/etc/uwsgi_buildconf.ini.in 
+bin-name = ${buildout:bin-directory}/uwsgi_original
+plugin-dir = ${buildout:directory}/var/uwsgi_plugins
+main-plugin = python, gevent
+
+[uwsgi_build_env]
+UWSGI_PROFILE = ${uwsgi_build_conf:output}
+PYTHONHOME = ${buildout:directory}
+UWSGI_INCLUDES = ${buildout:directory}/lib
+
+# compile egg
+[uwsgi]
+recipe = minitage.recipe.scripts 
+url = ${urls:uwsgi}
+eggs = 
+    ${buildout:eggs} 
+    uwsgi
+environment = uwsgi_build_env
+
+# uwsgi bin
+[uwsgi_bin]
+recipe = collective.recipe.template
+output = ${buildout:bin-directory}/uwsgi
+input = inline:
+    #!/bin/bash
+    source ${adhocpy:env-file}
+    LD_LIBRARY_PATH = ${uwsgi_build_env:UWSGI_INCLUDES}
+    export LD_LIBRARY_PATH
+    exec ${buildout:bin-directory}/uwsgi_original "$@"
+mode = 755 
+
+# configure
+[uwsgi_conf]
+conf =
+    #basics
+
+    http = ${domains:wsgi}:${ports:wsgi} 
+    pidfile = ${buildout:directory}/var/uwsgi.pid 
+    master = true
+    chdir = ${buildout:directory}/src
+    home = ${buildout:directory}
+
+    # extended
+
+    # Reload if ini changes
+    touch-reload = ${buildout:directory}/etc/adhocracy.ini  
+    # stop uwsgi if the wsgi app is not available
+    need-app = true
+    # allow threads
+    enable-threads = true
+    #Set close-on-exec on sockets (could be required for spawning processes in requests).
+    close-on-exec = true
+    #Automatically kill workers if master dies (can be dangerous for availability).
+    no-orphans = true
+    # Try to remove all of the generated files/sockets (UNIX sockets and pidfiles) upon exit.
+    vacuum = true
+
+    #performance tuning
+
+    processes = 1
+    worker = 20
+    post-buffering = 4096
+    max-requests = 1000
+    # Reload a worker if its address space usage is higher than the specified value (in megabytes).
+    reload-on-as = 128 
+    #gracefull dead
+    harakiri = 60
+    harakiri-verbose = true
+
+    #logging
+
+    log-date = true
+    log-slow = true
+
+    # debug
+
+    # IPDB(); opens python shell
+    shared-import = ${buildout:directory}/scripts/interact.py
+    # memory-report = true

etc/adhocracy.ini.in

@@ -7,8 +7,6 @@
 ############################################################## 
 
 [DEFAULT]
-{% if parts.adhocracy.debug == 'True' %}debug = True  {% end %}
-{% if parts.adhocracy.debug != 'True' %}debug = False {% end %}
 
 # Uncomment and replace with the address which should receive any error reports
 email_to = ${parts.adhocracy['email.to']}
@@ -18,13 +16,16 @@ error_email_from = ${parts.adhocracy['email.from']}
 
 [server:main]
 use = egg:Paste#http
-host = ${parts.adhocracy.host}
-port = ${parts.ports.main}
+host = ${parts.domains.wsgi}
+port = ${parts.ports.wsgi}
 {% if parts.adhocracy.debug == 'True' %}use_threadpool = False  {% end %}
 {% if parts.adhocracy.debug != 'True' %}use_threadpool = True {% end %}
 threadpool_workers = 20
 
-
+{% if "uwsgi_conf" in parts %}
+[uwsgi]
+${parts.uwsgi_conf.conf}
+{% end %}
 
 {% if parts.adhocracy.themed == 'False' %}
 ###################not themed adhocracy ###################
@@ -166,9 +167,9 @@ adhocracy.interactive_debugging = true
 # active instance, e.g. "Test Instance" for test.adhocracy.lan.
 
 {% python
-adhocracy_domain = parts['domains']['main']
-if parts['domains'].get('main.force_port', None):
-   adhocracy_domain += ':%s' % parts['ports']['main']
+adhocracy_domain = parts['adhocracy']['domain']
+if parts['domains'].get('adhocracy.force_port', None):
+   adhocracy_domain += ':%s' % parts['ports']['wsgi']
 %}
 adhocracy.domain = ${adhocracy_domain}
 adhocracy.protocol = ${parts.adhocracy.protocol}

etc/uwsgi_buildconf.ini.in

@@ -0,0 +1,50 @@
+; this is mainly copied from the uwsgi buildconf/default.ini
+; trying to surface some of the build options to buildout
+; properties.
+[uwsgi]
+bin_name = ${:bin-name}
+plugin_dir = ${:plugin-dir}
+main_plugin = ${:main-plugin}
+xml = false
+ini = true
+yaml = true
+json = auto
+sqlite3 = auto
+zeromq = auto
+snmp = true
+spooler = true
+embedded = true
+ssl = auto
+udp = true
+multicast = true
+threading = true
+minterpreters = true
+async = true
+ldap = auto
+pcre = auto
+routing = auto
+alarm = auto
+ipv6 = false
+debug = false
+unbit = false
+xml_implementation = libxml2
+yaml_implementation = auto
+malloc_implementation = libc
+extras =
+plugins =
+append_version =
+embedded_plugins = %(main_plugin)s, ping, cache, nagios, rrdtool, carbon, rpc, corerouter, fastrouter, http, ugreen, signal, syslog, rsyslog, logsocket, router_uwsgi, router_redirect, router_basicauth, zergpool, redislog, mongodblog, router_rewrite, router_http, logfile, router_cache, rawrouter
+as_shared_library = false
+
+locking = auto
+event = auto
+timer = auto
+filemonitor = auto
+
+blacklist =
+whitelist =
+
+embed_files =
+
+embed_config =
+