diff --git a/.gitignore b/.gitignore index 88a947542..d54f85968 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ parts bin local /src/*_theme +/lib lib64 eggs develop-eggs diff --git a/CHANGES.txt b/CHANGES.txt index 29313bda2..02e8839e3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,25 @@ adhocracy Changelog 1.2beta2dev ----------- +- Buildout configuration parts changed: + + * adhocracy.host is now domains.wsgi + * port.main is now ports.wsgi + * domains.main is no adhocracy.domain + +- link python lib and inclue directories, uwsgi needs this. + + Upgrade manual installation: + ____________________________ + + * cd python + * bin/buildout + * cd .. + * bin/buildout + +- run adhocarcy with uwsgi instead of paster to improve performance, + for development keep with paster [joka] + - added varnish reverse proxy [joka] - Readd bin/test to run tests (wrapper for py.test) [joka] diff --git a/INSTALLATION.rst b/INSTALLATION.rst index e44e2e888..02e8ec47e 100644 --- a/INSTALLATION.rst +++ b/INSTALLATION.rst @@ -70,13 +70,13 @@ Check out Adhocracy: $ git clone https://github.com/liqd/adhocracy $ cd adhocracy - + $ git submodule init $ git submodule update ## Setup an isolated python environment to run Adhocracy -To install Adhocracy you need python (2.6|2.7) with PIL (python imaging) but +To install Adhocracy you need python (2.6|2.7) with PIL (python imaging) but no other system-packages. Compile python and PIL with the included python buildout:: @@ -92,7 +92,7 @@ Run buildout: :: - $ bin/python bootstrap.py + $ bin/python bootstrap.py $ bin/buildout Start Adhocracy and dependent servers: @@ -109,6 +109,7 @@ have to setup the Adhocracy database manually: $ bin/paster setup-app etc/adhocracy.ini --name=content + Run Adhocracy ------------- @@ -143,6 +144,14 @@ Start the Adhocracy server in foreground mode: $ bin/supervisorctl stop adhocracy $ bin/paster serve etc/adhocracy.ini +To run adhocarcy with uwsgi instead of paster: + +:: + $ bin/uwsgi --ini-paste etc/adhocracy.ini + +To run the ipdb python debuger with uwsgi use "IPDB()" instead of "ipdb.set_trace()". + + Buildout configuration ---------------------- diff --git a/build.sh b/build.sh index 054037992..75924b5d1 100755 --- a/build.sh +++ b/build.sh @@ -274,7 +274,7 @@ if [ '!' -e python/buildout.python/src ]; then fi # Install local python if necessary -if [ '!' -x bin/python ]; then +if [ '!' -x bin/python ] || [ '!' -e lib ]; then if [ '!' -f python/bin/buildout ]; then (cd python && python bootstrap.py) fi diff --git a/buildout.cfg b/buildout.cfg index 0d2592b27..c06e15199 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -11,6 +11,7 @@ extends = buildouts/base.cfg buildouts/lxml.cfg buildouts/adhocracy.cfg + buildouts/uwsgi.cfg # buildouts/adhocracy_theming.cfg buildouts/stylesheets.cfg buildouts/memcache.cfg diff --git a/buildouts/adhocracy.cfg b/buildouts/adhocracy.cfg index 600c2b141..8ab9aea78 100644 --- a/buildouts/adhocracy.cfg +++ b/buildouts/adhocracy.cfg @@ -41,10 +41,10 @@ parts += ############################################################################## [domains] -main = adhocracy.lan +wsgi = 127.0.0.1 [ports] -main = 5001 +wsgi = 5001 supervisor = 5010 ############################################################################## @@ -52,15 +52,14 @@ supervisor = 5010 ############################################################################## [adhocracy] +domain = adhocracy.lan #start adhocracy in debug mode debug = False #TODO use geo features (You also need to include postgres_with_postgis instead of postgres) geo = False # theme adhocracy with diazo and merge a wordpress blog. themed = False - protocol = http -host = 127.0.0.1 #Session secret, replace this with a randomly generated hash value or use "autogenerated" to let #buildout do the job secret = autogenerated @@ -152,18 +151,19 @@ command = # python with all required eggs [adhocpy] -recipe = zc.recipe.egg +recipe = minitage.recipe.scripts eggs = ${buildout:eggs} extra-paths = ./ interpreter = adhocpy scripts = adhocpy +env-file = ${buildout:bin-directory}/adhocpy-env ############################################################################## # Set up supervisor to run it all ############################################################################## [supervisor] -environment = LD_LIBRARY_PATH="${buildout:directory}/python/python-2.7/lib/" +environment = LD_LIBRARY_PATH="${buildout:directory}/lib/" programs += ${buildout:adhocracy_worker-supervisor} ${buildout:adhocracy-supervisor} diff --git a/buildouts/developtools.cfg b/buildouts/developtools.cfg index 1af7c8bad..b67a9af26 100644 --- a/buildouts/developtools.cfg +++ b/buildouts/developtools.cfg @@ -37,17 +37,16 @@ eggs += ZopeSkel [adhocracy] debug = True +# generate internal urls with port 5001 to be able to work without a +# proxy (e.g. apache with ReverseProxy, see /etc/vhost.ini.in) +# on a port != 80 +force_port = True smtp_port = 8825 relative_urls = True sqlalchemy.url = sqlite:///${buildout:directory}/var/development.db [ports] mailserver_port = ${adhocracy:smtp_port} -[domains] -# generate internal urls with port 5001 to be able to work without a -# proxy (e.g. apache with ReverseProxy, see /etc/vhost.ini.in) -# on a port != 80 -main.force_port = True [supervisor] programs -= diff --git a/buildouts/uwsgi.cfg b/buildouts/uwsgi.cfg new file mode 100644 index 000000000..1c94103d5 --- /dev/null +++ b/buildouts/uwsgi.cfg @@ -0,0 +1,118 @@ +############################################################################## +# Buildout to install uwsgi https://github.com/unbit/uwsgi +# +# requires: +# +# - bin/adhocracpy-env to set the PYTHONPATH variable +# +############################################################################## + +[buildout] + +parts += + uwsgi + uwsgi_bin + +adhocracy-supervisor = + 45 adhocracy (environment=${supervisor:environment} stopsignal=INT redirect_stderr=true stdout_logfile=var/log/adhocracy.log stderr_logfile=NONE) ${buildout:bin-directory}/uwsgi [--ini-paste ${buildout:directory}/etc/adhocracy.ini] + +############################################################################## +# System settings +############################################################################## + +[domains] +wsgi = 127.0.0.1 + +[ports] +wsgi = 5001 + +[urls] +uwsgi = http://projects.unbit.it/downloads/uwsgi-1.9.13.tar.gz + +############################### +# Install and configure uwsgi # +############################### + +# compile options +[uwsgi_build_conf] +recipe = collective.recipe.template +output = ${buildout:directory}/etc/uwsgi_buildconf.ini +input = ${buildout:directory}/etc/uwsgi_buildconf.ini.in +bin-name = ${buildout:bin-directory}/uwsgi_original +plugin-dir = ${buildout:directory}/var/uwsgi_plugins +main-plugin = python, gevent + +[uwsgi_build_env] +UWSGI_PROFILE = ${uwsgi_build_conf:output} +PYTHONHOME = ${buildout:directory} +UWSGI_INCLUDES = ${buildout:directory}/lib + +# compile egg +[uwsgi] +recipe = minitage.recipe.scripts +url = ${urls:uwsgi} +eggs = + ${buildout:eggs} + uwsgi +environment = uwsgi_build_env + +# uwsgi bin +[uwsgi_bin] +recipe = collective.recipe.template +output = ${buildout:bin-directory}/uwsgi +input = inline: + #!/bin/bash + source ${adhocpy:env-file} + LD_LIBRARY_PATH = ${uwsgi_build_env:UWSGI_INCLUDES} + export LD_LIBRARY_PATH + exec ${buildout:bin-directory}/uwsgi_original "$@" +mode = 755 + +# configure +[uwsgi_conf] +conf = + #basics + + http = ${domains:wsgi}:${ports:wsgi} + pidfile = ${buildout:directory}/var/uwsgi.pid + master = true + chdir = ${buildout:directory}/src + home = ${buildout:directory} + + # extended + + # Reload if ini changes + touch-reload = ${buildout:directory}/etc/adhocracy.ini + # stop uwsgi if the wsgi app is not available + need-app = true + # allow threads + enable-threads = true + #Set close-on-exec on sockets (could be required for spawning processes in requests). + close-on-exec = true + #Automatically kill workers if master dies (can be dangerous for availability). + no-orphans = true + # Try to remove all of the generated files/sockets (UNIX sockets and pidfiles) upon exit. + vacuum = true + + #performance tuning + + processes = 1 + worker = 20 + post-buffering = 4096 + max-requests = 1000 + # Reload a worker if its address space usage is higher than the specified value (in megabytes). + reload-on-as = 128 + #gracefull dead + harakiri = 60 + harakiri-verbose = true + + #logging + + log-date = true + log-slow = true + + # debug + + # IPDB(); opens python shell + shared-import = ${buildout:directory}/scripts/interact.py + # memory-report = true diff --git a/etc/adhocracy.ini.in b/etc/adhocracy.ini.in index 79be1797c..07c0ad057 100644 --- a/etc/adhocracy.ini.in +++ b/etc/adhocracy.ini.in @@ -7,8 +7,6 @@ ############################################################## [DEFAULT] -{% if parts.adhocracy.debug == 'True' %}debug = True {% end %} -{% if parts.adhocracy.debug != 'True' %}debug = False {% end %} # Uncomment and replace with the address which should receive any error reports email_to = ${parts.adhocracy['email.to']} @@ -18,13 +16,16 @@ error_email_from = ${parts.adhocracy['email.from']} [server:main] use = egg:Paste#http -host = ${parts.adhocracy.host} -port = ${parts.ports.main} +host = ${parts.domains.wsgi} +port = ${parts.ports.wsgi} {% if parts.adhocracy.debug == 'True' %}use_threadpool = False {% end %} {% if parts.adhocracy.debug != 'True' %}use_threadpool = True {% end %} threadpool_workers = 20 - +{% if "uwsgi_conf" in parts %} +[uwsgi] +${parts.uwsgi_conf.conf} +{% end %} {% if parts.adhocracy.themed == 'False' %} ###################not themed adhocracy ################### @@ -166,9 +167,9 @@ adhocracy.interactive_debugging = true # active instance, e.g. "Test Instance" for test.adhocracy.lan. {% python -adhocracy_domain = parts['domains']['main'] -if parts['domains'].get('main.force_port', None): - adhocracy_domain += ':%s' % parts['ports']['main'] +adhocracy_domain = parts['adhocracy']['domain'] +if parts['domains'].get('adhocracy.force_port', None): + adhocracy_domain += ':%s' % parts['ports']['wsgi'] %} adhocracy.domain = ${adhocracy_domain} adhocracy.protocol = ${parts.adhocracy.protocol} diff --git a/etc/uwsgi_buildconf.ini.in b/etc/uwsgi_buildconf.ini.in new file mode 100644 index 000000000..d47110f6e --- /dev/null +++ b/etc/uwsgi_buildconf.ini.in @@ -0,0 +1,50 @@ +; this is mainly copied from the uwsgi buildconf/default.ini +; trying to surface some of the build options to buildout +; properties. +[uwsgi] +bin_name = ${:bin-name} +plugin_dir = ${:plugin-dir} +main_plugin = ${:main-plugin} +xml = false +ini = true +yaml = true +json = auto +sqlite3 = auto +zeromq = auto +snmp = true +spooler = true +embedded = true +ssl = auto +udp = true +multicast = true +threading = true +minterpreters = true +async = true +ldap = auto +pcre = auto +routing = auto +alarm = auto +ipv6 = false +debug = false +unbit = false +xml_implementation = libxml2 +yaml_implementation = auto +malloc_implementation = libc +extras = +plugins = +append_version = +embedded_plugins = %(main_plugin)s, ping, cache, nagios, rrdtool, carbon, rpc, corerouter, fastrouter, http, ugreen, signal, syslog, rsyslog, logsocket, router_uwsgi, router_redirect, router_basicauth, zergpool, redislog, mongodblog, router_rewrite, router_http, logfile, router_cache, rawrouter +as_shared_library = false + +locking = auto +event = auto +timer = auto +filemonitor = auto + +blacklist = +whitelist = + +embed_files = + +embed_config = + diff --git a/python/buildout.cfg b/python/buildout.cfg index 91ce47218..30de23b3f 100644 --- a/python/buildout.cfg +++ b/python/buildout.cfg @@ -13,19 +13,26 @@ parts = ${buildout:links-parts} python-buildout-root = ${buildout:directory}/buildout.python/src -pil-install-args = Pillow==2.0.0 +pil-install-args = Pillow==2.0 + [install-links] recipe = plone.recipe.command prefix = ${buildout:directory}/.. -command = +command = #!/bin/sh mkdir -p ${install-links:prefix}/bin/ + + # Symlink include, lib directory, uwsgi needs this there + ln -sfT ${buildout:directory}/python-2.7/include ${:prefix}/include + ln -sfT ${buildout:directory}/python-2.7/lib ${:prefix}/lib - #Symlink default python (2.7) - DEST=${install-links:prefix}/bin/python; - ln -vfs ${buildout:directory}/python-2.7/bin/python2.7 $DEST; -update-command = ${:command} - + # Symlink python, easy_install + for i in $(ls ${buildout:directory}/python-[23].*/bin/*[23].*); do + DEST=${:prefix}/bin/`basename $i` + ln -sf $i $DEST + done + ln -sf ${:prefix}/bin/python2.7 ${:prefix}/bin/python +update-command = ${:command} diff --git a/scripts/interact.py b/scripts/interact.py new file mode 100644 index 000000000..8653297bf --- /dev/null +++ b/scripts/interact.py @@ -0,0 +1,31 @@ +"""uwsgi development helpers: open interactive python shells""" +import os +import sys +import inspect +import code + + +class RestoredStandardInputContext(object): + + def __enter__(self): + self.backup_stdin = os.dup(sys.stdin.fileno()) + os.dup2(sys.stdout.fileno(), sys.stdin.fileno()) + + def __exit__(self, error_type, error, traceback): + os.dup2(self.backup_stdin, sys.stdin.fileno()) + + +def interact(locals=None, plain=False): + + with RestoredStandardInputContext(): + code.interact(local=locals or inspect.currentframe().f_back.f_locals) + + +def interact_ipdb(): + + with RestoredStandardInputContext(): + import ipdb + ipdb.set_trace() + +__builtins__['PDB'] = interact +__builtins__['IPDB'] = interact_ipdb