Various setup.sh fixes

README.md

@@ -11,7 +11,7 @@ For how the streaming system overall looks at the following diagram;
 ![Streaming System overall Diagram](https://docs.google.com/drawings/d/1ZN5uqd-fo62e0IZSzuOSo6YadRY_n7umkUThmqckACA/pub?w=960&h=720)
 
 
-## What is Tim Video's?
+# What is Tim Video's?
 
 Tim Video's is a collection of tools for live streaming conferences and user
 groups. It includes both software and hardware for both the recording and
@@ -20,6 +20,12 @@ viewing sides.
 It has been used with conferences like PyCon US and Linux.conf.au and at user
 groups like Sydney Linux User Group and Sydney Python User Group.
 
+# Important information
+
+```config.private.json``` is referenced by many parts of the Tim Videos system.
+It contains passwords, rooms/stream metadata, and other important information. 
+Please ensure this is populated early (base it from ```config.json.example```.
+
 # Parts
 
  * **website** - The actual website people end up seeing/going too. Uses
@@ -40,6 +46,9 @@ groups like Sydney Linux User Group and Sydney Python User Group.
       tools to push the configurations out to systems. Useful for conferences
       where you are managing a lot of encoders/collectors.
 
+      There is a script in here called ```pusher/pushconfig.py``` which should be used
+      to generate the correct Flumotion configuration files for encoders/collectors.      
+
     * **flumotion-extra** - Website controller of flumotion (replaces the GTK
       tool).
 

tools/register/README

@@ -0,0 +1 @@
+README.md

tools/register/README.md

@@ -0,0 +1,13 @@
+This script is used to register a flumotion encoder server onto the timvideos frontend system.
+This should be ran on the encoder machines.
+
+Usage (run as root in a screen/tmux session):
+
+```python register_wrapper.py -m user:password@encoder-server -g group -s sharedsecret```
+
+(where:
+ * user:password equal flumotion manager username/password,
+ * group equals group/room in config.private.json on timvideos frontend,
+ * sharedsecret is secret in config.private.json on timvideos frontend
+
+fake_register.py can be used to test room registration (see parent README.md)

tools/setup/25-python-stuff.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -ex
+#
+# Copyright 2014 Next Day Video P/L. All Rights Reserved.
+# Author: [email protected] (Ryan Verner)
+#
+# Installs required python packages
+#
+
+sudo apt-get install python-pip
+sudo pip install termcolor
+

tools/setup/30-custom-flumotion.sh

@@ -25,6 +25,9 @@ sudo apt-get install --assume-yes build-essential autoconf autopoint libtool lib
 # Dependencies to run flumotion
 sudo apt-get install --assume-yes python-kiwi python-twisted.* ssl-cert
 
+# Fix permissions issue (/usr/local/var/* can be chmod 700)
+sudo chmod 755 /usr/local/var
+
 # Install the core flumotion
 (
   cd ../flumotion
@@ -63,3 +66,7 @@ sudo chown flumotion:flumotion /usr/local/etc/flumotion/default.pem
 
 # Give access to the firewire ports
 sudo adduser flumotion video
+
+# Give access to audio (non-dvswitch ALSA sources)
+sudo adduser flumotion audio
+

tools/setup/40-gst-plugins-dvswitch.sh

@@ -17,6 +17,7 @@ sudo apt-get install --assume-yes build-essential autoconf autopoint libtool lib
 # build and install
 (
   cd 
+  rm -fr gst-plugins-dvswitch
   git clone https://github.com/timvideos/gst-plugins-dvswitch.git
   cd gst-plugins-dvswitch
   git clean -f -x

tools/setup/runall.sh

@@ -16,6 +16,7 @@ cd streaming-system/tools/setup
 ./05-update.sh
 ./20-gstreamer-packages.sh
 ./20-useful-packages.sh
+./25-python-stuff.sh
 ./30-custom-flumotion.sh
 ./40-gst-plugins-dvswitch.sh
 ./99-remove-ppa.sh

tools/watchdog/README

@@ -0,0 +1 @@
+README.md

tools/watchdog/README.md

@@ -0,0 +1,9 @@
+This script is used to watch and restart flumotion components.
+This should be ran on the collector + encoder machines.
+
+Usage (run as root in a screen/tmux session):
+
+```python watchdog_wrapper.sh -m user:[email protected]```
+
+(where user:password equal flumotion manager username/password)
+

website/Makefile

@@ -28,7 +28,7 @@ clean:
 	git clean -f -d
 
 bin/activate:
-	virtualenv --no-site-packages .
+	virtualenv --always-copy --no-site-packages .
 	-rm distribute*.tar.gz
 
 freeze:
@@ -45,7 +45,7 @@ third_party:
 
 src/pip-delete-this-directory.txt: requirements.txt
 	$(ACTIVATE) && pip install -r requirements.txt
-	$(ACTIVATE) && [ -f private/requirements.txt ] && pip install -r private/requirements.txt
+	$(ACTIVATE) && [ -f private/requirements.txt ] && pip install -r private/requirements.txt || exit 0
 	touch -r requirements.txt src/pip-delete-this-directory.txt
 
 install: lib/python2.6/site-packages/ez_setup.py lib/python2.6/site-packages/distribute-0.6.24-py2.6.egg-info src/pip-delete-this-directory.txt frontend/data.py
@@ -56,6 +56,7 @@ frontend/data.py: eventfeed2internal.py pycon2internal.py
 prepare-serve: install
 	$(ACTIVATE) && python manage.py collectstatic --noinput
 #	$(ACTIVATE) && python manage.py syncdb
+#	$(ACTIVATE) && python manage.py migrate
 
 lint: install
 	@# R0904 - Disable "Too many public methods" warning
@@ -84,8 +85,13 @@ config:
 upload:
 	$(ACTIVATE) && git-cl upload
 
-serve: prepare-serve install
+initdb:
 	$(ACTIVATE) && python manage.py syncdb
+	$(ACTIVATE) && python manage.py migrate
+
+serve: prepare-serve install
 	$(ACTIVATE) && python manage.py runserver ${ARGS} 
 
+testserve: prepare-serve install initdb serve
+
 .PHONY : lint upload deploy serve clean config prepare-serve

website/README.md

@@ -39,17 +39,19 @@ sudo pip install virtualenv
 Running a test server
 =====================
 
-Run ``make serve``; this will configure a virtualenv, download and install
+Run ``make testserve``; this will configure a virtualenv, download and install
 dependencies (inside the virtualenv; your system will not be touched); and a
 test server will be started.
 
-If this is your first time running ``make serve`` you'll be prompted to provide
+If you recieve a "bin/activate" error, try ```make distclean``` and try again. 
+
+If this is your first time running ``make testserve`` you'll be prompted to provide
 a username and password for an admin account.
 
 This test server should not be used in production (see relevant section below).
 
 Requests are served at http://127.0.0.1:8000 by default.  If you need to change
-the listening IP or port, try ```ARGS="192.168.1.1:8000" make serve``` where ARGS
+the listening IP or port, try ```ARGS="192.168.1.1:8000" make testserve``` where ARGS
 represents your desired listening IP and port.
 
 This test server **should NOT be used in production** (see relevant section below).
@@ -82,7 +84,7 @@ This keeps on going.
 
 Open http://127.0.0.1:8000/example to view the page
 
-Note: DO NOT run ``make serve`` as root. If you do, virtualenv folders won't
+Note: DO NOT run ``make testserve`` as root. If you do, virtualenv folders won't
 have write permissions, so the tracker won't be able to write to sqlite
 database.
 
@@ -138,22 +140,51 @@ Production Deployment
     be persisted across deployments, you may have to take care of this in your
     deploy script.
 
- *  To do the deployment, I use the ``setup.sh`` script in conf directory. It
-    takes a copy of the current code in ``~website/timvideos`` and puts it in
-    ``~website/$VERSION-$DATE-$TIME`` directory and then links
-    ``~website/current`` to that.
+ *  **To deploy on an Ubuntu 12.04 LTS system**, run the  ``setup.sh`` script in the conf/
+    directory.  If an existing deploy doesn't exist, it will create the relevant
+    groups, users, install the correct packages, copy in some configuration files for
+    relevant daemons (nginx, etc) and git pull the project into ~website/timvideos.
+    **Please look at the script first if deploying on a system already used for other
+    things as this assumes a fairly fresh install.**.
+
+    If already deployed into ~website/timvideos, it will update from git.
+
+    The script will then copy installation over into ~website/timvideos-$VERSION-DATE-TIME and
+    then links ~website/current to that. 
 
     This makes it relatively easy to revert to an earlier version of the code.
+
+    The script may work on other distros, but has only been tested on Ubuntu 12.04 LTS.
+
+ *  The script will ask for a production settings.py to be created using
+    non-sqlite details.  Populate this with a real DB, such as PostgreSQL.
+
+ *  On a new installation, you will need to initialise the database (this is
+    not done by setup.sh as it can be used for upgrades, too).  You can do this
+    by running ```make initdb```.
+
+ *  On an existing installation you may need to migrate any schema changes.
+    You can do this by running ```bin/python manage.py migrate```.
 
  *  The app runs inside [Green Unicorn][], and is started by
     [upstart][]. Take a look at conf/init.conf
 
  *  The app uses [nginx][] as a frontend, and to serve static files. Only a few
     changes from the default config are needed to accomplish this.
 
-    Take a look at conf/nginx.conf
+    Take a look at conf/nginx.conf (this needs to be modified).
 
   [deploy key]: http://help.github.com/deploy-keys/
   [Green Unicorn]: http://gunicorn.org/
   [upstart]: http://upstart.ubuntu.com/
   [nginx]: http://nginx.org/en/ "nginx"
+
+
+
+
+Important information
+=====================
+
+```config.private.json``` is referenced by many parts of the Tim Videos system.
+It contains passwords, rooms/stream metadata, and other important information.
+Please ensure this is populated early (base it from ```config.json.example```.

website/conf/setup.sh

@@ -13,6 +13,9 @@ fi
 set -x
 set -e
 
+# Remove old /tmp/timvideos-static
+rm -rf /tmp/timvideos-static
+
 # Add the groups required
 addgroup --system website
 addgroup --system website-run
@@ -29,8 +32,9 @@ adduser $USER website-run
 # Get python dependencies
 apt-get -y install python-pip python-setuptools python-virtualenv
 
-# Set up the website directory
+# Set up the website directory and set current directory test deployment is running in
 BASEDIR=/home/website
+CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 # Set up upstart config file
 ln -sf $BASEDIR/current/website/conf/init.conf /etc/init/website.conf
@@ -69,8 +73,16 @@ export VERSION=$(date +%Y%m%d-%H%M%S)-$(git describe --tags --long --always)
 cd ..
 
 if [ ! -f $BASEDIR/timvideos/website/private/settings.py ]; then
-  echo "Please create a settings.py file in $BASEDIR/timvideos/website/private/settings.py"
-  exit 1
+  if [ -f $CURDIR/../private/settings.py ]; then
+    echo "Copying settings.py from $CURDIR to $BASEDIR..."
+    cp $CURDIR/../private/settings.py $BASEDIR/timvideos/website/private/settings.py
+  else
+    echo "No settings.py file found in either test or production directories."
+    echo "Please create a settings.py file in $BASEDIR/timvideos/website/private/settings.py"
+    echo "This should contain production db settings (not sqlite!)."
+    echo "Once this is done, run this script again." 
+    exit 1
+  fi
 fi
 
 VERSIONDIR=$BASEDIR/timvideos-$VERSION
@@ -85,8 +97,15 @@ cp -R /tmp/timvideos-static $VERSIONDIR/website/static
 as_website rm current || true
 as_website ln -s timvideos-$VERSION current
 
-# Need to get config-private.json
-# Need to get settings-private.py
+# Copy over config.private.json from previous installation
+if [ ! -f $BASEDIR/timvideos/config.private.json ]; then
+  if [ -f $CURDIR/../../config.private.json ]; then
+    echo "Copying config.private.json from $CURDIR to $BASEDIR..."
+    cp $CURDIR/../../config.private.json $BASEDIR/timvideos/config.private.json 
+  else
+    echo "No config.private.json file found!"
+  fi
+fi 
 
 # Make upstart/gunicorn reload config file
 chmod 644 $VERSIONDIR/website/conf/init.conf

website/private/settings.py.example

@@ -10,6 +10,9 @@
 #    'default': {
 #        'ENGINE': 'django.db.backends.postgresql_psycopg2',
 #        'NAME': 'website-run',
+#	 'PASSWORD': '',
+#        'HOST': '', 	# Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP
+#        'PORT': '',    # Set to empty string for default.
 #    }
 #}