cloud-config-cluster.yml

#cloud-config

# Instance

apt_sources:
- source: "ppa:webupd8team/java"
- source: "deb http://packages.elasticsearch.org/elasticsearch/1.7/debian stable main"
  key: |
    -----BEGIN PGP PUBLIC KEY BLOCK-----
    Version: GnuPG v2.0.14 (GNU/Linux)

    mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD
    A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9
    CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ
    j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd
    1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD
    2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg
    KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy
    Y2gub3JnPokBOAQTAQIAIgUCUjceygIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgEC
    F4AACgkQ0n1mbNiOQrRzjAgAlTUQ1mgo3nK6BGXbj4XAJvuZDG0HILiUt+pPnz75
    nsf0NWhqR4yGFlmpuctgCmTD+HzYtV9fp9qW/bwVuJCNtKXk3sdzYABY+Yl0Cez/
    7C2GuGCOlbn0luCNT9BxJnh4mC9h/cKI3y5jvZ7wavwe41teqG14V+EoFSn3NPKm
    TxcDTFrV7SmVPxCBcQze00cJhprKxkuZMPPVqpBS+JfDQtzUQD/LSFfhHj9eD+Xe
    8d7sw+XvxB2aN4gnTlRzjL1nTRp0h2/IOGkqYfIG9rWmSLNlxhB2t+c0RsjdGM4/
    eRlPWylFbVMc5pmDpItrkWSnzBfkmXL3vO2X3WvwmSFiQbkBDQRSNx7KAQgA5JUl
    zcMW5/cuyZR8alSacKqhSbvoSqqbzHKcUQZmlzNMKGTABFG1yRx9r+wa/fvqP6OT
    RzRDvVS/cycws8YX7Ddum7x8uI95b9ye1/Xy5noPEm8cD+hplnpU+PBQZJ5XJ2I+
    1l9Nixx47wPGXeClLqcdn0ayd+v+Rwf3/XUJrvccG2YZUiQ4jWZkoxsA07xx7Bj+
    Lt8/FKG7sHRFvePFU0ZS6JFx9GJqjSBbHRRkam+4emW3uWgVfZxuwcUCn1ayNgRt
    KiFv9jQrg2TIWEvzYx9tywTCxc+FFMWAlbCzi+m4WD+QUWWfDQ009U/WM0ks0Kww
    EwSk/UDuToxGnKU2dQARAQABiQEfBBgBAgAJBQJSNx7KAhsMAAoJENJ9ZmzYjkK0
    c3MIAIE9hAR20mqJWLcsxLtrRs6uNF1VrpB+4n/55QU7oxA1iVBO6IFu4qgsF12J
    TavnJ5MLaETlggXY+zDef9syTPXoQctpzcaNVDmedwo1SiL03uMoblOvWpMR/Y0j
    6rm7IgrMWUDXDPvoPGjMl2q1iTeyHkMZEyUJ8SKsaHh4jV9wp9KmC8C+9CwMukL7
    vM5w8cgvJoAwsp3Fn59AxWthN3XJYcnMfStkIuWgR7U2r+a210W6vnUxU4oN0PmM
    cursYPyeV0NX/KQeUeNMwGTFB6QHS/anRaGQewijkrYYoTNtfllxIu9XYmiBERQ/
    qPDlGRlOgVTd9xUfHFkzB52c70E=
    =92oX
    -----END PGP PUBLIC KEY BLOCK-----
- source: "deb https://deb.nodesource.com/node_4.x trusty main"
  key: |
    -----BEGIN PGP PUBLIC KEY BLOCK-----
    Version: GnuPG v1
    Comment: GPGTools - https://gpgtools.org

    mQINBFObJLYBEADkFW8HMjsoYRJQ4nCYC/6Eh0yLWHWfCh+/9ZSIj4w/pOe2V6V+
    W6DHY3kK3a+2bxrax9EqKe7uxkSKf95gfns+I9+R+RJfRpb1qvljURr54y35IZgs
    fMG22Np+TmM2RLgdFCZa18h0+RbH9i0b+ZrB9XPZmLb/h9ou7SowGqQ3wwOtT3Vy
    qmif0A2GCcjFTqWW6TXaY8eZJ9BCEqW3k/0Cjw7K/mSy/utxYiUIvZNKgaG/P8U7
    89QyvxeRxAf93YFAVzMXhoKxu12IuH4VnSwAfb8gQyxKRyiGOUwk0YoBPpqRnMmD
    Dl7SdmY3oQHEJzBelTMjTM8AjbB9mWoPBX5G8t4u47/FZ6PgdfmRg9hsKXhkLJc7
    C1btblOHNgDx19fzASWX+xOjZiKpP6MkEEzq1bilUFul6RDtxkTWsTa5TGixgCB/
    G2fK8I9JL/yQhDc6OGY9mjPOxMb5PgUlT8ox3v8wt25erWj9z30QoEBwfSg4tzLc
    Jq6N/iepQemNfo6Is+TG+JzI6vhXjlsBm/Xmz0ZiFPPObAH/vGCY5I6886vXQ7ft
    qWHYHT8jz/R4tigMGC+tvZ/kcmYBsLCCI5uSEP6JJRQQhHrCvOX0UaytItfsQfLm
    EYRd2F72o1yGh3yvWWfDIBXRmaBuIGXGpajC0JyBGSOWb9UxMNZY/2LJEwARAQAB
    tB9Ob2RlU291cmNlIDxncGdAbm9kZXNvdXJjZS5jb20+iQI4BBMBAgAiBQJTmyS2
    AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAWVaCraFdigHTmD/9OKhUy
    jJ+h8gMRg6ri5EQxOExccSRU0i7UHktecSs0DVC4lZG9AOzBe+Q36cym5Z1di6JQ
    kHl69q3zBdV3KTW+H1pdmnZlebYGz8paG9iQ/wS9gpnSeEyx0Enyi167Bzm0O4A1
    GK0prkLnz/yROHHEfHjsTgMvFwAnf9uaxwWgE1d1RitIWgJpAnp1DZ5O0uVlsPPm
    XAhuBJ32mU8S5BezPTuJJICwBlLYECGb1Y65Cil4OALU7T7sbUqfLCuaRKxuPtcU
    VnJ6/qiyPygvKZWhV6Od0Yxlyed1kftMJyYoL8kPHfeHJ+vIyt0s7cropfiwXoka
    1iJB5nKyt/eqMnPQ9aRpqkm9ABS/r7AauMA/9RALudQRHBdWIzfIg0Mlqb52yyTI
    IgQJHNGNX1T3z1XgZhI+Vi8SLFFSh8x9FeUZC6YJu0VXXj5iz+eZmk/nYjUt4Mtc
    pVsVYIB7oIDIbImODm8ggsgrIzqxOzQVP1zsCGek5U6QFc9GYrQ+Wv3/fG8hfkDn
    xXLww0OGaEQxfodm8cLFZ5b8JaG3+Yxfe7JkNclwvRimvlAjqIiW5OK0vvfHco+Y
    gANhQrlMnTx//IdZssaxvYytSHpPZTYw+qPEjbBJOLpoLrz8ZafN1uekpAqQjffI
    AOqW9SdIzq/kSHgl0bzWbPJPw86XzzftewjKNbkCDQRTmyS2ARAAxSSdQi+WpPQZ
    fOflkx9sYJa0cWzLl2w++FQnZ1Pn5F09D/kPMNh4qOsyvXWlekaV/SseDZtVziHJ
    Km6V8TBG3flmFlC3DWQfNNFwn5+pWSB8WHG4bTA5RyYEEYfpbekMtdoWW/Ro8Kmh
    41nuxZDSuBJhDeFIp0ccnN2Lp1o6XfIeDYPegyEPSSZqrudfqLrSZhStDlJgXjea
    JjW6UP6txPtYaaila9/Hn6vF87AQ5bR2dEWB/xRJzgNwRiax7KSU0xca6xAuf+TD
    xCjZ5pp2JwdCjquXLTmUnbIZ9LGV54UZ/MeiG8yVu6pxbiGnXo4Ekbk6xgi1ewLi
    vGmz4QRfVklV0dba3Zj0fRozfZ22qUHxCfDM7ad0eBXMFmHiN8hg3IUHTO+UdlX/
    aH3gADFAvSVDv0v8t6dGc6XE9Dr7mGEFnQMHO4zhM1HaS2Nh0TiL2tFLttLbfG5o
    QlxCfXX9/nasj3K9qnlEg9G3+4T7lpdPmZRRe1O8cHCI5imVg6cLIiBLPO16e0fK
    yHIgYswLdrJFfaHNYM/SWJxHpX795zn+iCwyvZSlLfH9mlegOeVmj9cyhN/VOmS3
    QRhlYXoA2z7WZTNoC6iAIlyIpMTcZr+ntaGVtFOLS6fwdBqDXjmSQu66mDKwU5Ek
    fNlbyrpzZMyFCDWEYo4AIR/18aGZBYUAEQEAAYkCHwQYAQIACQUCU5sktgIbDAAK
    CRAWVaCraFdigIPQEACcYh8rR19wMZZ/hgYv5so6Y1HcJNARuzmffQKozS/rxqec
    0xM3wceL1AIMuGhlXFeGd0wRv/RVzeZjnTGwhN1DnCDy1I66hUTgehONsfVanuP1
    PZKoL38EAxsMzdYgkYH6T9a4wJH/IPt+uuFTFFy3o8TKMvKaJk98+Jsp2X/QuNxh
    qpcIGaVbtQ1bn7m+k5Qe/fz+bFuUeXPivafLLlGc6KbdgMvSW9EVMO7yBy/2JE15
    ZJgl7lXKLQ31VQPAHT3an5IV2C/ie12eEqZWlnCiHV/wT+zhOkSpWdrheWfBT+ac
    hR4jDH80AS3F8jo3byQATJb3RoCYUCVc3u1ouhNZa5yLgYZ/iZkpk5gKjxHPudFb
    DdWjbGflN9k17VCf4Z9yAb9QMqHzHwIGXrb7ryFcuROMCLLVUp07PrTrRxnO9A/4
    xxECi0l/BzNxeU1gK88hEaNjIfviPR/h6Gq6KOcNKZ8rVFdwFpjbvwHMQBWhrqfu
    G3KaePvbnObKHXpfIKoAM7X2qfO+IFnLGTPyhFTcrl6vZBTMZTfZiC1XDQLuGUnd
    sckuXINIU3DFWzZGr0QrqkuE/jyr7FXeUJj9B7cLo+s/TXo+RaVfi3kOc9BoxIvy
    /qiNGs/TKy2/Ujqp/affmIMoMXSozKmga81JSwkADO1JMgUy6dApXz9kP4EE3g==
    =CLGF
    -----END PGP PUBLIC KEY BLOCK-----

bootcmd:
- set -ex
- cloud-init-per once ssh-users-ca echo "TrustedUserCAKeys /etc/ssh/users_ca.pub" >> /etc/ssh/sshd_config
- cloud-init-per once accepted-oracle-license-v1-1 echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
- cloud-init-per once fallocate-swapfile fallocate -l 4G /swapfile
- cloud-init-per once chmod-swapfile chmod 600 /swapfile
- cloud-init-per once mkswap-swapfile mkswap /swapfile
- MEMGIGS=$(awk '/MemTotal/{printf "%%.0f", $2 / 1024**2}' /proc/meminfo)
- if [ "$MEMGIGS" -gt 32 ]
- then
-    echo "ES_HEAP_SIZE=8g" > /etc/default/elasticsearch
- elif [ "$MEMGIGS" -gt 12 ]
- then
-    echo "ES_HEAP_SIZE=4g" > /etc/default/elasticsearch
- else
-    echo "ES_HEAP_SIZE=2g" > /etc/default/elasticsearch
-    sysctl "vm.swappiness=1"
-    swapon /swapfile
- fi

package_upgrade: true

packages:
- apache2-mpm-worker
- bsdtar
- build-essential
- elasticsearch
- git
- graphviz
- libapache2-mod-wsgi-py3
- libevent-dev
- libfreetype6-dev
- libjpeg-dev
- liblcms2-dev
- libmagic-dev
- libpq-dev
- libssl-dev
- libtiff5-dev
- libwebp-dev
- libxml2-dev
- libxslt1-dev
- lzop
- nodejs
- ntp
- oracle-java8-installer
- oracle-java8-set-default
- postgresql-9.3
- pv
- python2.7-dev
- python3.4-dev
- python-software-properties
- python-virtualenv
- ruby-dev
- unattended-upgrades
- update-notifier-common
- zlib1g-dev

power_state:
  mode: reboot

output:
  all: '| tee -a /var/log/cloud-init-output.log'

runcmd:
# Ideally this would build as a different user so snowflakes only has read
# permissions
- set -ex
- update-rc.d elasticsearch defaults
- sudo bash /etc/elasticsearch/cluster.sh %(CLUSTER_NAME)s
- sudo /usr/share/elasticsearch/bin/plugin install elasticsearch/elasticsearch-cloud-aws/2.7.1
- service elasticsearch start
- chown postgres:postgres /etc/postgresql/9.3/main/*.conf
- echo "include 'custom.conf'" >> /etc/postgresql/9.3/main/postgresql.conf
- if test "%(ROLE)s" = "demo"
- then
-   echo "standby_mode = off" >> /etc/postgresql/9.3/main/recovery.conf
-   echo "include 'demo.conf'" >> /etc/postgresql/9.3/main/postgresql.conf
- fi
- sudo -u postgres createuser snowflakes
- sudo -u postgres createdb --owner=snowflakes snowflakes
- mkdir /srv/snowflakes
- chown snowflakes:snowflakes /srv/snowflakes
- cd /srv/snowflakes
- sudo -u snowflakes git clone --no-checkout https://github.com/ENCODE-DCC/snovault.git .
- sudo -u snowflakes git checkout %(COMMIT)s
- mkdir /opt/cloudwatchmon
- chown build:build /opt/cloudwatchmon
- sudo -u build virtualenv --python=python2.7 /opt/cloudwatchmon
- sudo -u build /opt/cloudwatchmon/bin/pip install -r cloudwatchmon-requirements.txt
- mkdir /opt/wal-e
- chown postgres:postgres /opt/wal-e
- sudo -u postgres virtualenv --python=python2.7 /opt/wal-e
- sudo -u postgres /opt/wal-e/bin/pip install -r wal-e-requirements.txt
- /etc/init.d/postgresql stop
- sudo -u postgres /opt/wal-e/bin/wal-e --aws-instance-profile --s3-prefix="$(cat /etc/postgresql/9.3/main/wale_s3_prefix)" backup-fetch /var/lib/postgresql/9.3/main LATEST
- sudo -u postgres ln -s /etc/postgresql/9.3/main/recovery.conf /var/lib/postgresql/9.3/main/
- /etc/init.d/postgresql start
- sudo -u snowflakes python3.4 bootstrap.py --buildout-version 2.4.1 --setuptools-version 18.1
- sudo -u snowflakes bin/aws s3 cp --recursive s3://encoded-conf-prod/.aws .aws
- until sudo -u postgres psql postgres -c ""; do sleep 10; done
- sudo -u snowflakes sh -c 'cat /dev/urandom | head -c 256 | base64 > session-secret.b64'
- sudo -u snowflakes bin/create-mapping production.ini --app-name app
- ln -s /srv/snowflakes/etc/snowflakes-apache.conf /etc/apache2/sites-available/snowflakes.conf
- ln -s /srv/snowflakes/etc/logging-apache.conf /etc/apache2/conf-available/logging.conf
- a2enmod headers
- a2enmod proxy_http
- a2enmod rewrite
- a2enmod ssl
- a2ensite snowflakes.conf
- a2dissite 000-default
- a2enconf logging
- a2disconf charset
- a2disconf security
- a2disconf localized-error-pages
- a2disconf other-vhosts-access-log
- a2disconf serve-cgi-bin
- if test "%(ROLE)s" = "demo"
- then
-   sudo -i -u snowflakes bin/batchupgrade production.ini --app-name app
- fi
# - sleep 10
# - sudo -u postgres echo "include 'master.conf'" >> /etc/postgresql/9.3/main/postgresql.conf
# - pg_ctlcluster 9.3 main reload
# - pg_ctlcluster 9.3 main promote
# - sudo -u snowflakes bin/update-keys-links production.ini --app-name app
# - sudo -u snowflakes bin/upgrade production.ini --app-name app
# - sudo -i -u postgres /opt/wal-e/bin/envfile --config ~postgres/.aws/credentials --section default --upper -- /opt/wal-e/bin/wal-e --s3-prefix="$(cat /etc/postgresql/9.3/main/wale_s3_prefix)" backup-push /var/lib/postgresql/9.3/main
# - sleep 60
# - sudo -i -u snowflakes PATH="/usr/share/elasticsearch/bin:/usr/lib/postgresql/9.3/bin:$PATH" bin/test -m "not bdd" > tests.out

users:
- default
- name: build
  gecos: Build user
  inactive: true
  system: true
- name: snowflakes
  gecos: Snowflake database daemon user
  inactive: true
  system: true
  # Specified homedir must exist
  # https://github.com/rubygems/rubygems/issues/689
  homedir: /srv/snowflakes

write_files:
- path: /etc/apt/apt.conf.d/20auto-upgrades
  content: |
    APT::Periodic::Update-Package-Lists "1";
    APT::Periodic::Unattended-Upgrade "1";

- path: /etc/apt/apt.conf.d/50unattended-upgrades
  content: |
    Unattended-Upgrade::Allowed-Origins {
        "${distro_id} ${distro_codename}-security";
    };
    Unattended-Upgrade::Automatic-Reboot "true";

- path: /etc/cron.d/cloudwatchmon
  content: |
    */5 * * * * nobody /opt/cloudwatchmon/bin/mon-put-instance-stats.py --mem-util --swap-util --disk-space-util --disk-path=/ --from-cron

- path: /etc/elasticsearch/elasticsearch.yml
  content: |
    index.search.slowlog.threshold.query.warn: 8s
    index.search.slowlog.threshold.fetch.warn: 1s
    index.indexing.slowlog.threshold.index.warn: 25s
    network.host: 0.0.0.0
    action.auto_create_index: false
    index.mapper.dynamic: false

    node.master: true
    node.data: false

    discovery:
        type: ec2

    cloud.aws.region: us-west-2

    discovery.ec2.groups: elasticsearch-https, ssh-http-https

    # discovery.zen.minimum_master_nodes: 2

- path: /etc/elasticsearch/cluster.sh
  content: |
    #!/bin/bash
    name=$1
    
    if [[ -n "$name" ]]; then
        echo "cluster.name: $name" >> /etc/elasticsearch/elasticsearch.yml
    else
        echo "argument error"
    fi


- path: /etc/postgresql/9.3/main/custom.conf
  content: |
    hot_standby = on
    max_standby_archive_delay = -1
    wal_level = hot_standby
    archive_mode = on
    archive_timeout = 60
    # http://www.postgresql.org/message-id/CAOycyLTm6X3mVLz+sLCex+W==WSMgu9giteV7efPoPXYDhPtzQ@mail.gmail.com
    checkpoint_timeout = 1h

- path: /etc/postgresql/9.3/main/demo.conf
  content: |
    archive_mode = off

- path: /etc/postgresql/9.3/main/master.conf
  content: |
    archive_command = '/opt/wal-e/bin/envfile --config ~postgres/.aws/credentials --section default --upper -- /opt/wal-e/bin/wal-e --s3-prefix="$(cat /etc/postgresql/9.3/main/wale_s3_prefix)" wal-push "%%p"'

- path: /etc/postgresql/9.3/main/recovery.conf
  content: |
    # recovery.conf must be linked into data dir to do anything
    recovery_target_timeline = 'latest'
    restore_command = '/opt/wal-e/bin/wal-e --aws-instance-profile --s3-prefix="$(cat /etc/postgresql/9.3/main/wale_s3_prefix)" wal-fetch "%%f" "%%p"'
    standby_mode = on

- path: /etc/postgresql/9.3/main/wale_s3_prefix
  content: "%(WALE_S3_PREFIX)s"

- path: /etc/ssh/users_ca.pub
  content: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv/ymOcnN4LhM4NACc3Or116XXJ6KytuOgB/+1qNkOFBqBosrn7cmJ35rsoNHRgYNrCsRE9ch74RKsN6H72FtSJgBhGh/9oUK7Os6Fqt3/ZZXxgxIx6ubs/MTgrxrAnujiBxUXMXQhLKMriNMpo8mt4nGYVtLk9PBjiyfncaS8H9ZKoNio9dhP8bmTuYvioAI35dqKdSlVLyzr/XkZxia8Ki+pQ0N6uuiEwMR3ToM+LSp8wpFOOAiu4PEAujRW7us/+1hlpKWfn0J7/V3826joHE+I967Vg/+ikcVhF77JjK1nib879VgCWfmn1HPQosIpk4yJfVgGvRVI7I2nfBPVw== snowflakes@demo-l.snowflakescc.org