translations: update tc, add papertrail (#121)

README.md

@@ -33,6 +33,8 @@ The intention here is to create a single Packer + cloud-init configuration set t
 
 ### Install locally
 
+#### Install Poetry and Python dependencies
+
 Install Poetry (https://python-poetry.org/) if you don't already have it.
 
 ```shell
@@ -43,6 +45,12 @@ poetry shell
 poetry install
 ```
 
+#### Install the GCP plugin for Packer
+
+```bash
+packer plugins install github.com/hashicorp/googlecompute
+```
+
 ## Usage
 
 See `monopacker --help` for details.
@@ -57,20 +65,35 @@ monopacker build builder1 builder2
 
 Note that you can get more logging from packer by setting `PACKER_LOG=1`.
 
-### Developing Templates
+### Template Development and Debugging
+
+See [TEMPLATING.md](./TEMPLATING.md) for information, another FAQ, and more.
+
+#### validate
 
 When developing templates, you can run the validation without running packer with `monopacker validate` (which otherwise has the same arguments as `monopacker build`):
 
 ```shell
 monopacker validate mynewbuilder
 ```
 
+#### view raw packer output
+
 To see the generated packer template:
 ```shell
 monopacker packer-template mynewbuilder
 ```
 
-See [TEMPLATING.md](./TEMPLATING.md) for information, another FAQ, and more.
+#### debugging when building
+
+```bash
+monopacker build generic_translations_gcp --packer-args '-on-error=ask'
+gcloud compute ssh --zone ...
+# when done on host, in monopacker choose to 'c' cleanup
+```
+
+`-on-error=abort` can also be handy.
+
 
 # FAQ
 
@@ -124,4 +147,4 @@ Mostly, I just haven't tried to make this work.
 
 To run the tests for this library, run `poetry run pytest`.
 
-To update dependencies, run `poetry update`.
+To update dependencies, run `poetry cache clear pypi --all && poetry update`.

builders/generic_translations_gcp.yaml

@@ -15,4 +15,5 @@ script_directories:
   - generic-worker-linux
   - worker-runner-linux
   - worker-runner-gw-systemd  # TODO: merge with 'generic-worker-linux'?
+  - relops-papertrail-tclogs  # needs to run after worker-runner-gw-systemd
   - translations-worker-requirements

monopacker/secrets.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import io
+import os
 import tarfile
 
 from ruamel.yaml import YAML
@@ -27,3 +28,16 @@ def pack_secrets(secrets_file, secrets_tar):
                 ti = tarfile.TarInfo(path)
                 ti.size = len(value)
                 tar.addfile(ti, io.BytesIO(value))
+
+def generate_packer_secret_chmod_shell(secrets_file):
+    command_arr = []
+    with open(secrets_file, "r") as f:
+        secrets = yaml.load(f)
+        for secret in secrets:
+            dirname = os.path.dirname(secret['path'])
+            command_arr.append(f"sudo chown -R root:root {dirname}")
+            command_arr.append(f"sudo chmod -R 0400 {dirname}")
+
+    # dedupe the array
+    command_arr = list(dict.fromkeys(command_arr))
+    return command_arr

monopacker/template_packer.py

@@ -15,7 +15,7 @@
 from ruamel.yaml import YAML
 
 from .filters import clean_gcp_image_name
-from .secrets import pack_secrets
+from .secrets import pack_secrets, generate_packer_secret_chmod_shell
 from .files import pack_files
 
 yaml = YAML(typ="safe")
@@ -293,6 +293,12 @@ def generate_packer_template(*,
             ],
             'only': linux_builders,
         })
+        # chmod/chown all secret files (above only gets /etc/taskcluster)
+        pkr["provisioners"].append({
+            'type': 'shell',
+            'inline': generate_packer_secret_chmod_shell(secrets_file),
+            'only': linux_builders,
+        })
         pkr["provisioners"].append({
             'type': 'shell',
             'inline': [

scripts/relops-papertrail-syslog/01-install.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -exv
+
+# init helpers
+helpers_dir=${MONOPACKER_HELPERS_DIR:-"/etc/monopacker/scripts"}
+for h in ${helpers_dir}/*.sh; do
+    . $h;
+done
+
+# steps from https://papertrailapp.com/systems/setup?type=system&platform=unix#unix-manual
+
+sudo wget -O /etc/papertrail-bundle.pem \
+  https://papertrailapp.com/tools/papertrail-bundle.pem
+
+# TODO: use helper functions
+sudo apt update
+sudo apt install rsyslog-gnutls -y
+
+# source secrets file
+. /etc/relops/relops_papertrail_secrets
+
+export RSYSLOG_FILE=/etc/rsyslog.conf
+
+cat << EOF >> $RSYSLOG_FILE
+
+# papertrail config
+\$DefaultNetstreamDriverCAFile /etc/papertrail-bundle.pem
+\$ActionSendStreamDriver gtls
+\$ActionSendStreamDriverMode 1
+\$ActionSendStreamDriverAuthMode x509/name
+\$ActionSendStreamDriverPermittedPeer *.papertrailapp.com
+
+*.*    @@$PAPERTRAIL_HOST:$PAPERTRAIL_PORT
+
+EOF
+
+# restart service (or wait for new instances to boot up?)
+# TOOD: remove/comment when testing is done
+sudo service rsyslog restart

scripts/relops-papertrail-syslog/90-clean.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -exv
+
+# init helpers
+helpers_dir=${MONOPACKER_HELPERS_DIR:-"/etc/monopacker/scripts"}
+for h in ${helpers_dir}/*.sh; do
+    . $h;
+done
+
+rm -rf /usr/src/*
+
+# Do one final package cleanup, just in case.
+apt-get autoremove -y --purge

scripts/relops-papertrail-tclogs/01-install.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -exv
+
+# init helpers
+helpers_dir=${MONOPACKER_HELPERS_DIR:-"/etc/monopacker/scripts"}
+for h in ${helpers_dir}/*.sh; do
+    . $h;
+done
+
+## using remote-syslog2 (recommended by PT)
+# - issues
+#    - no service...
+
+# cd /tmp
+# wget https://github.com/papertrail/remote_syslog2/releases/download/v0.21/remote-syslog2_0.21_amd64.deb
+# sudo dpkg -i remote-syslog*.deb
+
+## using systemd & ncat (used in ronin-puppet)
+
+# nmap provides ncat
+apt update
+apt install -y ncat
+
+export SERVICE_FILE=/etc/systemd/system/papertrail.service
+# TODO: support multipe units?
+export UNIT_TO_MONITOR="generic-worker"
+
+# source secrets file
+. /etc/relops/relops_papertrail_secrets
+
+cat << EOF >> $SERVICE_FILE
+[Unit]
+Description=Papertrail
+After=systemd-journald.service
+Requires=systemd-journald.service
+[Service]
+ExecStart=/bin/sh -c "journalctl -u $UNIT_TO_MONITOR -f | ncat --ssl $PAPERTRAIL_HOST $PAPERTRAIL_PORT"
+TimeoutStartSec=0
+Restart=on-failure
+RestartSec=5s
+[Install]
+WantedBy=multi-user.target
+
+EOF
+
+# reload systemctl so it knows about config
+systemctl daemon-reload
+
+# enable the service on boot
+systemctl enable papertrail
+
+# TODO: start also? can verify it's format is correct...
+# - shouldn't be any output on builder (w-m is not started)
+systemctl start papertrail

scripts/relops-papertrail-tclogs/90-clean.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -exv
+
+# init helpers
+helpers_dir=${MONOPACKER_HELPERS_DIR:-"/etc/monopacker/scripts"}
+for h in ${helpers_dir}/*.sh; do
+    . $h;
+done
+
+rm -rf /usr/src/*
+
+# Do one final package cleanup, just in case.
+apt-get autoremove -y --purge

template/vars/taskcluster_version_translations.yaml

@@ -1,3 +1,3 @@
 # This defines the current Taskcluster version, the default version for worker-runner and workers.
 env_vars:
-  TASKCLUSTER_VERSION: 55.1.1
+  TASKCLUSTER_VERSION: 59.1.3

tests/test_template_packer.py

@@ -172,7 +172,9 @@ def test_generate_packer_template(tmpdir):
           type: openstack
     """))
 
-    secrets_file.write(json.dumps([]))
+    # TODO: add a fake secret json... missing test coverage
+    # secrets_file.write(json.dumps([]))
+    secrets_file.write(json.dumps([{'name': 'blah_key', 'path': '/etc/taskcluster/secrets/test_blah', 'value': 'test123'}]))
 
     scripts_dir.mkdir("facebook-worker").join("01-fb.sh").write("echo hello")
 
@@ -240,6 +242,12 @@ def test_generate_packer_template(tmpdir):
                 ],
                 'only': ['linux'],
             },
+            {'inline': ['sudo chown -R root:root '
+                '/etc/taskcluster/secrets',
+                'sudo chmod -R 0400 /etc/taskcluster/secrets'],
+                'only': ['linux'],
+                'type': 'shell',
+            },
             {
                 'type': 'shell',
                 'inline': [