diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e2f173..fb9b74e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,7 @@ jobs: run: runs-on: ubuntu-latest + timeout-minutes: 5 strategy: fail-fast: false matrix: @@ -32,6 +33,10 @@ jobs: run: | python -m pip install --upgrade pip pip install hatch + - name: Pull images + run: | + docker pull mrcide/privateer-server:prototype + docker pull mrcide/privateer-client:prototype - name: Test run: | hatch run cov-ci diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4214466 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.pyc +__pycache__ +dist/ +.coverage +coverage.xml +*.tar +.privateer_identity +tmp/ +.coverage.* diff --git a/README.md b/README.md index 27c934d..9bcd4a0 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,94 @@ ----- -**Table of Contents** +## The idea -- [Installation](#installation) -- [License](#license) +We need a way of synchronising some docker volumes from a machine to some backup server, incrementally, using `rsync`. We previously used [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup) to backup volumes in their entirety to another machine as a tar file but the space and time requirements made this hard to use in practice. + +### The setup + +We assume some number of **server** machines -- these will receive data, and some number of **client** machines -- these will send data to the server(s). A client can back any number of volumes to any number of servers, and a server can receive and serve any number of volumes to any number of clients. + +A typical framework for us would be that we would have a "production" machine which is backing up to one or more servers, and then some additional set of "staging" machines that receive data from the servers, which in practice never send any data. + +Because we are going to use ssh for transport, we assume existence of [HashiCorp Vault](https://www.vaultproject.io/) to store secrets. + +### Configuration + +The system is configured via a single `json` document, `privateer.json` which contains information about all the moving parts: servers, clients, volumes and the vault configuration. See [`example/`](example/) for some examples. + +We imagine that your configuration will exist in some repo, and that that repo will be checked out on all involved machines. Please add `.privateer_identity` to your `.gitignore` for this repo. + +### Setup + +After writing a configuration, on any machine run + +``` +privateer2 keygen --all +``` + +which will generate ssh keypairs for all machines and put them in the vault. This only needs to be done once, but you might need to run it again if + +* you add more machines to your system +* you want to rotate keys + +Once keys are written to the vault, on each machine run + +``` +privateer2 configure +``` + +replacing `` with the name of the machine within either the `servers` or `clients` section of your configuration. This sets up a special docker volume that will persist ssh keys and configurations so that communication between clients and servers is straightforward and secure. It also leaves a file `.privateer_identity` at the same location as the configuration file, which is used as the default identity for subsequent commands. Typically this is what you want. + +### Manual backup + +``` +privateer2 backup [--server=NAME] +``` + +Add `--dry-run` to see the commands to run it yourself + +### Restore + +Restoration is always manual + +``` +privateer2 restore [--server=NAME] [--source=NAME] +``` + +where `--server` controls the server you are pulling from (useful if you have more than one configured) and `--source` controls the original machine that backed the data up (if more than one machine is pushing backups). + +For example, if you are on a "staging" machine, connecting to the "backup" server and want to pull the "user_data" volume that was backed up from "production" machine called you would type + +``` +privateer2 restore user_data --server=backup --source=production +``` + + +## What's the problem anyway? + +[Docker volumes](https://docs.docker.com/storage/volumes/) are useful for abstracting away some persistent storage for an application. They're much nicer to use than bind mounts because they don't pollute the host system with immovable files (docker containers often running as root or with a uid different to the user running docker). The docker [docs](https://docs.docker.com/storage/volumes/#back-up-restore-or-migrate-data-volumes) describe some approaches to backup and restore but in practice this ignores many practical issues, especially when the volumes are large or off-site backup is important. + +We want to be able to synchronise a volume to another volume on a different machine; our setup looks like this: + +``` +bob alice ++-------------------+ +-----------------------+ +| | | | +| application | | | +| | | | | +| volume1 | | volume2 | +| | | ssh/ | | | +| privateer-client--=----------=---> privateer-server | +| | | rsync | | | +| keys | | keys | +| | | | ++-------------------+ +-----------------------+ +``` + +so in this case `bob` runs a privateer client which sends data over ssh+rsync to a server running on `alice`, eventually meaning that the data in `volume1` on `bob` is replicated to `volume2` on `alice`. This process uses a set of ssh keys that each client and server will hold in a `keys` volume. This means that they do not interact with any ssh systems on the host. Note that if `alice` is also running sshd, this backup process will use a *second* ssh connection. + +In addition, we will support point-in-time backups on `alice`, creating `tar` files of the volume onto disk that can be easily restored onto any host. ## Installation diff --git a/buildkite/pipeline.yml b/buildkite/pipeline.yml new file mode 100644 index 0000000..e066a34 --- /dev/null +++ b/buildkite/pipeline.yml @@ -0,0 +1,3 @@ +steps: + - label: ":whale: Build and push" + command: docker/build diff --git a/development.md b/development.md new file mode 100644 index 0000000..1bbd75a --- /dev/null +++ b/development.md @@ -0,0 +1,115 @@ +# Development notes + +Because this uses docker, vault and requires work with hostnames, this is going to be hard to test properly without a lot of mocking. We'll update this as our strategy improves. + +## Vault server for testing + +We use [`vault-dev`](https://github.com/vimc/vault-dev) to bring up vault in testing mode. You can also do this manually (e.g., to match the configuration in [`example/simple.json`](example/simple.json) by running + +``` +vault server -dev -dev-kv-v1 +``` + +If you need to interact with this on the command line, use: + +``` +export VAULT_ADDR='http://127.0.0.1:8200' +export VAULT_TOKEN=$(cat ~/.vault-token) +``` + +within the hatch environment before running any commands. + +## Worked example + +We need to swap in the globally-findable address for alice (`alice.example.com`) for the value of the machine this is tested on: + +``` +mkdir -p tmp +sed "s/alice.example.com/$(hostname)/" example/local.json > tmp/privateer.json +``` + +Create a set of keys + +``` +privateer2 --path tmp keygen --all +``` + +You could also do this individually like + +``` +privateer2 --path tmp keygen alice +``` + +Set up the key volumes + +``` +privateer2 --path tmp configure alice +privateer2 --path tmp configure bob +``` + +Start the server, as a background process (note that if these were on different machine the `privateer2 configure ` step would generate the `.privateer_identity` automatically so the `--as` argument is not needed) + +``` +privateer2 --path tmp --as=alice server start +``` + +Once `alice` is running, we can test this connection from `bob`: + +``` +privateer2 --path tmp --as=bob check --connection +``` + +This command would be simpler to run if we are in the `tmp` directory, which would be the usual situation in a multi-machine setup + +``` +privateer2 check --connection +``` + +For all other commands below, you can drop the `--path` and `--as` arguments if you change directory. + +Create some random data within the `data` volume (this is the one that we want to send from `bob` to `alice`) + +``` +docker volume create data +docker run -it --rm -v data:/data ubuntu bash -c "base64 /dev/urandom | head -c 100000 > /data/file1.txt" +``` + +We can now backup from `bob` to `alice` as: + +``` +privateer2 --path tmp --as=bob backup data +``` + +or see what commands you would need in order to try this yourself: + +``` +privateer2 --path tmp --as=bob backup data --dry-run +``` + +Delete the volume + +``` +docker volume rm data +``` + +We can now restore it: + +``` +privateer2 --path tmp --as=bob restore data +``` + +or see the commands to do this ourselves: + +``` +privateer2 --path tmp --as=bob restore data --dry-run +``` + +Tear down the server with + +``` +privateer2 --path tmp --as=alice server stop +``` + +## Writing tests + +We use a lot of global resources, so it's easy to leave behind volumes and containers (often exited) after running tests. At best this is lazy and messy, but at worst it creates hard-to-diagnose dependencies between tests. Try and create names for auto-cleaned volumes and containers using the `managed_docker` fixture (see [`tests/conftest.py`](tests/conftest.py) for details). diff --git a/docker/Dockerfile.client b/docker/Dockerfile.client new file mode 100644 index 0000000..67de0c4 --- /dev/null +++ b/docker/Dockerfile.client @@ -0,0 +1,10 @@ +FROM ubuntu + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + openssh-client \ + rsync && \ + mkdir -p /root/.ssh + +COPY ssh_config /etc/ssh/ssh_config +VOLUME /privateer/keys diff --git a/docker/Dockerfile.server b/docker/Dockerfile.server new file mode 100644 index 0000000..7bf3f48 --- /dev/null +++ b/docker/Dockerfile.server @@ -0,0 +1,17 @@ +FROM ubuntu + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + openssh-client \ + openssh-server \ + rsync && \ + mkdir -p /var/run/sshd && \ + mkdir -p /root/.ssh + +COPY sshd_config /etc/ssh/sshd_config + +VOLUME /privateer/keys +VOLUME /privateer/volumes +EXPOSE 22 + +ENTRYPOINT ["/usr/sbin/sshd", "-D", "-E", "/dev/stderr"] diff --git a/docker/build b/docker/build new file mode 100755 index 0000000..399e264 --- /dev/null +++ b/docker/build @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -exu + +HERE=$(dirname $0) +. $HERE/common + +docker build --pull \ + --tag $TAG_SERVER_SHA \ + -f $HERE/Dockerfile.server \ + $HERE + +docker build --pull \ + --tag $TAG_CLIENT_SHA \ + -f $HERE/Dockerfile.client \ + $HERE + +docker push $TAG_SERVER_SHA +docker push $TAG_CLIENT_SHA + +docker tag $TAG_SERVER_SHA $TAG_SERVER_BRANCH +docker push $TAG_SERVER_BRANCH +docker tag $TAG_CLIENT_SHA $TAG_CLIENT_BRANCH +docker push $TAG_CLIENT_BRANCH + +if [ $GIT_BRANCH == "main" ]; then + docker tag $TAG_SERVER_SHA $TAG_SERVER_LATEST + docker push $TAG_SERVER_LATEST + docker tag $TAG_CLIENT_SHA $TAG_CLIENT_LATEST + docker push $TAG_CLIENT_LATEST +fi diff --git a/docker/common b/docker/common new file mode 100644 index 0000000..4a622e3 --- /dev/null +++ b/docker/common @@ -0,0 +1,24 @@ +# -*-sh-*- +DOCKER_ROOT=$(realpath $HERE/..) +PACKAGE_ORG=mrcide +CLIENT_NAME=privateer-client +SERVER_NAME=privateer-server + +# Buildkite doesn't check out a full history from the remote (just the +# single commit) so you end up with a detached head and git rev-parse +# doesn't work +if [ false && "$BUILDKITE" = "true" ]; then + GIT_SHA=${BUILDKITE_COMMIT:0:7} + GIT_BRANCH=$BUILDKITE_BRANCH +else + GIT_SHA=$(git -C "$DOCKER_ROOT" rev-parse --short=7 HEAD) + GIT_BRANCH=$(git -C "$DOCKER_ROOT" symbolic-ref --short HEAD) +fi + +TAG_CLIENT_SHA="${PACKAGE_ORG}/${CLIENT_NAME}:${GIT_SHA}" +TAG_CLIENT_BRANCH="${PACKAGE_ORG}/${CLIENT_NAME}:${GIT_BRANCH}" +TAG_CLIENT_LATEST="${PACKAGE_ORG}/${CLIENT_NAME}:latest" + +TAG_SERVER_SHA="${PACKAGE_ORG}/${SERVER_NAME}:${GIT_SHA}" +TAG_SERVER_BRANCH="${PACKAGE_ORG}/${SERVER_NAME}:${GIT_BRANCH}" +TAG_SERVER_LATEST="${PACKAGE_ORG}/${SERVER_NAME}:latest" diff --git a/docker/ssh_config b/docker/ssh_config new file mode 100644 index 0000000..bab51c4 --- /dev/null +++ b/docker/ssh_config @@ -0,0 +1,6 @@ +PasswordAuthentication no +IdentityFile /privateer/keys/id_rsa +SendEnv LANG LC_* +HashKnownHosts no +UserKnownHostsFile /privateer/keys/known_hosts +Include /privateer/keys/config diff --git a/docker/sshd_config b/docker/sshd_config new file mode 100644 index 0000000..4c70abf --- /dev/null +++ b/docker/sshd_config @@ -0,0 +1,28 @@ +Include /etc/ssh/sshd_config.d/*.conf + +PermitRootLogin prohibit-password +#StrictModes yes +#MaxAuthTries 1 +#MaxSessions 10 + +PubkeyAuthentication yes + +AuthorizedKeysFile /privateer/keys/authorized_keys +HostKey /privateer/keys/id_rsa + +PasswordAuthentication no +ChallengeResponseAuthentication no +UsePAM no + +#AllowAgentForwarding yes +#AllowTcpForwarding yes +#GatewayPorts no +X11Forwarding no +# PermitTTY no +PrintMotd no + +# Allow client to pass locale environment variables +AcceptEnv LANG LC_* + +# override default of no subsystems +# Subsystem sftp /usr/lib/openssh/sftp-server diff --git a/example/local.json b/example/local.json new file mode 100644 index 0000000..4f2b57b --- /dev/null +++ b/example/local.json @@ -0,0 +1,32 @@ +{ + "servers": [ + { + "name": "alice", + "hostname": "alice.example.com", + "port": 10022, + "key_volume": "privateer_keys_alice", + "data_volume": "privateer_data_alice" + } + ], + "clients": [ + { + "name": "bob", + "backup": ["data"], + "restore": ["data", "other"], + "key_volume": "privateer_keys_bob" + } + ], + "volumes": [ + { + "name": "data" + }, + { + "name": "other", + "local": true + } + ], + "vault": { + "url": "http://localhost:8200", + "prefix": "/secret/privateer" + } +} diff --git a/example/montagu.json b/example/montagu.json new file mode 100644 index 0000000..9af7402 --- /dev/null +++ b/example/montagu.json @@ -0,0 +1,47 @@ +{ + "servers": [ + { + "name": "annex", + "hostname": "annex.montagu.dide.ic.ac.uk", + "port": 10022 + }, + { + "name": "annex2", + "hostname": "annex2.montagu.dide.ic.ac.uk", + "port": 10022 + } + ], + "clients": [ + { + "name": "production", + "backup": ["montagu_orderly_volume"], + "restore": ["montagu_orderly_volume", "barman_recover"] + }, + { + "name": "production2", + "backup": ["montagu_orderly_volume"], + "restore": ["montagu_orderly_volume", "barman_recover"] + }, + { + "name": "science", + "restore": ["montagu_orderly_volume", "barman_recover"] + }, + { + "name": "uat", + "restore": ["montagu_orderly_volume", "barman_recover"] + } + ], + "volumes": [ + { + "name": "montagu_orderly_volume" + }, + { + "name": "barman_recover", + "local": true + } + ], + "vault": { + "url": "https://vault.dide.ic.ac.uk:8200", + "prefix": "/secret/vimc/privateer" + } +} diff --git a/example/simple.json b/example/simple.json new file mode 100644 index 0000000..57ae807 --- /dev/null +++ b/example/simple.json @@ -0,0 +1,25 @@ +{ + "servers": [ + { + "name": "alice", + "hostname": "alice.example.com", + "port": 10022 + } + ], + "clients": [ + { + "name": "bob", + "backup": ["data"], + "restore": ["data"] + } + ], + "volumes": [ + { + "name": "data" + } + ], + "vault": { + "url": "http://localhost:8200", + "prefix": "/secret/privateer" + } +} diff --git a/pyproject.toml b/pyproject.toml index bfcc9b5..2f7fa5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,27 +24,38 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dependencies = [] +dependencies = [ + "cryptography>=3.1", + "docker", + "docopt", + "hvac", + "pydantic" +] [project.urls] Documentation = "https://github.com/unknown/privateer2#readme" Issues = "https://github.com/unknown/privateer2/issues" Source = "https://github.com/unknown/privateer2" +[project.scripts] +privateer2 = "privateer2.cli:main" + [tool.hatch.version] path = "src/privateer2/__about__.py" [tool.hatch.envs.default] +python = "python3" dependencies = [ "coverage[toml]>=6.5", "pytest", + "vault-dev>=0.1.1" ] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" test-cov = "coverage run -m pytest {args:tests}" cov-report = [ "- coverage combine", - "coverage report", + "coverage report --show-missing", ] cov = [ "test-cov", @@ -87,12 +98,12 @@ all = [ [tool.black] target-version = ["py37"] -line-length = 120 +line-length = 80 skip-string-normalization = true [tool.ruff] target-version = "py37" -line-length = 120 +line-length = 80 select = [ "A", "ARG", @@ -129,6 +140,10 @@ ignore = [ "S105", "S106", "S107", # Ignore complexity "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", + # Let us use random + "S311", + # Let us use print + "T201" ] unfixable = [ # Don't touch unused imports diff --git a/src/privateer2/__about__.py b/src/privateer2/__about__.py index 8813381..df46152 100644 --- a/src/privateer2/__about__.py +++ b/src/privateer2/__about__.py @@ -1,4 +1,4 @@ # SPDX-FileCopyrightText: 2023-present Rich FitzJohn # # SPDX-License-Identifier: MIT -__version__ = "0.0.1" +__version__ = "0.0.2" diff --git a/src/privateer2/backup.py b/src/privateer2/backup.py new file mode 100644 index 0000000..577c9a5 --- /dev/null +++ b/src/privateer2/backup.py @@ -0,0 +1,50 @@ +import docker +from privateer2.keys import check +from privateer2.util import match_value, mounts_str, run_container_with_command + + +def backup_command(name, volume, server): + return [ + "rsync", + "-av", + "--delete", + f"/privateer/{volume}", + f"{server}:/privateer/volumes/{name}", + ] + + +def backup(cfg, name, volume, *, server=None, dry_run=False): + machine = check(cfg, name, quiet=True) + server = match_value(server, cfg.list_servers(), "server") + volume = match_value(volume, machine.backup, "volume") + image = f"mrcide/privateer-client:{cfg.tag}" + src = f"/privateer/{volume}" + mounts = [ + docker.types.Mount( + "/privateer/keys", machine.key_volume, type="volume", read_only=True + ), + docker.types.Mount(src, volume, type="volume", read_only=True), + ] + command = backup_command(name, volume, server) + if dry_run: + cmd = ["docker", "run", "--rm", *mounts_str(mounts), image, *command] + print("Command to manually run backup:") + print() + print(f" {' '.join(cmd)}") + print() + print( + f"This will copy the volume '{volume}' from '{name}' " + f"to the server '{server}'" + ) + print() + print("Note that this uses hostname/port information for the server") + print("contained within (config), along with our identity (id_rsa)") + print("in the directory /privateer/keys") + else: + print(f"Backing up '{volume}' from '{name}' to '{server}'") + run_container_with_command( + "Backup", image, command=command, mounts=mounts + ) + # TODO: also copy over some metadata at this point, via + # ssh; probably best to write tiny utility in the client + # container that will do this for us. diff --git a/src/privateer2/cli.py b/src/privateer2/cli.py new file mode 100644 index 0000000..4d5002a --- /dev/null +++ b/src/privateer2/cli.py @@ -0,0 +1,167 @@ +"""Usage: + privateer2 --version + privateer2 [options] pull + privateer2 [options] keygen ( | --all) + privateer2 [options] configure + privateer2 [options] check [--connection] + privateer2 [options] server (start | stop | status) + privateer2 [options] backup [--server=NAME] + privateer2 [options] restore [--server=NAME] [--source=NAME] + +Options: + --path=PATH The path to the configuration, or directory with privateer.json + --as=NAME The machine to run the command as + --dry-run Do nothing, but print docker commands + +Commentary: + In all the above '--as' (or ) refers to the name of the client + or server being acted on; the machine we are generating keys for, + configuring, checking, serving, backing up from or restoring to. +""" + +import os + +import docopt + +import docker +import privateer2.__about__ as about +from privateer2.backup import backup +from privateer2.config import read_config +from privateer2.keys import check, configure, keygen, keygen_all +from privateer2.restore import restore +from privateer2.server import server_start, server_status, server_stop + + +def pull(cfg): + img = [ + f"mrcide/privateer-client:{cfg.tag}", + f"mrcide/privateer-server:{cfg.tag}", + ] + cl = docker.from_env() + for nm in img: + print(f"pulling '{nm}'") + cl.images.pull(nm) + + +def _dont_use(name, opts, cmd): + if opts[name]: + msg = f"Don't use '{name}' with '{cmd}'" + raise Exception(msg) + + +def _find_identity(name, root_config): + if name: + return name + path_as = os.path.join(root_config, ".privateer_identity") + if not os.path.exists(path_as): + msg = ( + "Can't determine identity; did you forget to configure?" + "Alternatively, pass '--as=NAME' to this command" + ) + raise Exception(msg) + with open(path_as) as f: + return f.read().strip() + + +def _do_configure(cfg, name, root): + configure(cfg, name) + with open(os.path.join(root, ".privateer_identity"), "w") as f: + f.write(f"{name}\n") + + +def _show_version(): + print(f"privateer {about.__version__}") + + +class Call: + def __init__(self, target, **kwargs): + self.target = target + self.kwargs = kwargs + + def run(self): + return self.target(**self.kwargs) + + def __eq__(self, other): + return self.target == other.target and self.kwargs == other.kwargs + + +def _parse_argv(argv): + opts = docopt.docopt(__doc__, argv) + return _parse_opts(opts) + + +def _path_config(path): + if not path: + path = "privateer.json" + elif os.path.isdir(path): + path = os.path.join(path, "privateer.json") + if not os.path.exists(path): + msg = f"Did not find privateer configuration at '{path}'" + raise Exception(msg) + return path + + +def _parse_opts(opts): + if opts["--version"]: + return Call(_show_version) + + dry_run = opts["--dry-run"] + name = opts["--as"] + path_config = _path_config(opts["--path"]) + root_config = os.path.dirname(path_config) + cfg = read_config(path_config) + if opts["keygen"]: + _dont_use("--as", opts, "keygen") + if opts["--all"]: + return Call(keygen_all, cfg=cfg) + else: + return Call(keygen, cfg=cfg, name=opts[""]) + elif opts["configure"]: + _dont_use("--as", opts, "configure") + return Call( + _do_configure, + cfg=cfg, + name=opts[""], + root=root_config, + ) + elif opts["pull"]: + _dont_use("--as", opts, "configure") + return Call(pull, cfg=cfg) + else: + name = _find_identity(opts["--as"], root_config) + if opts["check"]: + connection = opts["--connection"] + return Call(check, cfg=cfg, name=name, connection=connection) + elif opts["server"]: + if opts["start"]: + return Call(server_start, cfg=cfg, name=name, dry_run=dry_run) + elif opts["stop"]: + return Call(server_stop, cfg=cfg, name=name) + else: + return Call(server_status, cfg=cfg, name=name) + elif opts["backup"]: + return Call( + backup, + cfg=cfg, + name=name, + volume=opts[""], + server=opts["--server"], + dry_run=dry_run, + ) + elif opts["restore"]: + return Call( + restore, + cfg=cfg, + name=name, + volume=opts[""], + server=opts["--server"], + source=opts["--source"], + dry_run=dry_run, + ) + else: + msg = "Invalid cli call -- privateer bug" + raise Exception(msg) + + +def main(argv=None): + _parse_argv(argv).run() diff --git a/src/privateer2/config.py b/src/privateer2/config.py new file mode 100644 index 0000000..f7cf7c4 --- /dev/null +++ b/src/privateer2/config.py @@ -0,0 +1,114 @@ +import json +from typing import List + +from pydantic import BaseModel + +from privateer2.util import match_value +from privateer2.vault import vault_client + + +def read_config(path): + with open(path) as f: + return Config(**json.loads(f.read().strip())) + + +class Server(BaseModel): + name: str + hostname: str + port: int + key_volume: str = "privateer_keys" + data_volume: str = "privateer_data" + container: str = "privateer_server" + + +class Client(BaseModel): + name: str + backup: List[str] = [] + restore: List[str] = [] + key_volume: str = "privateer_keys" + + +class Volume(BaseModel): + name: str + local: bool = False + + +class Vault(BaseModel): + url: str + prefix: str + + def client(self): + return vault_client(self.url) + + +class Config(BaseModel): + servers: List[Server] + clients: List[Client] + volumes: List[Volume] + vault: Vault + tag: str = "prototype" + + def model_post_init(self, __context): + _check_config(self) + + def list_servers(self): + return [x.name for x in self.servers] + + def list_clients(self): + return [x.name for x in self.clients] + + def machine_config(self, name): + for el in self.servers + self.clients: + if el.name == name: + return el + valid = self.list_servers() + self.list_clients() + valid_str = ", ".join(f"'{x}'" for x in valid) + msg = f"Invalid configuration '{name}', must be one of {valid_str}" + raise Exception(msg) + + +# this could be put elsewhere; we find the plausible sources (original +# clients) that backed up a source to any server. +def find_source(cfg, volume, source): + for v in cfg.volumes: + if v.name == volume and v.local: + if source is not None: + msg = f"'{volume}' is a local source, so 'source' must be empty" + raise Exception(msg) + return "local" + pos = [cl.name for cl in cfg.clients if volume in cl.backup] + return match_value(source, pos, "source") + + +def _check_config(cfg): + servers = cfg.list_servers() + clients = cfg.list_clients() + _check_not_duplicated(servers, "servers") + _check_not_duplicated(clients, "clients") + err = set(cfg.list_servers()).intersection(set(cfg.list_clients())) + if err: + err_str = ", ".join(f"'{nm}'" for nm in err) + msg = f"Invalid machine listed as both a client and a server: {err_str}" + raise Exception(msg) + vols_local = [x.name for x in cfg.volumes if x.local] + vols_all = [x.name for x in cfg.volumes] + for cl in cfg.clients: + for v in cl.restore: + if v not in vols_all: + msg = f"Client '{cl.name}' restores from unknown volume '{v}'" + raise Exception(msg) + for v in cl.backup: + if v not in vols_all: + msg = f"Client '{cl.name}' backs up unknown volume '{v}'" + raise Exception(msg) + if v in vols_local: + msg = f"Client '{cl.name}' backs up local volume '{v}'" + raise Exception(msg) + if cfg.vault.prefix.startswith("/secret"): + cfg.vault.prefix = cfg.vault.prefix[7:] + + +def _check_not_duplicated(els, name): + if len(els) > len(set(els)): + msg = f"Duplicated elements in {name}" + raise Exception(msg) diff --git a/src/privateer2/keys.py b/src/privateer2/keys.py new file mode 100644 index 0000000..cf8b987 --- /dev/null +++ b/src/privateer2/keys.py @@ -0,0 +1,162 @@ +from cryptography.hazmat.primitives import serialization as crypto_serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +import docker +from privateer2.util import string_from_volume, string_to_volume + + +def keygen(cfg, name): + _keygen(cfg, name, cfg.vault.client()) + + +def keygen_all(cfg): + vault = cfg.vault.client() + for name in cfg.list_servers() + cfg.list_clients(): + _keygen(cfg, name, vault) + + +def _keygen(cfg, name, vault): + data = _create_keypair() + path = f"{cfg.vault.prefix}/{name}" + # TODO: The docs are here: + # https://hvac.readthedocs.io/en/stable/usage/secrets_engines/kv_v1.html + # They do not indicate if this will error if the write fails though. + print(f"Writing keypair for {name}") + _r = vault.secrets.kv.v1.create_or_update_secret(path, secret=data) + + +def configure(cfg, name): + cl = docker.from_env() + data = _keys_data(cfg, name) + vol = cfg.machine_config(name).key_volume + cl.volumes.create(vol) + print(f"Copying keypair for '{name}' to volume '{vol}'") + string_to_volume( + data["public"], vol, "id_rsa.pub", uid=0, gid=0, mode=0o644 + ) + string_to_volume(data["private"], vol, "id_rsa", uid=0, gid=0, mode=0o600) + if data["authorized_keys"]: + print("Authorising public keys") + string_to_volume( + data["authorized_keys"], + vol, + "authorized_keys", + uid=0, + gid=0, + mode=0o600, + ) + if data["known_hosts"]: + print("Recognising servers") + string_to_volume( + data["known_hosts"], vol, "known_hosts", uid=0, gid=0, mode=0o600 + ) + if data["config"]: + print("Adding ssh config") + string_to_volume( + data["config"], vol, "config", uid=0, gid=0, mode=0o600 + ) + string_to_volume(name, vol, "name", uid=0, gid=0) + + +def check(cfg, name, *, connection=False, quiet=False): + machine = cfg.machine_config(name) + vol = machine.key_volume + try: + docker.from_env().volumes.get(vol) + except docker.errors.NotFound: + msg = f"'{name}' looks unconfigured" + raise Exception(msg) from None + found = string_from_volume(vol, "name") + if found != name: + msg = f"Configuration is for '{found}', not '{name}'" + raise Exception(msg) + if not quiet: + print(f"Volume '{vol}' looks configured as '{name}'") + if connection and name in cfg.list_clients(): + _check_connections(cfg, machine) + return machine + + +def _get_pubkeys(vault, prefix, nms): + return { + nm: vault.secrets.kv.v1.read_secret(f"{prefix}/{nm}")["data"]["public"] + for nm in nms + } + + +def _create_keypair(): + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + private = key.private_bytes( + crypto_serialization.Encoding.PEM, + crypto_serialization.PrivateFormat.PKCS8, + crypto_serialization.NoEncryption(), + ).decode("UTF-8") + + public = ( + key.public_key() + .public_bytes( + crypto_serialization.Encoding.OpenSSH, + crypto_serialization.PublicFormat.OpenSSH, + ) + .decode("UTF-8") + ) + + return {"public": public, "private": private} + + +def _keys_data(cfg, name): + vault = cfg.vault.client() + response = vault.secrets.kv.v1.read_secret(f"{cfg.vault.prefix}/{name}") + ret = { + "name": name, + **response["data"], + "authorized_keys": None, + "known_hosts": None, + "config": None, + } + if name in cfg.list_servers(): + keys = _get_pubkeys(vault, cfg.vault.prefix, cfg.list_clients()) + ret["authorized_keys"] = "".join([f"{v}\n" for v in keys.values()]) + if name in cfg.list_clients(): + keys = _get_pubkeys(vault, cfg.vault.prefix, cfg.list_servers()) + known_hosts = [] + config = [] + for s in cfg.servers: + known_hosts.append(f"[{s.hostname}]:{s.port} {keys[s.name]}\n") + config.append(f"Host {s.name}\n") + config.append(" User root\n") + config.append(f" Port {s.port}\n") + config.append(f" HostName {s.hostname}\n") + ret["known_hosts"] = "".join(known_hosts) + ret["config"] = "".join(config) + return ret + + +def _check_connections(cfg, machine): + image = f"mrcide/privateer-client:{cfg.tag}" + mounts = [ + docker.types.Mount( + "/privateer/keys", machine.key_volume, type="volume", read_only=True + ) + ] + cl = docker.from_env() + result = {} + for server in cfg.servers: + print( + f"checking connection to '{server.name}' ({server.hostname})...", + end="", + flush=True, + ) + try: + command = ["ssh", server.name, "cat", "/privateer/keys/name"] + cl.containers.run( + image, mounts=mounts, command=command, remove=True + ) + result[server.name] = True + print("OK") + except docker.errors.ContainerError as e: + result[server.name] = False + print("ERROR") + print(e.stderr.decode("utf-8").strip()) + return result diff --git a/src/privateer2/restore.py b/src/privateer2/restore.py new file mode 100644 index 0000000..20656b0 --- /dev/null +++ b/src/privateer2/restore.py @@ -0,0 +1,43 @@ +import docker +from privateer2.config import find_source +from privateer2.keys import check +from privateer2.util import match_value, mounts_str, run_container_with_command + + +def restore(cfg, name, volume, *, server=None, source=None, dry_run=False): + machine = check(cfg, name, quiet=True) + server = match_value(server, cfg.list_servers(), "server") + volume = match_value(volume, machine.backup, "volume") + source = find_source(cfg, volume, source) + image = f"mrcide/privateer-client:{cfg.tag}" + dest_mount = f"/privateer/{volume}" + mounts = [ + docker.types.Mount( + "/privateer/keys", machine.key_volume, type="volume", read_only=True + ), + docker.types.Mount(dest_mount, volume, type="volume", read_only=False), + ] + command = [ + "rsync", + "-av", + "--delete", + f"{server}:/privateer/volumes/{name}/{volume}/", + f"{dest_mount}/", + ] + if dry_run: + cmd = ["docker", "run", "--rm", *mounts_str(mounts), image, *command] + print("Command to manually run restore:") + print() + print(f" {' '.join(cmd)}") + print() + print(f"This will data from the server '{server}' into into our") + print(f"local volume '{volume}'; data originally from '{source}'") + print() + print("Note that this uses hostname/port information for the server") + print("contained within (config), along with our identity (id_rsa)") + print("in the directory /privateer/keys") + else: + print(f"Restoring '{volume}' from '{server}'") + run_container_with_command( + "Restore", image, command=command, mounts=mounts + ) diff --git a/src/privateer2/server.py b/src/privateer2/server.py new file mode 100644 index 0000000..dcd2d3c --- /dev/null +++ b/src/privateer2/server.py @@ -0,0 +1,45 @@ +import docker +from privateer2.keys import check +from privateer2.service import service_start, service_status, service_stop + + +def server_start(cfg, name, *, dry_run=False): + machine = check(cfg, name, quiet=True) + + mounts = [ + docker.types.Mount( + "/privateer/keys", machine.key_volume, type="volume", read_only=True + ), + docker.types.Mount( + "/privateer/volumes", machine.data_volume, type="volume" + ), + ] + for v in cfg.volumes: + if v.local: + mounts.append( + docker.types.Mount( + f"/privateer/local/{v.name}", + v.name, + type="volume", + read_only=True, + ) + ) + service_start( + name, + machine.container, + image=f"mrcide/privateer-server:{cfg.tag}", + mounts=mounts, + ports={"22/tcp": machine.port}, + dry_run=dry_run, + ) + print(f"Server {name} now running on port {machine.port}") + + +def server_stop(cfg, name): + machine = check(cfg, name, quiet=True) + service_stop(name, machine.container) + + +def server_status(cfg, name): + machine = check(cfg, name, quiet=False) + service_status(machine.container) diff --git a/src/privateer2/service.py b/src/privateer2/service.py new file mode 100644 index 0000000..ffd53ce --- /dev/null +++ b/src/privateer2/service.py @@ -0,0 +1,79 @@ +import docker +from privateer2.util import ( + container_exists, + container_if_exists, + ensure_image, + mounts_str, + ports_str, +) + + +def service_command(image, name, *, mounts=None, ports=None, command=None): + return [ + "docker", + "run", + "--rm", + "-d", + "--name", + name, + *mounts_str(mounts), + *ports_str(ports), + image, + *(command or []), + ] + + +def service_start( + name, + container_name, + image, + *, + dry_run=False, + mounts=None, + ports=None, + command=None, +): + if dry_run: + cmd = service_command( + image, container_name, mounts=mounts, ports=ports, command=command + ) + print("Command to manually launch server:") + print() + print(f" {' '.join(cmd)}") + print() + print("(remove the '-d' flag to run in blocking mode)") + return + + if container_exists(container_name): + msg = f"Container '{container_name}' for '{name}' already running" + raise Exception(msg) + + ensure_image(image) + print("Starting server '{name}' as container '{container_name}'") + client = docker.from_env() + client.containers.run( + image, + auto_remove=True, + detach=True, + name=container_name, + mounts=mounts, + ports=ports, + command=command, + ) + + +def service_stop(name, container_name): + container = container_if_exists(container_name) + if container: + if container.status == "running": + container.stop() + else: + print(f"Container '{container_name}' for '{name}' does not exist") + + +def service_status(container_name): + container = container_if_exists(container_name) + if container: + print(container.status) + else: + print("not running") diff --git a/src/privateer2/util.py b/src/privateer2/util.py new file mode 100644 index 0000000..d5effc9 --- /dev/null +++ b/src/privateer2/util.py @@ -0,0 +1,264 @@ +import datetime +import os +import os.path +import random +import re +import string +import tarfile +import tempfile +from contextlib import contextmanager +from pathlib import Path + +import docker + + +def string_to_volume(text, volume, path, **kwargs): + if isinstance(text, list): + text = "".join(x + "\n" for x in text) + ensure_image("alpine") + dest = Path("/dest") + mounts = [docker.types.Mount(str(dest), volume, type="volume")] + cl = docker.from_env() + container = cl.containers.create("alpine", mounts=mounts) + try: + string_to_container(text, container, dest / path, **kwargs) + finally: + container.remove() + + +def string_from_volume(volume, path): + ensure_image("alpine") + src = Path("/src") + mounts = [docker.types.Mount(str(src), volume, type="volume")] + cl = docker.from_env() + container = cl.containers.create("alpine", mounts=mounts) + try: + return string_from_container(container, src / path) + finally: + container.remove() + + +def string_to_container(text, container, path, **kwargs): + with simple_tar_string(text, os.path.basename(path), **kwargs) as tar: + container.put_archive(os.path.dirname(path), tar) + + +def string_from_container(container, path): + return bytes_from_container(container, path).decode("utf-8") + + +def bytes_from_container(container, path): + stream, status = container.get_archive(path) + try: + fd, tmp = tempfile.mkstemp(text=False) + with os.fdopen(fd, "wb") as f: + for d in stream: + f.write(d) + with open(tmp, "rb") as f: + t = tarfile.open(mode="r", fileobj=f) + p = t.extractfile(os.path.basename(path)) + return p.read() + finally: + os.remove(tmp) + + +def set_permissions(mode=None, uid=None, gid=None): + def ret(tarinfo): + if mode is not None: + tarinfo.mode = mode + if uid is not None: + tarinfo.uid = uid + if gid is not None: + tarinfo.gid = gid + return tarinfo + + return ret + + +def simple_tar_string(text, name, **kwargs): + text = bytes(text, "utf-8") + try: + fd, tmp = tempfile.mkstemp(text=True) + with os.fdopen(fd, "wb") as f: + f.write(text) + return simple_tar(tmp, name, **kwargs) + finally: + os.remove(tmp) + + +def simple_tar(path, name, **kwargs): + f = tempfile.NamedTemporaryFile() + t = tarfile.open(mode="w", fileobj=f) + abs_path = os.path.abspath(path) + t.add( + abs_path, + arcname=name, + recursive=False, + filter=set_permissions(**kwargs), + ) + t.close() + f.seek(0) + return f + + +@contextmanager +def transient_envvar(**kwargs): + prev = { + k: os.environ[k] if k in os.environ else None for k in kwargs.keys() + } + try: + _setdictvals(kwargs, os.environ) + yield + finally: + _setdictvals(prev, os.environ) + + +def _setdictvals(new, container): + for k, v in new.items(): + if v is None: + if k in container: + del container[k] + else: + container[k] = v + return container + + +def ensure_image(name): + cl = docker.from_env() + try: + cl.images.get(name) + except docker.errors.ImageNotFound: + print(f"Pulling {name}") + cl.images.pull(name) + + +def container_exists(name): + return bool(container_if_exists(name)) + + +def container_if_exists(name): + try: + return docker.from_env().containers.get(name) + except docker.errors.NotFound: + return None + + +def volume_exists(name): + return bool(volume_if_exists(name)) + + +def volume_if_exists(name): + try: + return docker.from_env().volumes.get(name) + except docker.errors.NotFound: + return None + + +def rand_str(n=8): + return "".join(random.choices(string.ascii_lowercase + string.digits, k=n)) + + +def log_tail(container, n): + logs = container.logs().decode("utf-8").strip().split("\n") + if len(logs) > n: + return [f"(ommitting {len(logs) - n} lines of logs)"] + logs[-n:] + else: + return logs + + +def mounts_str(mounts): + ret = [] + if mounts: + for m in mounts: + ret += mount_str(m) + return ret + + +def mount_str(mount): + ret = f"{mount['Source']}:{mount['Target']}" + if mount["ReadOnly"]: + ret += ":ro" + return ["-v", ret] + + +# This could be improved, there are more formats possible here. +def ports_str(ports): + ret = [] + if ports: + for k, v in ports.items(): + ret.append("-p") + ret.append(f"{v}:{re.sub('/.+', '', k)}") + return ret + + +def match_value(given, valid, name): + if given is None: + if len(valid) == 1: + return valid[0] + msg = f"Please provide a value for {name}" + raise Exception(msg) + if given not in valid: + valid_str = ", ".join([f"'{x}'" for x in valid]) + msg = f"Invalid {name} '{given}': valid options: {valid_str}" + raise Exception(msg) + return given + + +def isotimestamp(): + now = datetime.datetime.now(tz=datetime.timezone.utc) + return now.strftime("%Y%m%d-%H%M%S") + + +def take_ownership(filename, directory, *, command_only=False): # tar + uid = os.geteuid() + gid = os.getegid() + cl = docker.from_env() + ensure_image("alpine") + mounts = [docker.types.Mount("/src", directory, type="bind")] + command = ["chown", f"{uid}.{gid}", filename] + if command_only: + return [ + "docker", + "run", + *mounts_str(mounts), + "-w", + "/src", + "alpine", + *command, + ] + else: + cl.containers.run( + "alpine", + mounts=mounts, + working_dir="/src", + command=command, + remove=True, + ) + + +def run_container_with_command(display, image, **kwargs): + ensure_image(image) + client = docker.from_env() + container = client.containers.run(image, **kwargs, detach=True) + print(f"{display} command started. To stream progress, run:") + print(f" docker logs -f {container.name}") + result = container.wait() + if result["StatusCode"] == 0: + print(f"{display} completed successfully! Container logs:") + print("\n".join(log_tail(container, 10))) + container.remove() + else: + print("An error occured! Container logs:") + print("\n".join(log_tail(container, 20))) + msg = f"{display} failed; see {container.name} logs for details" + raise Exception(msg) + + +@contextmanager +def transient_working_directory(path): + origin = os.getcwd() + try: + os.chdir(path) + yield + finally: + os.chdir(origin) diff --git a/src/privateer2/vault.py b/src/privateer2/vault.py new file mode 100644 index 0000000..9313fc1 --- /dev/null +++ b/src/privateer2/vault.py @@ -0,0 +1,33 @@ +import os +import re + +import hvac + + +def vault_client(addr, token=None): + token = _get_vault_token(token) + if _is_github_token(token): + print("logging into vault using github") + client = hvac.Client(addr) + client.auth.github.login(token) + else: + client = hvac.Client(addr, token=token) + return client + + +def _get_vault_token(token): + if token is not None: + re_envvar = re.compile("^\\$[A-Z0-9_-]+$") + if re_envvar.match(token): + token = os.environ[token[1:]] + return token + check = ["VAULT_TOKEN", "VAULT_AUTH_GITHUB_TOKEN"] + for token_type in check: + if token_type in os.environ: + return os.environ[token_type] + return input("Enter token for vault: ").strip() + + +def _is_github_token(token): + re_gh = re.compile("^ghp_[A-Za-z0-9]{36}$") + return re_gh.match(token) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7e741df --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,33 @@ +import pytest +import vault_dev + +from privateer2.util import ( + container_if_exists, + match_value, + rand_str, + volume_if_exists, +) + +vault_dev.ensure_installed() + + +@pytest.fixture +def managed_docker(): + created = {"container": [], "volume": []} + + def _new(what, *, prefix="privateer_test"): + match_value(what, ["container", "volume"], "what") + name = f"{prefix}_{rand_str()}" + created[what].append(name) + return name + + yield _new + + for name in created["container"]: + container = container_if_exists(name) + if container: + container.remove() + for name in created["volume"]: + volume = volume_if_exists(name) + if volume: + volume.remove() diff --git a/tests/test_backup.py b/tests/test_backup.py new file mode 100644 index 0000000..a999a60 --- /dev/null +++ b/tests/test_backup.py @@ -0,0 +1,67 @@ +from unittest.mock import MagicMock, call + +import vault_dev + +import docker +import privateer2.server +from privateer2.backup import backup +from privateer2.config import read_config +from privateer2.keys import configure, keygen_all + + +def test_can_print_instructions_to_run_backup(capsys, managed_docker): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol = managed_docker("volume") + cfg.clients[0].key_volume = vol + keygen_all(cfg) + configure(cfg, "bob") + capsys.readouterr() # flush previous output + backup(cfg, "bob", "data", dry_run=True) + out = capsys.readouterr() + lines = out.out.strip().split("\n") + assert "Command to manually run backup:" in lines + cmd = ( + " docker run --rm " + f"-v {vol}:/privateer/keys:ro -v data:/privateer/data:ro " + f"mrcide/privateer-client:{cfg.tag} " + "rsync -av --delete /privateer/data alice:/privateer/volumes/bob" + ) + assert cmd in lines + + +def test_can_run_backup(monkeypatch, managed_docker): + mock_run = MagicMock() + monkeypatch.setattr( + privateer2.backup, "run_container_with_command", mock_run + ) + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol = managed_docker("volume") + cfg.clients[0].key_volume = vol + keygen_all(cfg) + configure(cfg, "bob") + backup(cfg, "bob", "data") + + image = f"mrcide/privateer-client:{cfg.tag}" + command = [ + "rsync", + "-av", + "--delete", + "/privateer/data", + "alice:/privateer/volumes/bob", + ] + mounts = [ + docker.types.Mount( + "/privateer/keys", vol, type="volume", read_only=True + ), + docker.types.Mount( + "/privateer/data", "data", type="volume", read_only=True + ), + ] + assert mock_run.call_count == 1 + assert mock_run.call_args == call( + "Backup", image, command=command, mounts=mounts + ) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..9959066 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,295 @@ +import shutil +from unittest.mock import MagicMock, call + +import pytest + +import privateer2.cli +from privateer2.cli import ( + Call, + _do_configure, + _find_identity, + _parse_argv, + _parse_opts, + _path_config, + _show_version, + main, + pull, +) +from privateer2.config import read_config +from privateer2.util import transient_working_directory + + +def test_can_create_and_run_call(): + def f(a, b=1): + return [a, b] + + call = Call(f, a=1, b=2) + assert call.run() == [1, 2] + + +def test_can_parse_version(): + res = _parse_argv(["--version"]) + assert res.target == privateer2.cli._show_version + assert res.kwargs == {} + + +def test_can_parse_keygen_all(): + res = _parse_argv(["keygen", "--path=example/simple.json", "--all"]) + assert res.target == privateer2.cli.keygen_all + assert res.kwargs == {"cfg": read_config("example/simple.json")} + + +def test_can_parse_keygen_one(): + res = _parse_argv(["keygen", "--path=example/simple.json", "alice"]) + assert res.target == privateer2.cli.keygen + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + } + + +def test_can_prevent_use_of_as_with_keygen(): + with pytest.raises(Exception, match="Don't use '--as' with 'keygen'"): + _parse_argv(["keygen", "--path=example/local.json", "--as", "x", "y"]) + + +def test_can_parse_configure(): + res = _parse_argv(["configure", "--path=example/simple.json", "alice"]) + assert res.target == privateer2.cli._do_configure + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + "root": "example", + } + + +def test_can_parse_configure_without_explicit_path(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with transient_working_directory(tmp_path): + res = _parse_argv(["configure", "alice"]) + assert res.target == privateer2.cli._do_configure + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + "root": "", + } + + +def test_can_parse_pull(): + res = _parse_argv(["pull", "--path=example/simple.json"]) + assert res.target == privateer2.cli.pull + assert res.kwargs == {"cfg": read_config("example/simple.json")} + + +def test_can_parse_check(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with transient_working_directory(tmp_path): + res = _parse_argv(["check"]) + assert res.target == privateer2.cli.check + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + "connection": False, + } + path = str(tmp_path / "privateer.json") + _parse_argv(["check", "--path", path]) + assert _parse_argv(["check", "--path", path]) == res + assert _parse_argv(["check", "--path", path, "--as", "alice"]) == res + res.kwargs["name"] = "bob" + assert _parse_argv(["check", "--path", path, "--as", "bob"]) == res + res.kwargs["connection"] = True + assert ( + _parse_argv(["check", "--path", path, "--as", "bob", "--connection"]) + == res + ) + + +def test_can_parse_server_start(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with transient_working_directory(tmp_path): + res = _parse_argv(["server", "start"]) + assert res.target == privateer2.cli.server_start + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + "dry_run": False, + } + + +def test_can_parse_server_status(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with transient_working_directory(tmp_path): + res = _parse_argv(["server", "status"]) + assert res.target == privateer2.cli.server_status + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + } + + +def test_can_parse_server_stop(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with transient_working_directory(tmp_path): + res = _parse_argv(["server", "stop"]) + assert res.target == privateer2.cli.server_stop + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + } + + +def test_can_parse_backup(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with transient_working_directory(tmp_path): + res = _parse_argv(["backup", "v"]) + assert res.target == privateer2.cli.backup + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + "volume": "v", + "server": None, + "dry_run": False, + } + + +def test_can_parse_backup_with_server(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with transient_working_directory(tmp_path): + res = _parse_argv(["backup", "v", "--server", "alice"]) + assert res.target == privateer2.cli.backup + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + "volume": "v", + "server": "alice", + "dry_run": False, + } + + +def test_can_parse_restore(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with transient_working_directory(tmp_path): + res = _parse_argv(["restore", "v"]) + assert res.target == privateer2.cli.restore + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + "volume": "v", + "server": None, + "source": None, + "dry_run": False, + } + + +def test_can_parse_complex_restore(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with transient_working_directory(tmp_path): + res = _parse_argv(["restore", "v", "--server=alice", "--source=bob"]) + assert res.target == privateer2.cli.restore + assert res.kwargs == { + "cfg": read_config("example/simple.json"), + "name": "alice", + "volume": "v", + "server": "alice", + "source": "bob", + "dry_run": False, + } + + +def test_error_if_unknown_identity(tmp_path): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + msg = "Can't determine identity; did you forget to configure" + with pytest.raises(Exception, match=msg): + _find_identity(None, tmp_path) + assert _find_identity("alice", tmp_path) == "alice" + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + assert _find_identity(None, tmp_path) == "alice" + assert _find_identity("bob", tmp_path) == "bob" + + +def test_configuration_writes_identity(tmp_path, monkeypatch): + shutil.copy("example/simple.json", tmp_path / "privateer.json") + cfg = read_config("example/simple.json") + mock_configure = MagicMock() + monkeypatch.setattr(privateer2.cli, "configure", mock_configure) + _do_configure(cfg, "alice", str(tmp_path)) + assert tmp_path.joinpath(".privateer_identity").exists() + assert _find_identity(None, str(tmp_path)) == "alice" + mock_configure.assert_called_once_with(cfg, "alice") + + +def test_can_print_version(capsys): + _show_version() + out = capsys.readouterr() + assert out.out == f"privateer {privateer2.cli.about.__version__}\n" + + +def test_options_parsing_else_clause(tmp_path): + class empty: # noqa + def __getitem__(self, name): + return None + + shutil.copy("example/simple.json", tmp_path / "privateer.json") + with open(tmp_path / ".privateer_identity", "w") as f: + f.write("alice\n") + with pytest.raises(Exception, match="Invalid cli call -- privateer bug"): + with transient_working_directory(tmp_path): + _parse_opts(empty()) + + +def test_call_main(monkeypatch): + mock_call = MagicMock() + monkeypatch.setattr(privateer2.cli, "_parse_argv", mock_call) + main(["--version"]) + assert mock_call.call_count == 1 + assert mock_call.call_args == call(["--version"]) + assert mock_call.return_value.run.call_count == 1 + assert mock_call.return_value.run.call_args == call() + + +def test_run_pull(monkeypatch): + cfg = read_config("example/simple.json") + image_client = f"mrcide/privateer-client:{cfg.tag}" + image_server = f"mrcide/privateer-server:{cfg.tag}" + mock_docker = MagicMock() + monkeypatch.setattr(privateer2.cli, "docker", mock_docker) + pull(cfg) + assert mock_docker.from_env.call_count == 1 + client = mock_docker.from_env.return_value + assert client.images.pull.call_count == 2 + assert client.images.pull.call_args_list[0] == call(image_client) + assert client.images.pull.call_args_list[1] == call(image_server) + + +def test_clean_path(tmp_path): + with pytest.raises(Exception, match="Did not find privateer configuration"): + with transient_working_directory(str(tmp_path)): + _path_config(None) + with pytest.raises(Exception, match="Did not find privateer configuration"): + _path_config(tmp_path) + with pytest.raises(Exception, match="Did not find privateer configuration"): + _path_config(tmp_path / "foo.json") + with pytest.raises(Exception, match="Did not find privateer configuration"): + _path_config("foo.json") + shutil.copy("example/simple.json", tmp_path / "privateer.json") + assert _path_config(str(tmp_path)) == str(tmp_path / "privateer.json") + assert _path_config("example/simple.json") == "example/simple.json" + with transient_working_directory(str(tmp_path)): + assert _path_config(None) == "privateer.json" diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..fcf881c --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,145 @@ +import pytest +import vault_dev + +from privateer2.config import _check_config, find_source, read_config + + +def test_can_read_config(): + cfg = read_config("example/simple.json") + assert len(cfg.servers) == 1 + assert cfg.servers[0].name == "alice" + assert cfg.servers[0].hostname == "alice.example.com" + assert cfg.servers[0].port == 10022 + assert len(cfg.clients) == 1 + assert cfg.clients[0].name == "bob" + assert cfg.clients[0].backup == ["data"] + assert cfg.clients[0].restore == ["data"] + assert len(cfg.volumes) == 1 + assert cfg.volumes[0].name == "data" + assert cfg.vault.url == "http://localhost:8200" + assert cfg.vault.prefix == "/privateer" + assert cfg.list_servers() == ["alice"] + assert cfg.list_clients() == ["bob"] + + +def test_can_create_vault_client(): + cfg = read_config("example/simple.json") + with vault_dev.Server(export_token=True) as server: + cfg.vault.url = server.url() + client = cfg.vault.client() + assert client.is_authenticated() + + +# These are annoying to setup, the rest just run the validation manually: +def test_validation_is_run_on_load(tmp_path): + path = tmp_path / "privateer.json" + with path.open("w") as f: + f.write( + """{ + "servers": [ + { + "name": "alice", + "hostname": "alice.example.com", + "port": 10022 + } + ], + "clients": [ + { + "name": "alice", + "backup": ["data"], + "restore": ["data", "other"] + } + ], + "volumes": [ + { + "name": "data" + } + ], + "vault": { + "url": "http://localhost:8200", + "prefix": "/secret/privateer" + } +}""" + ) + msg = "Invalid machine listed as both a client and a server: 'alice'" + with pytest.raises(Exception, match=msg): + read_config(path) + + +def test_machines_cannot_be_duplicated(): + cfg = read_config("example/simple.json") + cfg.clients = cfg.clients + cfg.clients + with pytest.raises(Exception, match="Duplicated elements in clients"): + _check_config(cfg) + cfg.servers = cfg.servers + cfg.servers + with pytest.raises(Exception, match="Duplicated elements in servers"): + _check_config(cfg) + + +def test_machines_cannot_be_client_and_server(): + cfg = read_config("example/simple.json") + tmp = cfg.clients[0].model_copy() + tmp.name = "alice" + cfg.clients.append(tmp) + msg = "Invalid machine listed as both a client and a server: 'alice'" + with pytest.raises(Exception, match=msg): + _check_config(cfg) + + +def test_restore_volumes_are_known(): + cfg = read_config("example/simple.json") + cfg.clients[0].restore.append("other") + msg = "Client 'bob' restores from unknown volume 'other'" + with pytest.raises(Exception, match=msg): + _check_config(cfg) + + +def test_backup_volumes_are_known(): + cfg = read_config("example/simple.json") + cfg.clients[0].backup.append("other") + msg = "Client 'bob' backs up unknown volume 'other'" + with pytest.raises(Exception, match=msg): + _check_config(cfg) + + +def test_local_volumes_cannot_be_backed_up(): + cfg = read_config("example/simple.json") + cfg.volumes[0].local = True + msg = "Client 'bob' backs up local volume 'data'" + with pytest.raises(Exception, match=msg): + _check_config(cfg) + + +def test_can_find_appropriate_source(): + cfg = read_config("example/simple.json") + tmp = cfg.clients[0].model_copy() + tmp.name = "carol" + cfg.clients.append(tmp) + assert find_source(cfg, "data", "bob") == "bob" + assert find_source(cfg, "data", "carol") == "carol" + msg = "Invalid source 'alice': valid options: 'bob', 'carol'" + with pytest.raises(Exception, match=msg): + find_source(cfg, "data", "alice") + + +def test_can_find_appropriate_source_if_local(): + cfg = read_config("example/simple.json") + cfg.volumes[0].local = True + find_source(cfg, "data", None) + msg = "'data' is a local source, so 'source' must be empty" + with pytest.raises(Exception, match=msg): + find_source(cfg, "data", "bob") + with pytest.raises(Exception, match=msg): + find_source(cfg, "data", "local") + + +def test_can_strip_leading_secret_from_path(): + cfg = read_config("example/simple.json") + + cfg.vault.prefix = "/secret/my/path" + _check_config(cfg) + assert cfg.vault.prefix == "/my/path" + + cfg.vault.prefix = "/my/path" + _check_config(cfg) + assert cfg.vault.prefix == "/my/path" diff --git a/tests/test_keys.py b/tests/test_keys.py new file mode 100644 index 0000000..57e1040 --- /dev/null +++ b/tests/test_keys.py @@ -0,0 +1,243 @@ +from unittest.mock import MagicMock, call + +import pytest +import vault_dev + +import docker +import privateer2.keys +from privateer2.config import read_config +from privateer2.keys import ( + _check_connections, + _keys_data, + check, + configure, + keygen, + keygen_all, +) +from privateer2.util import string_from_volume + + +def test_can_create_keys(): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + keygen(cfg, "alice") + client = cfg.vault.client() + response = client.secrets.kv.v1.read_secret("/privateer/alice") + pair = response["data"] + assert set(pair.keys()) == {"private", "public"} + assert pair["public"].startswith("ssh-rsa") + assert "PRIVATE KEY" in pair["private"] + + +def test_can_generate_server_keys_data(): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + keygen_all(cfg) + dat = _keys_data(cfg, "alice") + assert dat["name"] == "alice" + assert dat["known_hosts"] is None + assert dat["authorized_keys"].startswith("ssh-rsa") + + +def test_can_generate_client_keys_data(): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + keygen_all(cfg) + dat = _keys_data(cfg, "bob") + assert dat["name"] == "bob" + assert dat["authorized_keys"] is None + assert dat["known_hosts"].startswith( + "[alice.example.com]:10022 ssh-rsa" + ) + + +def test_can_unpack_keys_for_server(managed_docker): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol = managed_docker("volume") + cfg.servers[0].key_volume = vol + keygen_all(cfg) + configure(cfg, "alice") + client = docker.from_env() + mounts = [docker.types.Mount("/keys", vol, type="volume")] + name = managed_docker("container") + res = client.containers.run( + "alpine", + mounts=mounts, + command=["ls", "/keys"], + name=name, + ) + assert set(res.decode("UTF-8").strip().split("\n")) == { + "authorized_keys", + "id_rsa", + "id_rsa.pub", + "name", + } + assert string_from_volume(vol, "name") == "alice" + + +def test_can_unpack_keys_for_client(managed_docker): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol = managed_docker("volume") + cfg.clients[0].key_volume = vol + keygen_all(cfg) + configure(cfg, "bob") + client = docker.from_env() + mounts = [docker.types.Mount("/keys", vol, type="volume")] + name = managed_docker("container") + res = client.containers.run( + "alpine", + mounts=mounts, + command=["ls", "/keys"], + name=name, + ) + assert set(res.decode("UTF-8").strip().split("\n")) == { + "known_hosts", + "id_rsa", + "id_rsa.pub", + "name", + "config", + } + assert string_from_volume(vol, "name") == "bob" + assert check(cfg, "bob").key_volume == vol + msg = "Configuration is for 'bob', not 'alice'" + cfg.servers[0].key_volume = vol + with pytest.raises(Exception, match=msg): + check(cfg, "alice") + + +def test_can_check_quietly(capsys, managed_docker): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol = managed_docker("volume") + cfg.servers[0].key_volume = vol + keygen_all(cfg) + configure(cfg, "alice") + capsys.readouterr() # flush capture so far + assert check(cfg, "alice", quiet=True).key_volume == vol + assert capsys.readouterr().out == "" + assert check(cfg, "alice", quiet=False).key_volume == vol + out_loud = capsys.readouterr() + assert out_loud.out == f"Volume '{vol}' looks configured as 'alice'\n" + + +def test_error_on_check_if_unconfigured(managed_docker): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol = managed_docker("volume") + cfg.servers[0].key_volume = vol + with pytest.raises(Exception, match="'alice' looks unconfigured"): + check(cfg, "alice") + + +def test_error_on_check_if_unknown_machine(): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + msg = "Invalid configuration 'eve', must be one of 'alice', 'bob'" + with pytest.raises(Exception, match=msg): + check(cfg, "eve") + + +def test_can_check_connections(capsys, monkeypatch, managed_docker): + mock_docker = MagicMock() + monkeypatch.setattr(privateer2.keys, "docker", mock_docker) + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol_keys_bob = managed_docker("volume") + cfg.servers[0].key_volume = managed_docker("volume") + cfg.clients[0].key_volume = vol_keys_bob + keygen_all(cfg) + configure(cfg, "bob") + capsys.readouterr() # flush previous output + _check_connections(cfg, cfg.clients[0]) + + out = capsys.readouterr().out + assert ( + out == "checking connection to 'alice' (alice.example.com)...OK\n" + ) + assert mock_docker.from_env.called + client = mock_docker.from_env.return_value + mount = mock_docker.types.Mount + assert mount.call_count == 1 + assert mount.call_args_list[0] == call( + "/privateer/keys", vol_keys_bob, type="volume", read_only=True + ) + assert client.containers.run.call_count == 1 + assert client.containers.run.call_args == call( + f"mrcide/privateer-client:{cfg.tag}", + mounts=[mount.return_value], + command=["ssh", "alice", "cat", "/privateer/keys/name"], + remove=True, + ) + + +def test_can_report_connection_failure(capsys, monkeypatch, managed_docker): + mock_docker = MagicMock() + mock_docker.errors = docker.errors + err = docker.errors.ContainerError("nm", 1, "ssh", "img", b"the reason") + monkeypatch.setattr(privateer2.keys, "docker", mock_docker) + client = mock_docker.from_env.return_value + client.containers.run.side_effect = err + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol_keys_bob = managed_docker("volume") + cfg.servers[0].key_volume = managed_docker("volume") + cfg.clients[0].key_volume = vol_keys_bob + keygen_all(cfg) + configure(cfg, "bob") + capsys.readouterr() # flush previous output + _check_connections(cfg, cfg.clients[0]) + + out = capsys.readouterr().out + assert out == ( + "checking connection to 'alice' (alice.example.com)...ERROR\n" + "the reason\n" + ) + assert mock_docker.from_env.called + client = mock_docker.from_env.return_value + mount = mock_docker.types.Mount + assert mount.call_count == 1 + assert mount.call_args_list[0] == call( + "/privateer/keys", vol_keys_bob, type="volume", read_only=True + ) + assert client.containers.run.call_count == 1 + assert client.containers.run.call_args == call( + f"mrcide/privateer-client:{cfg.tag}", + mounts=[mount.return_value], + command=["ssh", "alice", "cat", "/privateer/keys/name"], + remove=True, + ) + + +def test_only_test_connection_for_clients(monkeypatch, managed_docker): + mock_check = MagicMock() + monkeypatch.setattr(privateer2.keys, "_check_connections", mock_check) + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + cfg.servers[0].key_volume = managed_docker("volume") + cfg.servers[0].data_volume = managed_docker("volume") + cfg.clients[0].key_volume = managed_docker("volume") + keygen_all(cfg) + configure(cfg, "alice") + configure(cfg, "bob") + check(cfg, "alice") + assert mock_check.call_count == 0 + check(cfg, "bob") + assert mock_check.call_count == 0 + check(cfg, "alice", connection=True) + assert mock_check.call_count == 0 + check(cfg, "bob", connection=True) + assert mock_check.call_count == 1 + assert mock_check.call_args == call(cfg, cfg.clients[0]) diff --git a/tests/test_restore.py b/tests/test_restore.py new file mode 100644 index 0000000..9a09ad6 --- /dev/null +++ b/tests/test_restore.py @@ -0,0 +1,68 @@ +from unittest.mock import MagicMock, call + +import vault_dev + +import docker +import privateer2.restore +from privateer2.config import read_config +from privateer2.keys import configure, keygen_all +from privateer2.restore import restore + + +def test_can_print_instructions_to_run_restore(capsys, managed_docker): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol = managed_docker("volume") + cfg.clients[0].key_volume = vol + keygen_all(cfg) + configure(cfg, "bob") + capsys.readouterr() # flush previous output + restore(cfg, "bob", "data", dry_run=True) + out = capsys.readouterr() + lines = out.out.strip().split("\n") + assert "Command to manually run restore:" in lines + cmd = ( + " docker run --rm " + f"-v {vol}:/privateer/keys:ro -v data:/privateer/data " + f"mrcide/privateer-client:{cfg.tag} " + "rsync -av --delete alice:/privateer/volumes/bob/data/ " + "/privateer/data/" + ) + assert cmd in lines + + +def test_can_run_restore(monkeypatch, managed_docker): + mock_run = MagicMock() + monkeypatch.setattr( + privateer2.restore, "run_container_with_command", mock_run + ) + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol = managed_docker("volume") + cfg.clients[0].key_volume = vol + keygen_all(cfg) + configure(cfg, "bob") + restore(cfg, "bob", "data") + + image = f"mrcide/privateer-client:{cfg.tag}" + command = [ + "rsync", + "-av", + "--delete", + "alice:/privateer/volumes/bob/data/", + "/privateer/data/", + ] + mounts = [ + docker.types.Mount( + "/privateer/keys", vol, type="volume", read_only=True + ), + docker.types.Mount( + "/privateer/data", "data", type="volume", read_only=False + ), + ] + assert mock_run.call_count == 1 + assert mock_run.call_args == call( + "Restore", image, command=command, mounts=mounts + ) diff --git a/tests/test_server.py b/tests/test_server.py new file mode 100644 index 0000000..26ab7ec --- /dev/null +++ b/tests/test_server.py @@ -0,0 +1,148 @@ +from unittest.mock import MagicMock, call + +import vault_dev + +import privateer2.server +from privateer2.config import read_config +from privateer2.keys import configure, keygen_all +from privateer2.server import server_start, server_status, server_stop + + +def test_can_print_instructions_to_start_server(capsys, managed_docker): + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol_keys = managed_docker("volume") + vol_data = managed_docker("volume") + name = managed_docker("container") + cfg.servers[0].key_volume = vol_keys + cfg.servers[0].data_volume = vol_data + cfg.servers[0].container = name + keygen_all(cfg) + configure(cfg, "alice") + capsys.readouterr() # flush previous output + server_start(cfg, "alice", dry_run=True) + out = capsys.readouterr() + lines = out.out.strip().split("\n") + assert "Command to manually launch server:" in lines + cmd = ( + f" docker run --rm -d --name {name} " + f"-v {vol_keys}:/privateer/keys:ro " + f"-v {vol_data}:/privateer/volumes " + f"-p 10022:22 mrcide/privateer-server:{cfg.tag}" + ) + assert cmd in lines + + +def test_can_start_server(monkeypatch, managed_docker): + mock_docker = MagicMock() + mock_start = MagicMock() + monkeypatch.setattr(privateer2.server, "docker", mock_docker) + monkeypatch.setattr(privateer2.server, "service_start", mock_start) + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/simple.json") + cfg.vault.url = server.url() + vol_keys = managed_docker("volume") + vol_data = managed_docker("volume") + name = managed_docker("container") + cfg.servers[0].key_volume = vol_keys + cfg.servers[0].data_volume = vol_data + cfg.servers[0].container = name + keygen_all(cfg) + configure(cfg, "alice") + server_start(cfg, "alice") + mount = mock_docker.types.Mount + assert mount.call_count == 2 + assert mount.call_args_list[0] == call( + "/privateer/keys", vol_keys, type="volume", read_only=True + ) + assert mount.call_args_list[1] == call( + "/privateer/volumes", vol_data, type="volume" + ) + assert mock_start.call_count == 1 + image = f"mrcide/privateer-server:{cfg.tag}" + mounts = [mount.return_value] * 2 + ports = {"22/tcp": 10022} + assert mock_start.call_args == call( + "alice", + name, + image=image, + mounts=mounts, + ports=ports, + dry_run=False, + ) + + +def test_can_start_server_with_local_volume(monkeypatch, managed_docker): + mock_docker = MagicMock() + mock_start = MagicMock() + monkeypatch.setattr(privateer2.server, "docker", mock_docker) + monkeypatch.setattr(privateer2.server, "service_start", mock_start) + with vault_dev.Server(export_token=True) as server: + cfg = read_config("example/local.json") + cfg.vault.url = server.url() + vol_keys = managed_docker("volume") + vol_data = managed_docker("volume") + vol_other = managed_docker("volume") + name = managed_docker("container") + cfg.servers[0].key_volume = vol_keys + cfg.servers[0].data_volume = vol_data + cfg.servers[0].container = name + cfg.volumes[1].name = vol_other + keygen_all(cfg) + configure(cfg, "alice") + server_start(cfg, "alice") + mount = mock_docker.types.Mount + assert mount.call_count == 3 + assert mount.call_args_list[0] == call( + "/privateer/keys", vol_keys, type="volume", read_only=True + ) + assert mount.call_args_list[1] == call( + "/privateer/volumes", vol_data, type="volume" + ) + assert mount.call_args_list[2] == call( + f"/privateer/local/{vol_other}", + vol_other, + type="volume", + read_only=True, + ) + assert mock_start.call_count == 1 + image = f"mrcide/privateer-server:{cfg.tag}" + mounts = [mount.return_value] * 3 + ports = {"22/tcp": 10022} + assert mock_start.call_args == call( + "alice", + name, + image=image, + mounts=mounts, + ports=ports, + dry_run=False, + ) + + +def test_can_stop_server(monkeypatch): + mock_check = MagicMock() + mock_stop = MagicMock() + cfg = MagicMock() + monkeypatch.setattr(privateer2.server, "check", mock_check) + monkeypatch.setattr(privateer2.server, "service_stop", mock_stop) + server_stop(cfg, "alice") + assert mock_check.call_count == 1 + assert mock_check.call_args == call(cfg, "alice", quiet=True) + container = mock_check.return_value.container + assert mock_stop.call_count == 1 + assert mock_stop.call_args == call("alice", container) + + +def test_can_get_server_status(monkeypatch): + mock_check = MagicMock() + mock_status = MagicMock() + cfg = MagicMock() + monkeypatch.setattr(privateer2.server, "check", mock_check) + monkeypatch.setattr(privateer2.server, "service_status", mock_status) + server_status(cfg, "alice") + assert mock_check.call_count == 1 + assert mock_check.call_args == call(cfg, "alice", quiet=False) + container = mock_check.return_value.container + assert mock_status.call_count == 1 + assert mock_status.call_args == call(container) diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..81e3ac9 --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,148 @@ +from unittest.mock import MagicMock, Mock, call + +import pytest + +import docker +import privateer2.service +from privateer2.service import ( + service_command, + service_start, + service_status, + service_stop, +) + + +def test_can_create_command(): + base = ["docker", "run", "--rm", "-d", "--name", "nm"] + assert service_command("img", "nm") == [*base, "img"] + assert service_command("img", "nm", command=["a", "b"]) == [ + *base, + "img", + "a", + "b", + ] + mounts = [docker.types.Mount("/dest", "vol", type="volume", read_only=True)] + assert service_command("img", "nm", mounts=mounts) == [ + *base, + "-v", + "vol:/dest:ro", + "img", + ] + assert service_command("img", "nm", ports={"22/tcp": 10022}) == [ + *base, + "-p", + "10022:22", + "img", + ] + + +def test_can_launch_container(monkeypatch): + mock_docker = MagicMock() + client = mock_docker.from_env.return_value + mock_exists = MagicMock() + mock_exists.return_value = False + mock_ensure_image = MagicMock() + mounts = Mock() + ports = Mock() + command = Mock() + monkeypatch.setattr(privateer2.service, "docker", mock_docker) + monkeypatch.setattr(privateer2.service, "container_exists", mock_exists) + monkeypatch.setattr(privateer2.service, "ensure_image", mock_ensure_image) + service_start( + "alice", "nm", "img", mounts=mounts, ports=ports, command=command + ) + assert mock_exists.call_count == 1 + assert mock_exists.call_args == call("nm") + assert mock_ensure_image.call_count == 1 + assert mock_ensure_image.call_args == call("img") + assert mock_docker.from_env.call_count == 1 + assert client.containers.run.call_count == 1 + assert client.containers.run.call_args == call( + "img", + auto_remove=True, + detach=True, + name="nm", + mounts=mounts, + ports=ports, + command=command, + ) + + +def test_throws_if_container_already_exists(monkeypatch): + mock_exists = MagicMock() + mock_exists.return_value = True + monkeypatch.setattr(privateer2.service, "container_exists", mock_exists) + msg = "Container 'nm' for 'alice' already running" + with pytest.raises(Exception, match=msg): + service_start("alice", "nm", "img") + assert mock_exists.call_count == 1 + mock_exists.assert_called_with("nm") + + +def test_returns_cmd_even_if_container_already_exists(capsys, monkeypatch): + mock_exists = MagicMock() + mock_exists.return_value = True + monkeypatch.setattr(privateer2.service, "container_exists", mock_exists) + service_start("alice", "nm", "img", dry_run=True) + expected = service_command("img", "nm") + out = capsys.readouterr().out + assert " ".join(expected) in out + assert mock_exists.call_count == 0 + + +def test_can_stop_service(monkeypatch, capsys): + mock_container = MagicMock() + mock_container.status = "running" + mock_container_if_exists = MagicMock(return_value=mock_container) + monkeypatch.setattr( + privateer2.service, + "container_if_exists", + mock_container_if_exists, + ) + + service_stop("alice", "nm") + assert mock_container_if_exists.call_count == 1 + assert mock_container_if_exists.call_args == call("nm") + assert mock_container.stop.call_count == 1 + assert mock_container.stop.call_args == call() + assert capsys.readouterr().out == "" + + mock_container.status = "exited" + service_stop("alice", "nm") + assert mock_container_if_exists.call_count == 2 + assert mock_container.stop.call_count == 1 + assert capsys.readouterr().out == "" + + mock_container_if_exists.return_value = None + service_stop("alice", "nm") + assert mock_container_if_exists.call_count == 3 + assert mock_container.stop.call_count == 1 + assert ( + capsys.readouterr().out == "Container 'nm' for 'alice' does not exist\n" + ) + + +def test_can_get_service_status(monkeypatch, capsys): + mock_container = MagicMock() + mock_container.status = "running" + mock_container_if_exists = MagicMock(return_value=mock_container) + monkeypatch.setattr( + privateer2.service, + "container_if_exists", + mock_container_if_exists, + ) + + service_status("nm") + assert mock_container_if_exists.call_count == 1 + assert mock_container_if_exists.call_args == call("nm") + assert capsys.readouterr().out == "running\n" + + mock_container.status = "exited" + service_status("nm") + assert mock_container_if_exists.call_count == 2 + assert capsys.readouterr().out == "exited\n" + + mock_container_if_exists.return_value = None + service_status("nm") + assert mock_container_if_exists.call_count == 3 + assert capsys.readouterr().out == "not running\n" diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..c8e2320 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,189 @@ +import os +import re +import tarfile + +import pytest + +import docker +import privateer2.util + + +def test_create_simple_tar_from_string(): + p = privateer2.util.simple_tar_string("hello", "path") + t = tarfile.open(fileobj=p) + els = t.getmembers() + assert len(els) == 1 + assert els[0].name == "path" + assert els[0].uid == os.geteuid() + assert els[0].gid == os.getegid() + + +def test_create_simple_tar_with_permissions(): + p = privateer2.util.simple_tar_string( + "hello", "path", uid=0, gid=0, mode=0o600 + ) + t = tarfile.open(fileobj=p) + els = t.getmembers() + assert len(els) == 1 + assert els[0].name == "path" + assert els[0].uid == 0 + assert els[0].gid == 0 + assert els[0].mode == 0o600 + + +def test_can_match_values(): + match_value = privateer2.util.match_value + assert match_value(None, "x", "nm") == "x" + assert match_value("x", "x", "nm") == "x" + assert match_value("x", ["x", "y"], "nm") == "x" + with pytest.raises(Exception, match="Please provide a value for nm"): + match_value(None, ["x", "y"], "nm") + msg = "Invalid nm 'z': valid options: 'x', 'y'" + with pytest.raises(Exception, match=msg): + match_value("z", ["x", "y"], "nm") + + +def test_can_format_timestamp(): + assert re.match("^[0-9]{8}-[0-9]{6}", privateer2.util.isotimestamp()) + + +def test_can_pull_image_if_required(): + def image_exists(name): + cl = docker.from_env() + try: + cl.images.get(name) + return True + except docker.errors.ImageNotFound: + return False + + cl = docker.from_env() + if image_exists("hello-world:latest"): + cl.images.get("hello-world:latest").remove() # pragma: no cover + assert not image_exists("hello-world:latest") + privateer2.util.ensure_image("hello-world:latest") + assert image_exists("hello-world:latest") + + +def test_can_tail_logs_from_container(managed_docker): + privateer2.util.ensure_image("alpine") + name = managed_docker("container") + command = ["seq", "1", "10"] + cl = docker.from_env() + cl.containers.run("alpine", name=name, command=command) + assert privateer2.util.log_tail(cl.containers.get(name), 5) == [ + "(ommitting 5 lines of logs)", + "6", + "7", + "8", + "9", + "10", + ] + assert privateer2.util.log_tail(cl.containers.get(name), 100) == [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + ] + + +def test_can_run_long_command(capsys, managed_docker): + name = managed_docker("container") + command = ["seq", "1", "3"] + privateer2.util.run_container_with_command( + "Test", "alpine", name=name, command=command + ) + out = capsys.readouterr().out + lines = out.strip().split("\n") + assert lines[0] == "Test command started. To stream progress, run:" + assert lines[1] == f" docker logs -f {name}" + assert lines[2] == "Test completed successfully! Container logs:" + assert lines[3:] == ["1", "2", "3"] + + +def test_can_run_failing_command(capsys, managed_docker): + name = managed_docker("container") + command = ["false"] + msg = f"Test failed; see {name} logs for details" + with pytest.raises(Exception, match=msg): + privateer2.util.run_container_with_command( + "Test", "alpine", name=name, command=command + ) + out = capsys.readouterr().out + lines = out.strip().split("\n") + assert lines[0] == "Test command started. To stream progress, run:" + assert lines[1] == f" docker logs -f {name}" + assert lines[2] == "An error occured! Container logs:" + + +def test_can_detect_if_volume_exists(managed_docker): + name = managed_docker("volume") + cl = docker.from_env() + cl.volumes.create(name) + assert privateer2.util.volume_exists(name) + cl.volumes.get(name).remove() + assert not privateer2.util.volume_exists(name) + + +def test_can_take_ownership_of_a_file(tmp_path, managed_docker): + cl = docker.from_env() + mounts = [docker.types.Mount("/src", str(tmp_path), type="bind")] + command = ["touch", "/src/newfile"] + name = managed_docker("container") + cl.containers.run("ubuntu", name=name, mounts=mounts, command=command) + path = tmp_path / "newfile" + info = os.stat(path) + assert info.st_uid == 0 + assert info.st_gid == 0 + uid = os.geteuid() + gid = os.getegid() + cmd = privateer2.util.take_ownership( + "newfile", str(tmp_path), command_only=True + ) + expected = [ + "docker", + "run", + *privateer2.util.mounts_str(mounts), + "-w", + "/src", + "alpine", + "chown", + f"{uid}.{gid}", + "newfile", + ] + assert cmd == expected + privateer2.util.take_ownership("newfile", str(tmp_path)) + info = os.stat(path) + assert info.st_uid == uid + assert info.st_gid == gid + + +def test_can_format_ports(): + ports_str = privateer2.util.ports_str + assert ports_str(None) == [] + assert ports_str({"22/tcp": 10022}) == ["-p", "10022:22"] + assert ports_str({"22": 10022}) == ["-p", "10022:22"] + + +def test_can_test_if_container_exists(managed_docker): + name = managed_docker("container") + assert not privateer2.util.container_exists(name) + assert privateer2.util.container_if_exists(name) is None + privateer2.util.ensure_image("alpine") + cl = docker.from_env() + container = cl.containers.create("alpine", name=name) + assert privateer2.util.container_exists(name) + assert privateer2.util.container_if_exists(name) == container + + +def test_can_copy_string_into_volume(managed_docker): + vol = managed_docker("volume") + privateer2.util.string_to_volume("hello", vol, "test") + assert privateer2.util.string_from_volume(vol, "test") == "hello" + privateer2.util.string_to_volume(["hello", "world"], vol, "test") + assert privateer2.util.string_from_volume(vol, "test") == "hello\nworld\n" diff --git a/tests/test_vault.py b/tests/test_vault.py new file mode 100644 index 0000000..5c33fd4 --- /dev/null +++ b/tests/test_vault.py @@ -0,0 +1,39 @@ +from unittest.mock import MagicMock, call + +import privateer2.vault +from privateer2.util import transient_envvar +from privateer2.vault import _get_vault_token, vault_client + + +def test_pass_back_given_token(): + assert _get_vault_token("a") == "a" + + +def test_lookup_token_from_env_if_given(): + with transient_envvar(MY_TOKEN="token"): + assert _get_vault_token("$MY_TOKEN") == "token" + + +def test_fallback_on_known_good_tokens(): + with transient_envvar(VAULT_TOKEN="vt", VAULT_AUTH_GITHUB_TOKEN="gt"): + assert _get_vault_token(None) == "vt" + with transient_envvar(VAULT_TOKEN=None, VAULT_AUTH_GITHUB_TOKEN="gt"): + assert _get_vault_token(None) == "gt" + + +def test_prompt_if_no_tokens_found(monkeypatch): + monkeypatch.setattr("builtins.input", lambda _: "foo") + with transient_envvar(VAULT_TOKEN=None, VAULT_AUTH_GITHUB_TOKEN=None): + assert _get_vault_token(None) == "foo" + + +def test_can_use_github_auth(monkeypatch): + token = f"ghp_{'x' * 36}" + mock_client = MagicMock() + monkeypatch.setattr(privateer2.vault.hvac, "Client", mock_client) + client = vault_client("https://vault.example.com:8200", token) + assert mock_client.call_count == 1 + assert mock_client.call_args == call("https://vault.example.com:8200") + assert client == mock_client.return_value + assert client.auth.github.login.call_count == 1 + assert client.auth.github.login.call_args == call(token)