Merge pull request #4 from reside-ic/reside-355

example/complex.json

@@ -0,0 +1,37 @@
+{
+    "servers": [
+        {
+            "name": "alice",
+            "hostname": "alice.example.com",
+            "port": 10022
+        },
+        {
+            "name": "carol",
+            "hostname": "alice.example.com",
+            "port": 10022
+        }
+    ],
+    "clients": [
+        {
+            "name": "bob",
+            "backup": ["data"]
+        },
+        {
+            "name": "dan",
+            "backup": []
+        }
+    ],
+    "volumes": [
+        {
+            "name": "data"
+        },
+        {
+            "name": "other",
+            "local": true
+        }
+    ],
+    "vault": {
+        "url": "http://localhost:8200",
+        "prefix": "/secret/privateer"
+    }
+}

example/local.json

@@ -12,7 +12,6 @@
         {
             "name": "bob",
             "backup": ["data"],
-            "restore": ["data", "other"],
             "key_volume": "privateer_keys_bob"
         }
     ],

example/montagu.json

@@ -14,21 +14,17 @@
     "clients": [
         {
             "name": "production",
-            "backup": ["montagu_orderly_volume"],
-            "restore": ["montagu_orderly_volume", "barman_recover"]
+            "backup": ["montagu_orderly_volume"]
         },
         {
             "name": "production2",
-            "backup": ["montagu_orderly_volume"],
-            "restore": ["montagu_orderly_volume", "barman_recover"]
+            "backup": ["montagu_orderly_volume"]
         },
         {
-            "name": "science",
-            "restore": ["montagu_orderly_volume", "barman_recover"]
+            "name": "science"
         },
         {
-            "name": "uat",
-            "restore": ["montagu_orderly_volume", "barman_recover"]
+            "name": "uat"
         }
     ],
     "volumes": [

example/simple.json

@@ -9,8 +9,7 @@
     "clients": [
         {
             "name": "bob",
-            "backup": ["data"],
-            "restore": ["data"]
+            "backup": ["data"]
         }
     ],
     "volumes": [

src/privateer2/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2023-present Rich FitzJohn <[email protected]>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.2"
+__version__ = "0.0.3"

src/privateer2/config.py

@@ -24,7 +24,6 @@ class Server(BaseModel):
 class Client(BaseModel):
     name: str
     backup: List[str] = []
-    restore: List[str] = []
     key_volume: str = "privateer_keys"
 
 
@@ -57,6 +56,9 @@ def list_servers(self):
     def list_clients(self):
         return [x.name for x in self.clients]
 
+    def list_volumes(self):
+        return [x.name for x in self.volumes]
+
     def machine_config(self, name):
         for el in self.servers + self.clients:
             if el.name == name:
@@ -75,7 +77,7 @@ def find_source(cfg, volume, source):
             if source is not None:
                 msg = f"'{volume}' is a local source, so 'source' must be empty"
                 raise Exception(msg)
-            return "local"
+            return None
     pos = [cl.name for cl in cfg.clients if volume in cl.backup]
     return match_value(source, pos, "source")
 
@@ -93,10 +95,6 @@ def _check_config(cfg):
     vols_local = [x.name for x in cfg.volumes if x.local]
     vols_all = [x.name for x in cfg.volumes]
     for cl in cfg.clients:
-        for v in cl.restore:
-            if v not in vols_all:
-                msg = f"Client '{cl.name}' restores from unknown volume '{v}'"
-                raise Exception(msg)
         for v in cl.backup:
             if v not in vols_all:
                 msg = f"Client '{cl.name}' backs up unknown volume '{v}'"

src/privateer2/restore.py

@@ -7,7 +7,7 @@
 def restore(cfg, name, volume, *, server=None, source=None, dry_run=False):
     machine = check(cfg, name, quiet=True)
     server = match_value(server, cfg.list_servers(), "server")
-    volume = match_value(volume, machine.backup, "volume")
+    volume = match_value(volume, cfg.list_volumes(), "volume")
     source = find_source(cfg, volume, source)
     image = f"mrcide/privateer-client:{cfg.tag}"
     dest_mount = f"/privateer/{volume}"
@@ -17,13 +17,12 @@ def restore(cfg, name, volume, *, server=None, source=None, dry_run=False):
         ),
         docker.types.Mount(dest_mount, volume, type="volume", read_only=False),
     ]
-    command = [
-        "rsync",
-        "-av",
-        "--delete",
-        f"{server}:/privateer/volumes/{name}/{volume}/",
-        f"{dest_mount}/",
-    ]
+    if source:
+        src = f"{server}:/privateer/volumes/{source}/{volume}/"
+    else:
+        src = f"{server}:/privateer/local/{volume}/"
+        source = "(source)"  # just for printing now
+    command = ["rsync", "-av", "--delete", src, f"{dest_mount}/"]
     if dry_run:
         cmd = ["docker", "run", "--rm", *mounts_str(mounts), image, *command]
         print("Command to manually run restore:")
@@ -37,7 +36,8 @@ def restore(cfg, name, volume, *, server=None, source=None, dry_run=False):
         print("contained within (config), along with our identity (id_rsa)")
         print("in the directory /privateer/keys")
     else:
-        print(f"Restoring '{volume}' from '{server}'")
+        print(f"Restoring '{volume}' from '{server}'; data originally")
+        print(f"from '{source}'")
         run_container_with_command(
             "Restore", image, command=command, mounts=mounts
         )

src/privateer2/service.py

@@ -49,7 +49,7 @@ def service_start(
         raise Exception(msg)
 
     ensure_image(image)
-    print("Starting server '{name}' as container '{container_name}'")
+    print(f"Starting server '{name}' as container '{container_name}'")
     client = docker.from_env()
     client.containers.run(
         image,

src/privateer2/vault.py

@@ -25,7 +25,8 @@ def _get_vault_token(token):
     for token_type in check:
         if token_type in os.environ:
             return os.environ[token_type]
-    return input("Enter token for vault: ").strip()
+    prompt = "Enter GitHub or Vault token to log into the vault:\n> "
+    return input(prompt).strip()
 
 
 def _is_github_token(token):

tests/test_config.py

@@ -13,7 +13,6 @@ def test_can_read_config():
     assert len(cfg.clients) == 1
     assert cfg.clients[0].name == "bob"
     assert cfg.clients[0].backup == ["data"]
-    assert cfg.clients[0].restore == ["data"]
     assert len(cfg.volumes) == 1
     assert cfg.volumes[0].name == "data"
     assert cfg.vault.url == "http://localhost:8200"
@@ -86,14 +85,6 @@ def test_machines_cannot_be_client_and_server():
         _check_config(cfg)
 
 
-def test_restore_volumes_are_known():
-    cfg = read_config("example/simple.json")
-    cfg.clients[0].restore.append("other")
-    msg = "Client 'bob' restores from unknown volume 'other'"
-    with pytest.raises(Exception, match=msg):
-        _check_config(cfg)
-
-
 def test_backup_volumes_are_known():
     cfg = read_config("example/simple.json")
     cfg.clients[0].backup.append("other")

tests/test_restore.py

@@ -3,6 +3,7 @@
 import vault_dev
 
 import docker
+import privateer2.config
 import privateer2.restore
 from privateer2.config import read_config
 from privateer2.configure import configure
@@ -67,3 +68,68 @@ def test_can_run_restore(monkeypatch, managed_docker):
         assert mock_run.call_args == call(
             "Restore", image, command=command, mounts=mounts
         )
+
+
+def test_restore_from_local_volume(capsys, managed_docker):
+    with vault_dev.Server(export_token=True) as server:
+        cfg = read_config("example/local.json")
+        cfg.vault.url = server.url()
+        vol = managed_docker("volume")
+        cfg.clients[0].key_volume = vol
+        keygen_all(cfg)
+        configure(cfg, "bob")
+        capsys.readouterr()  # flush previous output
+        restore(cfg, "bob", "other", dry_run=True)
+        out = capsys.readouterr()
+        lines = out.out.strip().split("\n")
+        assert "Command to manually run restore:" in lines
+        cmd = (
+            "  docker run --rm "
+            f"-v {vol}:/privateer/keys:ro -v other:/privateer/other "
+            f"mrcide/privateer-client:{cfg.tag} "
+            "rsync -av --delete alice:/privateer/local/other/ "
+            "/privateer/other/"
+        )
+        assert cmd in lines
+
+
+def test_restore_from_alternative_source(capsys, managed_docker):
+    with vault_dev.Server(export_token=True) as server:
+        cfg = read_config("example/complex.json")
+        cfg.vault.url = server.url()
+        vol_bob = managed_docker("volume")
+        vol_dan = managed_docker("volume")
+        cfg.clients[0].key_volume = vol_bob
+        cfg.clients[1].key_volume = vol_dan
+        keygen_all(cfg)
+        configure(cfg, "bob")
+        configure(cfg, "dan")
+        capsys.readouterr()  # flush previous output
+
+        # Data from carol, put there by bob, coming down to dan
+        restore(cfg, "dan", "data", source="bob", server="carol", dry_run=True)
+        out = capsys.readouterr()
+        lines = out.out.strip().split("\n")
+        assert "Command to manually run restore:" in lines
+        cmd = (
+            "  docker run --rm "
+            f"-v {vol_dan}:/privateer/keys:ro -v data:/privateer/data "
+            f"mrcide/privateer-client:{cfg.tag} "
+            "rsync -av --delete carol:/privateer/volumes/bob/data/ "
+            "/privateer/data/"
+        )
+        assert cmd in lines
+
+        # Data from carol, local volume, coming down to dan
+        restore(cfg, "dan", "other", source=None, server="carol", dry_run=True)
+        out = capsys.readouterr()
+        lines = out.out.strip().split("\n")
+        assert "Command to manually run restore:" in lines
+        cmd = (
+            "  docker run --rm "
+            f"-v {vol_dan}:/privateer/keys:ro -v other:/privateer/other "
+            f"mrcide/privateer-client:{cfg.tag} "
+            "rsync -av --delete carol:/privateer/local/other/ "
+            "/privateer/other/"
+        )
+        assert cmd in lines