first public version

README.md

@@ -1 +1,86 @@
+<img src="./icon.png" height="96">
+
 # jmap-backup
+
+This is a Python program to back up messages from your Fastmail JMAP mailbox.
+
+Based on the amazing [work by Nathan Grigg][1] 🙏
+
+## Prerequisites
+
+- a Fastmail API key (get from https://app.fastmail.com/settings/security/tokens)
+- Python 3 (`brew install python3` if you're on macOS)
+- Python's `requests` and `pyyaml` modules
+
+To get the required modules, either install them in a virtualenv, or globally with:
+
+```shell
+PIP_REQUIRE_VIRTUALENV=false python3 -m pip install --break-system-packages requests pyyaml
+```
+
+## Setup
+
+1. Create a configuration file (YAML) to store your API key, destination directory where the backup will be kept, and other settings. You can create multiple config files to back up different accounts or to keep copies on different storage (local, SMB/NFS etc).
+
+> If you don't specify a config file with the `-c` option, the program will assume a default path of `~/.jmapbackup/fastmail.yml`.
+
+A bare minimum config file should look something like:
+
+```yaml
+dest_dir: /Volumes/storage/backups/Fastmail
+token: {your_api_key_here e.g. fmu1-xxxxxx...}
+```
+
+### Other optional parameters for the config file
+
+- `delay_hours` - back up only messages at least this many hours old
+- `not_before`  - cut off time before which messages will not be backed up
+- `pre_cmd`     - command (and args) to run prior to execution, most often used to mount some remote storage location such as an SMB or NFS share. It is formatted as an array so you can provide additional args as needed.
+- `post_cmd`    - command to run post-execution (e.g. unmount the share)
+
+Example of pre/post commands in config file (`~` will be expanded by Python):
+
+```yml
+pre_cmd: [ '/sbin/mount', '-t', 'smbfs', '//luckman212:hunter2@nas/backups', '/mnt/jmap' ]
+post_cmd: [ '/sbin/umount', '-t', 'smbfs', '/mnt/jmap' ]
+```
+
+When Python saves the configuration, the commands will be "unwrapped" and rewritten in this format (which is also valid)
+
+```yml
+pre_cmd:
+- /sbin/mount
+- -t
+- smbfs
+- //luckman212:hunter2@nas/backups
+- /mnt/jmap
+```
+
+### Environment Variables
+
+- You can export `JMAP_DEBUG` to `1` to see additional debugging info printed to the console
+- You can export `NOT_BEFORE` to override the default of `2000-01-01` or whatever date is specified in the config file
+
+2. Save the `jmap-backup.py` script somewhere in your `$PATH` and ensure it's executable (`chmod +x jmap-backup.py`)
+
+## Run
+
+```shell
+jmap-backup.py -c ~/.jmapbackup/fastmail.yml
+```
+
+Progress messages will be printed to the console. When the job is finished, you should see your messages in the destination directory, organized in folders in `YYYY-MM` format. The individual messages are saved as standard `.eml` format files with the filename made up of a datestamp, messageid and subject.
+
+This is designed to run quickly and often, so running it daily is no problem and should complete within a minute or two. It's a good idea to stick it in your crontab or set up a LaunchAgent to trigger it at regular intervals. I suggest [LaunchControl][3] (no affiliation) if you're on a Mac and don't want to fiddle about with XML files.
+
+## Verification
+
+Every so often, it's a good idea to run the script with the `--verify` argument. This will be slower, but will thoroughly check that every message in your mailbox exists on the filesystem, and will "fill in the blanks" if any are missing.
+
+## Good luck
+
+I've been using this script for a few months with good success, but it has been tested on exactly _one_ system! So you may encounter issues. If you do, please [report them][2].
+
+[1]: https://nathangrigg.com/2021/08/fastmail-backup
+[2]: https://github.com/luckman212/jmap-backup/issues
+[3]: https://www.soma-zone.com/LaunchControl/

jmap-backup.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 """
-Back up messages from a JMAP mailbox (e.g. Fastmail)
+Back up a Fastmail JMAP mailbox in .eml format
 
 https://nathangrigg.com/2021/08/fastmail-backup/
 https://www.fastmail.com/for-developers/integrating-with-fastmail/
@@ -14,28 +14,34 @@
 import collections
 import datetime
 import os
-import requests
 import string
 import sys
 import subprocess
+import importlib
 
-try:
-    import yaml
-except ImportError:
-    print(f"yaml module could not be loaded: try `pip install pyyaml`")
-    exit(1)
+def import_module_globally(module_name):
+    module = importlib.import_module(module_name)
+    globals()[module_name] = module
+
+def str_to_bool(s):
+    return s and s.lower() in [ 'true', '1', 'yes', 'on' ]
+
+# prereqs
+for module in ['requests', 'yaml']:
+    try:
+        m = importlib.import_module(module)
+        globals()[module] = m
+    except ImportError:
+        print(f"{module} module could not be loaded, check README for installation requirements")
+        exit(1)
 
 Session         = collections.namedtuple('Session', 'headers account_id api_url download_template')
 Email           = collections.namedtuple('Email', 'id blob_id date subject')
-DEBUG           = os.getenv('JMAP_DEBUG', False)
+DEBUG           = str_to_bool(os.getenv('JMAP_DEBUG'))
 NOT_BEFORE      = os.getenv('NOT_BEFORE', '2000-01-01')
+DEFAULT_CONFIG  = '~/.jmapbackup/fastmail.yml'
 CONNECT_TIMEOUT = 3
 READ_TIMEOUT    = 20
-UNMOUNT_ON_EXIT = False
-MOUNT_COMMANDS  = {
-    'mount':   [ os.path.expanduser('~/Sync/Scripts/smbmount.sh'), '--unattended', '--mount', 'nas/unattended' ],
-    'unmount': [ os.path.expanduser('~/Sync/Scripts/smbmount.sh'), '--unmount', 'nas/unattended' ]
-}
 
 def dbg(*args, newline=True):
     if not DEBUG:
@@ -115,7 +121,6 @@ def query(session, start, end):
             date = datetime.datetime.fromisoformat(item['receivedAt'].rstrip('Z'))
             yield Email(item['id'], item['blobId'], date, item['subject'])
 
-        # Set anchor to get the next set of emails.
         query_request = json_request['methodCalls'][0][1]
         query_request['anchor'] = response[0]['ids'][-1]
         query_request['anchorOffset'] = 1
@@ -129,6 +134,14 @@ def email_filename(email):
     filename = f'{date}_{email.id}_{subject.strip()}.eml'
     return directory, filename
 
+def run_if(cmd):
+    if cmd:
+        if os.path.exists(cmd[0]):
+            dbg(f'executing: {cmd}')
+            subprocess.run(cmd)
+        else:
+            print(f'invalid command: {cmd}', file=sys.stderr)
+
 def check_dest_dir(dest_dir, retry=True):
     dir_exists = os.path.exists(dest_dir)
     if retry or dir_exists:
@@ -164,32 +177,36 @@ def download_email(session, email, base_dir):
     return True
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Back up a JMAP mailbox in .eml format', add_help=False)
+    parser = argparse.ArgumentParser(description='Back up a Fastmail JMAP mailbox in .eml format', add_help=False)
     parser.add_argument('-h','--help', action='store_true', help=argparse.SUPPRESS)
     parser.add_argument('-v','--verify', action='store_true', help='Fully verify backed up emails and redownload if missing')
     parser.add_argument('-o','--open', action='store_true', help='Open the configured dest_dir in Finder')
-    parser.add_argument('-c','--config', help='Path to config file', nargs=1)
+    parser.add_argument('-c','--config', help=f'Path to config file (default: {DEFAULT_CONFIG})', nargs=1)
     args = parser.parse_args()
     if args.help:
         parser.print_help()
         sys.exit(0)
-    if not args.config:
-        sys.exit(f'you must specify a config file (run with -h for help)')
 
-    cfg_file = os.path.expanduser(args.config[0])
+    if args.config:
+        cfg_file = os.path.expanduser(args.config[0])
+    else:
+        cfg_file = os.path.expanduser(DEFAULT_CONFIG)
     if not os.path.exists(cfg_file):
         sys.exit(f"Error: configuration file '{cfg_file}' does not exist")
     with open(cfg_file, 'r') as fh:
         config = yaml.safe_load(fh)
 
+    # parse pre- and post-commands
+    PRE_COMMAND = [os.path.expanduser(c) for c in config.get('pre_cmd', [])]
+    POST_COMMAND = [os.path.expanduser(c) for c in config.get('post_cmd', [])]
+    run_if(PRE_COMMAND)
+
     dest_dir = config['dest_dir']
-    if not check_dest_dir(dest_dir, True):
-        subprocess.run(MOUNT_COMMANDS['mount'])
-        check_dest_dir(dest_dir, False)
-        UNMOUNT_ON_EXIT = True
+    check_dest_dir(dest_dir, False)
 
     if args.open:
         subprocess.run(['open', dest_dir])
+        #subprocess.run(POST_COMMAND)
         sys.exit(0)
 
     #calculate date window
@@ -206,12 +223,12 @@ def download_email(session, email, base_dir):
     not_before = datetime.datetime.strptime(not_before_str, '%Y-%m-%d').replace(tzinfo=datetime.timezone.utc)
 
     if args.verify:
+        dbg('Verification enabled (this will take longer)')
         start_window = not_before
         last_verify_count = config.get('last_verify_count', None)
     else:
         start_window = config.get('last_end_time', not_before)
 
-    # Start backup
     num_results = 0
     num_verified = 0
     failed_downloads = []
@@ -242,7 +259,7 @@ def download_email(session, email, base_dir):
 
     dbg('Done!')
 
-    # Retry failed downloads
+    # retry failed downloads
     if failed_downloads:
         dbg(f'Retrying {len(failed_downloads)} failed downloads')
         for email in failed_downloads:
@@ -257,11 +274,9 @@ def download_email(session, email, base_dir):
         print(f'Verified: {num_verified}')
     print(f'Archived: {num_results}')
 
-    # Write config
     config['last_end_time'] = end_window
     if num_verified > 0:
         config['last_verify_count'] = num_verified
     with open(cfg_file, 'w') as fh:
-        yaml.dump(config, fh)
-    if UNMOUNT_ON_EXIT:
-        subprocess.run(MOUNT_COMMANDS['unmount'])
+        yaml.safe_dump(config, fh, indent=4, default_flow_style=False)
+    run_if(POST_COMMAND)