-
Notifications
You must be signed in to change notification settings - Fork 0
/
imapfix.py
executable file
·2514 lines (2345 loc) · 131 KB
/
imapfix.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python2
# (Requires Python 2.x, not 3; search for "3.3+" in
# comment below to see how awkward forward-port would be)
"ImapFix v1.894 (c) 2013-24 Silas S. Brown. License: Apache 2"
# Put your configuration into imapfix_config.py,
# overriding these options:
hostname = "imap4-ssl.example.org" # or host:port
username = "me"
password = "xxxxxxxx"
# (If you are likely to be editing configuration rules when
# others can see your screen, you might like to store the
# password in a separate file and do, for example,
# password = open(".imapfix-pass").read().strip()
# instead, so that it won't be shown on-screen when you're
# editing other things.)
# You can also use OAuth2 by setting password like this:
# password = ("command to generate oauth2 string", 3500)
# where 3500 is the number of seconds before script is called again.
# (You'll then have to sort out calls to oauth2.py or whatever)
login_retry = False # True = don't stop on login failure
# (useful if your network connection is not always on)
filtered_inbox = "in" # or =None if you don't want to do
# any moving from inbox to filtered_inbox (i.e. no rules)
# but just use maildirs_to_imap or maildir_to_copyself,
# e.g. because you're on an auxiliary machine (which does
# not have your spamprobe database etc) but still want to
# use maildir_to_copyself for mutt (see notes below).
# You can also use environment variables with override
# e.g. import os; filtered_inbox = os.getenv("ImapFolder","in")
# so it can be overridden with ImapFolder for --upload
# If you want to keep your filtered_inbox on local maildir
# instead of IMAP server, set it to a tuple with the first
# item being "maildir": ("maildir","path/to/maildir")
leave_note_in_inbox = True # for "new mail" indicators
change_message_id = False # set this to True for
# providers like Gmail that refuse to accept new
# versions of messages with the same Message-ID. Adds
# an extra digit to Message-ID (if present) when saving
# a changed version of the message to the same imap.
# This may result in a breakage of "In-Reply-To" (unless
# you edit the extra character out on the client side),
# but might be a necessary loss if you're using Gmail.
newmail_directory = None
# or newmail_directory = "/path/to/a/directory"
# - any new mail put into any folder by header_rules will
# result in a file being created in that local directory
# with the same name as the folder
max_size_of_first_part = None
# e.g. max_size_of_first_part = 48*1024
# any messages whose first part is longer than this will be
# converted into attachments. This is for sync'ing in
# bandwidth-limited situations using a device that knows to
# not fetch attachments but doesn't necessarily know to not
# fetch more than a certain amount of text (especially if
# it's only HTML).
image_size = None # or e.g. image_size = (320,240)
# - adds scaled-down versions of any image attachment that
# exceeds this size, for previewing on low bandwidth
# (this option requires the PIL library). The scaled-down
# versions are added only if the file is actually smaller.
# This option also tries to ensure that all images are set to
# an image/ rather than application/ Content-Type, which
# helps some mailers.
office_convert = None # or "html" or "pdf", to use
# LibreOffice/OpenOffice's 'soffice --convert-to' option
# to produce converted versions of office documents
# (beware, I have not checked soffice for vulnerabilities)
# - resulting HTML might not work on old WM phones as they
# don't support data: image URLs.
# This option also enables summary generation of
# Microsoft Calendar attachments.
pdf_convert = False # True = use "pdftohtml" on pdf files
use_tnef = False # True = use "tnef" command on winmail.dat
headers_to_delete = [] # prefixes of headers to delete from all
# messages, in case you use Mutt's "Edit Message" on notes to self
# e.g. headers_to_delete = ["X-MS","X-Microsoft"]
# (note however that extra_rules and spam detection runs first)
header_rules = [
("folder-name-1",
["regexp to check for in header",
"regexp to check for in header",
"regexp to check for in header"]),
("folder-name-2",
["regexp to check for in header"]),
# etc; folder name None = delete the message;
# "inbox" = change to filtered_inbox;
# "spam" = change to spam_folder;
# start with a * if this folder does not need any
# notification in newmail_directory, e.g. "*boxname"
# For saving to local maildir instead of IMAP, set
# folder name to ("maildir","path/to/maildir")
# or to skip newmail_directory notification also,
# set it to ("*","maildir","path/to/maildir")
]
def extra_rules(message_as_string): return False
# you can override this to any function you want, which
# returns the name of a folder (with or without a *), or
# None to delete the message, or False = no decision,
# or ("maildir","path/to/maildir") or ("*","maildir"...)
catch_extraRules_errors = True
def handle_authenticated_message(subject,firstPart,attach):
return False
# - you can override this to any function you want, which
# does anything you want in response to messages that are
# SSL-authenticated as coming from yourself (see below).
# Returns the name of a folder (or maildir as above), or
# None to delete the message, or False = undecided (normal
# rules will apply, except spamprobe will be bypassed).
# If folder name starts with *, mail will be marked 'seen'.
# firstPart is a UTF-8 copy of the first part of the body
# (which will typically contain plain text even if you
# were using an HTML mailer); subject is also UTF-8 coded.
# attach is a dictionary of filename:contents.
# Note: although we do check for SSL authentication, you
# are advised not to give too much "power" to authenticated
# messages (e.g. don't let them run arbitrary shell commands)
# in case your configuration does end up with a loophole
# or a trusted IMAP server gets compromised. The expected
# use of handle_authenticated_message is to save it in a
# folder it specifies, or add something to a database, or
# trigger a preconfigured build script, etc.
trusted_domain = None # or e.g. ".example.org" specifying
# the domain of "our" network whose Received headers we
# can trust for SMTPS authentication (below). You may
# also set it to a list of strings, e.g.
# trusted_domain=["example.net","example.org"]
# in which case you might want to add IP addresses to the
# list if one of your machines fails to do reverse-DNS to
# the other one on its Received header (but domain names
# must also be provided on this list).
# Including a blank ("") entry causes "Received" headers
# that do not sperify a "from" to not interrupt the scan
# for trusted Received headers (some IMAP networks have
# one of those first, but use with caution).
smtps_auth = None # or e.g. " with esmtpsa (LOGIN:me)"
# - if a Received header generated by your trusted domain
# (and listed before any untrusted headers) contains this
# string, the message is considered authentically from you
# (this string should be generated by your network only if
# you really did authenticate over HTTPS). You may also
# set smtps_auth to a list of strings, any one of which is
# acceptable, e.g. smtps_auth=["esmtpsa (LOGIN:me)","esmtpsa (PLAIN:me)"]
# If any item starts "from " then the next word is assumed to
# be a user ID that is trusted if the domain reports it as a
# local delivery (no IP address).
# Note: if trusted_domain and smtps_auth is set, any message
# that does NOT contain any Received headers will be assumed
# to be generated from your own account and therefore treated
# as though it had been authenticated by smtps_auth (which
# might be useful for using --multinote with the real inbox).
# You should therefore ensure that your local network ALWAYS
# adds at least one Received header to incoming mail.
debug_trusted_domain=False # if setting this to True, change
# quiet to not be True
super_trusted_domain = None # like trusted_domain but assumes
# messages delivered entirely within this domain (or list)
# will not be allowed to have a fake From: smtp_fromHeader
# (below), which can then be used to authenticate a message.
# This is useful for services that don't add anything that
# can be picked up from smtps_auth, but do block forged
# From addresses on local messages. Use carefully!
# If handle_authenticated_message needs to send other mail
# via SMTP, it can call
# send_mail(to,subject_u8,txt,attachment_filenames=[],copyself=True)
# if the following SMTP options are set:
smtp_fromHeader = "Example Name <[email protected]>"
smtp_fromAddr = "[email protected]"
smtp_host = "localhost"
smtp_user = ""
smtp_password = "" # or e.g. ("oauth2 cmd",3600)
smtp_delay = 60 # seconds between each message
smtp_fcc_Copyself = True
# (These smtp_ settings are not currently used by anything
# except user-supplied handle_authenticated_message functions)
rewrite_return_path_SRS=True # to undo the Sender Rewriting Scheme
# in the Return-Path: you might want this if you rely on
# Return-Path in extra rules and you want to be able to port
# them to be behind a different forwarder
add_return_path=True # if missing, before applying rules
spamprobe_command = "spamprobe -H all" # (or = None)
spam_folder = "spam"
# you can also set this to a local maildir: ("maildir","path/to/spam")
spamprobe_remove_images = True # work around a bug in some
# versions of spamprobe that causes them to crash when an
# image is present in the email to be tested
poll_interval = 4*60
# Note: set poll_interval=False if you want just one run,
# or poll_interval="idle" to use imap's IDLE command (this
# requires the imaplib2 module; you can add to sys.path from
# imapfix_config.py if you have it in your home directory).
# imaplib2 writes debug information to the console unless
# run with python -O; to suppress this, do
# sed -e s/__debug__/False/g < imaplib2.py > i && mv i imaplib2.py
# Some imap servers (including possibly Outlook) sometimes
# fail to send new messages to the IDLE command (not always)
# - if that happens, it'll probably be a 29-minute poll interval
logout_before_sleep = False # suggest set to True if using
# a long poll_interval (not "idle"), or if
# filtered_inbox=None, or if the server can't take more
# than one connection at a time
midnight_command = None
# or, midnight_command = "system command to run at midnight"
# (useful if you don't have crontab access on the machine)
calendar_file = None # set to run 'calendar' command on it
# and send output lines as messages
postponed_foldercheck = False
# if True, check for folders named YYYY-MM-DD according to
# the current date, and, if any are found, move their mail
# into filtered_inbox, updating Date lines (but inserting
# the original date into the message body in case it's
# needed for attribution in quoted replies, unless the
# message is from --note or --multinote below). This is so
# you can postpone a message for handling later, simply by
# saving it into a folder named according to the date you
# want to see it, in YYYY-MM-DD format. The check is made
# after midnight for the new date, and on startup for all
# previous dates (in case imapfix hadn't been run every day).
# Additionally, messages that are SSL-authenticated as coming
# from yourself, and whose subject lines start with a date in
# YYYY-MM-DD format, will be saved into a folder of that name
# (with the date removed from the subject line to save screen space).
postponed_daynames = False
# If True, similar to postponed_foldercheck above, but checks
# for lower-case abbreviated month and weekday names
# (e.g. mon, tue, jan, feb - the current locale is used, so
# if you want non-English names then set a different locale).
# This check is performed after midnight, but not on startup
# because they're not absolute dates. To avoid
# false positives, any use of these at the start of
# self-written Subject lines must be followed by : if
# anything else is on the line, e.g. mon: things to do today.
# You can set postponed_foldercheck and postponed_daynames in
# any combination.
postponed_maildir = None # or "path/to/maildir", will
# result in the above postponed_ options checking
# subfolders of this maildir as well as the IMAP server,
# and authenticated messages for postponing being written
# to subfolders of this maildir instead of the server
quiet = True # False = print messages (including quota)
# If you set quiet = 2, will be quiet if and only if the
# standand output is not connected to a terminal
# If you set quiet = None, will print to standard error
maildirs_to_imap = None # or path to a local directory of
# maildirs; messages will be moved to their corresponding
# folders on IMAP (renaming inbox to filtered_inbox
# and spam to spam_folder, and converting character sets)
maildir_colon = ':'
maildir_delete_emacs_backup_files = True # in case need to
# edit any (e.g. postponed notes) directly in maildir
maildir_to_process = None # or path to a local directory
# which is assumed to be a maildir where unfiltered messages
# are left, to be treated in same way as IMAP inbox
# (use this for example if an imapfix on another machine
# leaves notes there and you want them to be further
# processed on this machine)
maildir_dedot = None # or path to a local directory of
# maildirs: any non-symlink Maildir++ "dot" folders
# in it will be moved to non-"dot" folders. Use if
# sharing e.g. local mutt with Dovecot
imap_to_maildirs = None # or path to a local directory of
# maildirs; messages will be moved from IMAP folders to
# subfolders of these maildirs, converting character sets,
# except for filtered_inbox and spam_folder if these are
# on IMAP. If you're using the 'postponed' options, you
# should set postponed_maildir above to the same value as
# imap_to_maildirs.
imap_maildir_exceptions = [] # or list folders like Drafts
# that should not be moved to maildirs
sync_command = None # or command to run after every cycle,
# e.g. "mbsync -a" or "~/.local/bin/offlineimap" if you
# want to maintain maildirs + remote IMAP folders in same state
# (if poll_interval is "idle" this can wait for the next
# change on the _remote_ side before it runs)
sync_needed_only_if_wrote_maildirs = False # use e.g. if
# sync_command is just a 'backup maildirs to somewhere'
# and you don't mind backing up changes by non-imapfix
# mail clients less frequently
maildir_to_copyself = None # or path to a maildir whose
# messages should be moved to imap's copyself (for example if
# mutt is run on the same machine and saves its messages
# locally, which is more responsive than uploading to imap,
# but you still want them to be uploaded to imap eventually)
copyself_delete_attachments = False # if True, attachments
# are DELETED when moving from maildir_to_copyself to the
# IMAP folder (but a record is still kept of what was
# attached, unlike the fcc_attach='no' setting in Mutt 1.5)
copyself_folder_name = "Sent Items"
# Used as a destination folder for maildir_to_copyself
# and/or copyself_delete_attachments, etc.
# (I call it "copyself" because I first used email on a
# local-area network in 1995 with Pegasus Mail for DOS,
# which saved "copies to self" in "copyself.pmi";
# nowadays "sent" or "sent items" seems more common for
# new setups.)
# If you want to keep this on a local maildir instead, you
# can set it to ("maildir","path/to/maildir")
# (in which case also setting maildir_to_copyself will
# result in messages simply being moved from one maildir to
# another, but with copyself_delete_attachments still applied)
copyself_alt_folder = None # or the name of an IMAP
# folder, any messages found (if folder exists) will be
# moved to copyself_folder_name, with their attachments
# deleted if copyself_delete_attachments is True. You can
# specify more than one folder by separating them with a
# comma. Might be useful if some of your IMAP programs
# insist on doing their own sent-mail filing to folders of
# their own choice rather than yours.
check_copyself_alt_folder_on_secondary_too = False
# (see secondary_imap below)
auto_delete_folder = None # or "Trash" if you want everything
# in it to be automatically deleted periodically; useful if
# your IMAP client insists on moving messages there when you
# wanted them deleted permanently; use with caution
archive_path = "oldmail"
archive_rules = [
# These rules are used when you run with --archive
# Each rule is: ("folder-name", max-age (days), spamprobe-action)
# If max-age is not None, older messages will be archived (so you can set it to 0 to archive all messages). Independently of this, spamprobe-action (if specified) will be run on all messages.
# Folder name can be ("maildir","path/to/maildir") if
# you also want to archive from local maildirs to mbox
("spam-confirmed", 30, "train-spam"),
("some-folder", 90, None),
("some-other-folder", None, "train-good"),
]
compression = "bz2" # or "gz" or None
archived_attachments_path = "archived-attachments" # or None
save_attachments_for_confirmed_spam_too = False
attachment_filename_maxlen = 30
train_spamprobe_nightly = False # if true, will also run
# spamprobe training commands on yesterday's messages in
# these folders every midnight, not just during --archive,
# to catch any message-filing done the previous day
additional_inbox = None
# If you're using an IMAP server that runs its own spam
# filtering, but you want to replace this with yours, you
# can set additional_inbox to their spam folder to treat
# their spam folder as an extra inbox to be reclassified.
# Make sure additional_inbox does not equal spam_folder.
# Alternatively, you can leave additional_inbox at None
# and set spam_folder to match their spam folder to have
# 'or'-logic spam detection, but then you'd have to make
# any manual reviews / saves to spam-confirmed are done
# on the timetable set by the IMAP server (typically
# every 30 days will suffice), plus it may help to keep
# spam-confirmed off-IMAP to avoid accidentally telling
# their system that these messages are non-spam for you.
additional_inbox_train_spam = False # setting this
# to True is another way to get the 'or'-logic if you'd
# rather keep all spam elsewhere e.g. spam_folder is a
# local maildir. header_rules and extra_rules still
# override this.
check_additional_inbox_on_secondary_too = False
# (see secondary_imap below)
additional_inbox_might_not_exist = False # set to True if
# the folder does not appear when it's empty, in which
# case folder not existing will not be treated as an error
forced_names = {} # you can set this to a dictionary
# mapping email address to From name, and whatever other
# From name is used will be replaced by the one specified;
# this is for when one of your contacts has their "From"
# name set to something confusing like their ISP's name
important_regexps = [
# List here any regular expressions that will result in
# the "Importance" flag of a message being set. If the
# list is empty then Importance is unchanged.
# The default rule here marks as 'important' any message
# that seems to include a phone number, as an aid to
# quickly finding these when processing email on a phone
# (if you'd rather call than type and would therefore
# like to find messages that contain phone numbers)
r"0(?:\s*[0-9]){10}",
]
# Set secondary_imap_hostname if you also want to check
# some other IMAP server and treat its messages as being
# in the inbox of the main server (i.e. copied over and
# processed as normal). You can check this less often by
# setting secondary_imap_delay. (Will NOT stay logged in
# between checks.) It's also possible to set the first
# three of these options to lists, in order to check
# multiple secondary IMAP servers or multiple identities.
secondary_imap_hostname = "" # host or host:port
secondary_imap_username = "me"
secondary_imap_password = "xxxxxxxx"
secondary_imap_delay = 24 * 3600
report_secondary_login_failures = False # if True, put a
# message in filtered_inbox about any login failures for
# any of the secondary_imap boxes (default just ignores &
# tries again next time)
secondary_is_insecure = False # if True, the --copy option
# will remove all email addresses (but not names) when
# copying to secondary. This is for when the secondary
# IMAP server is easier to log in to than the primary, but
# is less secure. E.g. your old WM6.5 phone can't do the
# modern TLS version of IMAPS, and the only other option
# is a completely unencrypted connection to some throwaway
# account somewhere. Using --copy to copy over messages
# to read on that phone, but with secondary_is_insecure as
# True, will mean anyone who breaks into that throwaway
# account can see the messages but won't easily be able to
# reply to them for scamming purposes. It will of course
# mean you can't reply so easily as well, but the idea is
# that the throwaway account is only for READING messages
# on the mobile, not for actual replying or management.
first_secondary_is_copy_only = False # if True, the first
# server listed in secondary will be used ONLY for --copy
# and not for periodic checking.
secLimit = 99999 # max number of bytes checked for email
# addresses by secondary_is_insecure (to prevent holdups
# if you have multi-megabyte attachments)
insecure_login = [] # set to a list of host names (or host:port)
# not expected to support SSL. This speeds things up by
# trying the non-SSL login first on those hosts. See also
# secondary_is_insecure option above.
exit_if_imapfix_config_py_changes = False # if True, does
# what it says, on the assumption that a wrapper script
# will restart it (TODO: make it restart by itself?)
# - and if this is set to the special value "stamp", it
# will additionally try to update the timestamp of the
# config file before starting, so that any other instance
# (even on another machine in a cluster) will stop; this
# can be more reliable than exit_if_other_running below
# if running on a cluster.
failed_address_to_subject = True # try to rewrite delivery
# failure reports so that failed addresses are included in
# the Subject line and are therefore visible from a table
# of subjects without having to go in to the message.
# Not all delivery failure reports can be adjusted in this
# way; it depends on how the bouncing relay formats them.
imap_8bit = False # set to True if your IMAP server & clients
# will accept raw 8-bit UTF-8 strings in message bodies (but
# this is not universal, so make sure it re-encodes as MIME
# when forwarding messages to others). Raw UTF-8 messages
# (if supported) save a little bandwidth and allow characters
# to be more readily displayed in Mutt's "Edit Message" etc.
archive_8bit = False # similar if you want --archive as 8bit
exit_if_other_running = True # when run without options,
# try to detect if another no-options imapfix is running
# as the same user, and exit if so. Not guaranteed (for
# example, it can't protect against instances being run on
# different machines of a cluster), but might help to
# reduce the build-up of processes if there is a runaway
# situation with whatever you're using to start them.
# Multiple imapfix instances are sort-of OK but may lead
# to messages being processed in duplicate and/or load the
# IMAP server too much; on the other hand lock files etc
# can have the problem of 'stale' locks. At least a limit
# of 1 process per server in the cluster is better than no
# limit at all. On the other hand you might want to set
# this to False if you run different instances of imapfix
# with different configuration files as the same user and
# don't want to rename (or symlink) imapfix so that these
# look different in the process table.
# (exit_if_other_running needs the Unix 'ps' command.)
# The newer exit_if_imapfix_config_py_changes="stamp"
# option (above) is likely better if you don't mind
# changing the timestamp of your imapfix_config.py.
alarm_delay = 0 # with some Unix networked filesystems it is
# possible for imapfix or one of its subprocesses to get
# "stuck" - if this happens, set alarm_delay to a number of
# seconds (preferably thousands) after which to terminate the
# process using the Unix "alarm clock" mechanism. The clock
# will be reset every half that number of seconds if imapfix
# is still functioning. Note that a long-running filter etc
# could also cause imapfix to become "stuck" this long.
# Command-line options
# --------------------
# Run with no options = process mail as normal
# Run with --once = as if poll_interval=False
# Run with --quicksearch (search string) to search both
# archive_path and the server (all folders), but "quick"
# in that it doesn't decode MIME attachments etc
# (TODO: implement more thorough search, with regexps, but
# it would have to download all messages from the server)
# TODO: this does not yet search local maildirs
# Run with --delete (folder name) to delete a folder
# --delete-secondary to do it on secondary_imap_hostname
# (if there's more than one secondary, the first is used)
# --create (folder name) to create an empty folder
# (e.g. "Sent", some Android apps e.g. K-9 Mail require the
# folder to already exist before they can save messages to it
# (if the folder has been deleted, to get the functionality
# back do Manage folders / Refresh folder list, and Account
# Settings / Folders / Sent folder - set it to Sent)
# --create-secondary to do this on secondary_imap_hostname
# Run with --backup to take a full backup of ALL folders to local .mbox files
# without separating off attachments. Use this for example if your account is
# about to be cancelled and you want to be able to restore things as-is after
# reinstatement or migration to a new service, Read/Answered flags preserved.
# Run with --copy (folder name) to copy a folder onto secondary_imap_hostname
# (updating any that already exists of that name: may delete
# messages on secondary_imap_hostname that aren't on the primary;
# 1st secondary is used if there's a list of many)
# Run with --note (subject) to put a note to yourself
# (taken from standard input) directly into filtered_inbox
# - useful from scripts etc (you can get the note without
# having to wait for it to go through SMTP and polling).
# If filtered_inbox is None, --note uses real inbox,
# or use --note-inbox for the same effect.
# Run with --htmlnote to do the same but send as HTML.
# Run with --maybenote to do --note only if standard input
# has text (no mail left if your script printed nothing).
# Use --from=X to override the From name to X for notes
# (quote it for the shell if it's more than one word)
# Run with --multinote (files) to transfer a group of text
# files into filtered_inbox notes. The first line of each
# file is the subject line of the note. Files will be
# deleted after being successfully uploaded to IMAP.
# Directories are processed recursively but not deleted.
# Backup files (with names ending ~) are deleted without
# being uploaded to IMAP.
# If handle_authenticated_message has been redefined, this
# will be called as well when using --multinote. (If you
# are using this function on a machine that is different
# from the one that does your mail processing, you may
# want to define handle_authenticated_message to
# return "" which will result in the messages being placed
# in the real inbox for processing by the other machine.
# Alternatively you can use --multinote-inbox for the same effect.)
# Use --multinote-fname (files) to use each file's filename
# instead of its first line as the subject, or --multinote-inbox-fname.
# All note options assume that any non-ASCII characters in
# the input will be encoded as UTF-8.
# Run with --upload (files) to upload as attachments into
# filtered_inbox messages, e.g. for transfer to a mobile
# (likely more efficient than SMTP and bypasses filtering)
# - recurses into directories if specified; won't delete
# files after uploading.
# End of options - non-developers can stop reading now :)
# -------------------------------------------------------
# CHANGES
# -------
# If you want to compare this code to old versions, most old
# versions are being kept on SourceForge's E-GuideDog SVN repository
# http://sourceforge.net/p/e-guidedog/code/HEAD/tree/ssb22/setup/
# use: svn co http://svn.code.sf.net/p/e-guidedog/code/ssb22/setup
# and on GitHub at https://github.com/ssb22/web-imap-etc
# and on GitLab at https://gitlab.com/ssb22/web-imap-etc
# and on Bitbucket https://bitbucket.org/ssb22/web-imap-etc
# and at https://gitlab.developers.cam.ac.uk/ssb22/web-imap-etc
# and in China: https://gitee.com/ssb22/web-imap-etc
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
if not "--help" in sys.argv and not "--version" in sys.argv:
import imapfix_config
from imapfix_config import *
import email,email.utils,time,os,sys,re,base64,quopri,mailbox,traceback,mimetypes
if not sys.version_info[0]==2:
print ("ERROR: ImapFix is a Python 2 program and should be run with 'python2'.\nIt needs major revision for Python 3's version of the email library.\nTry compiling Python 2.7 in your home directory if it's no longer installed on your system.")
# In particular, Python 3.3+ revised the Message class into an EmailMessage class (with Message as a compatibility option), need to use as_bytes rather than as_string; set_payload available only in compatibility mode and works in Python3 Unicode-strings so we'd need to figure out how to handle other charsets including invalid coding.
# That's on top of the usual 'make sure all our code works whether or not type("")==type(u"")' issue.
# (And imghdr is deprecated in 3.11 for removal in 3.13, so will need an alternative to imghdr.what())
sys.exit(1)
from email import encoders
from cStringIO import StringIO
if poll_interval=="idle":
import imaplib2 as imaplib
assert not logout_before_sleep, "Can't logout_before_sleep when poll_interval==\"idle\""
else: import imaplib
assert not secondary_imap_delay=="idle", "'idle' polling not implemented for secondary"
if filtered_inbox==None: spamprobe_command = None
if compression=="bz2":
import bz2
compression_ext = ".bz2"
elif compression=="gz":
import gzip
compression_ext = ".gz"
else: compression_ext = ""
if image_size:
from PIL import Image
import imghdr
import commands
running_mainloop = False
def debug(*args):
args = reduce((lambda a,b:a+str(b)), args, time.asctime()+": " if running_mainloop else "")
if quiet==False: print (args)
elif not quiet: # quiet=None (not False) means use stderr. (In Python, 0==False but None!=False)
sys.stderr.write(args+"\n")
def check_ok(r):
"Checks that the return value of an IMAP call 'r' is OK, raises exception if not"
typ, data = r
if not typ=='OK': raise Exception(typ+' '+repr(data))
def spamprobe_rules(message,already_spam=False):
if already_spam: # additional_inbox_train_spam
run_spamprobe("train-spam",message) # might help our own classifier to spot other features that weren't what the IMAP server's classifier picked up on (as long as we train-good if it wasn't)
return spam_folder # (if JUST doing this, rename additional_inbox_train_spam to additional_inbox_direct_to_spam or something)
elif run_spamprobe("train",message).startswith("SPAM"): return spam_folder # "train" = classify + maybe update database if wasn't already confident ("receive" = always update database)
else: return filtered_inbox
def run_spamprobe(action,message):
if not spamprobe_command: return ""
if spamprobe_remove_images:
msg = email.message_from_string(message)
if delete_images(msg):
message=myAsString(msg)
cIn, cOut = os.popen2(spamprobe_command+" "+action)
cIn.write(message); cIn.close()
return cOut.read()
def spamprobe_cleanup():
if not spamprobe_command: return
debug("spamprobe cleanup")
os.system(spamprobe_command+" cleanup")
def process_header_rules(header):
for box,matchList in header_rules:
for headerLine in header.split("\n"):
for m in matchList:
i=re.finditer(m,headerLine.rstrip())
try: i.next()
except StopIteration: continue
return rename_folder(box)
return False
def myAsString(msg):
message = msg.as_string()
if not "\r" in message: message=message.replace("\n","\r\n") # in case it came in from Maildir and the library didn't add the CRs back in
if not "\r\n\r\n" in message or ("\n\n" in message and message.index("\n\n")<message.index("\r\n\r\n")):
# oops, broken library?
message=message.replace("\n\n","\r\n\r\n",1)
a,b = message.split("\r\n\r\n",1)
message = re.sub('\r*\n','\r\n',a)+"\r\n\r\n"+b
# Bug in python2.7/email/header.py: CRLF + whitespace sometimes added by _split_ascii after semicolons or commas, but if this occurs inside quoted-printable strings it'll break the display in some versions of alpine & mutt (RFC 2822 says CRLF + whitespace is interpreted as whitespace, and that's not allowed inside ?=..?= sections)
a,b = message.split("\r\n\r\n",1)
message = re.sub(header_charset_regex,lambda x:re.sub(r"\s_","_",re.sub("\r\n\s","",x.group())),a,flags=re.DOTALL)+"\r\n\r\n"+b
return message
def headers(msg):
# some Python libraries buggy, so do it ourselves
return set(l.split(None,1)[0][:-1] for l in myAsString(msg).split("\r\n\r\n",1)[0].split("\n") if l and l[0].strip())
imapfix_name = sys.argv[0]
if os.sep in imapfix_name: imapfix_name=imapfix_name[imapfix_name.rindex(os.sep)+1:]
if not imapfix_name: imapfix_name = "imapfix.py"
from_name = imapfix_name
for i in range(1,len(sys.argv)):
if sys.argv[i].startswith('--from='):
from_name = sys.argv[i][len('--from='):]
del sys.argv[i] ; break
try: _a,_b = imapfix_name.split('.',1)
except: _a,_b = imapfix_name,'0'
from_addr = " <%s@%s>" % (_a,_b) # (the part that doesn't change under --from=)
from_line = from_name+from_addr # previously just imapfix_name, but K-9 Mail Version 5.8 started to require this to be formatted with an @ or it would just say "Unknown Sender"; imapfix 1.76+ supports any name before the address
del _a,_b
def postponed_match(subject):
if postponed_foldercheck:
m = re.match(isoDate,subject)
if m and m.group() >= isoToday(): return m.end() # TODO: Y10K (lexicographic comparison)
if postponed_daynames:
m = re.match(weekMonth,subject)
if m: return m.end()
def authenticated_wrapper(subject,firstPart,attach={}):
subject = subject.replace('\n',' ').replace('\r','') # in case the original header is corrupt (c.f. globalise_charsets)
mLen = postponed_match(subject)
if mLen:
newSubj = re.sub("^:","",subject[mLen:]).lstrip() # take the date itself out of the subject line before putting the message in that folder: it could take up valuable screen real-estate in summaries on large-print or mobile devices, and just duplicates information that can be found in the folder name (or in the Date: field after it's put back into filtered_inbox)
if not newSubj: # oops, subject contained only the date and nothing else: take from 1st line of text instead
newSubj = firstPart.lstrip().split('\n')[0]
if len(newSubj) > 60: newSubj = newSubj[:57] + "..." # TODO: configurable abbreviation length?
if postponed_maildir: box=('maildir',postponed_maildir+os.sep+subject[:mLen])
else: box=subject[:mLen] # don't resolve weekday/month names to a date here, because user might actually rely on the "doesn't check for past days on startup" behaviour to postpone to after a trip or something
return box, newSubj
try: r=handle_authenticated_message(subject,firstPart,attach)
except:
if not catch_extraRules_errors: raise # TODO: document it's also catch_authMsg_errors, or have another variable for that
o = StringIO() ; traceback.print_exc(None,o)
save_to(filtered_inbox,"From: "+from_line+"\r\nSubject: imapfix_config exception in handle_authenticated_message, treating it as return False\r\nDate: %s\r\n\r\n%s\n" % (email.utils.formatdate(localtime=True),o.getvalue()))
r=False
return r, None
def yield_all_messages(searchQuery=None,since=None):
"Generator giving (message ID, flags, message) for each message in the current folder of 'imap', without setting the 'seen' flag as a side effect. Optional searchQuery limits to a text search."
if searchQuery: typ, data = imap.search(None, 'TEXT', '"'+searchQuery.replace("\\","\\\\").replace('"',r'\"')+'"')
elif since: typ, data = imap.search(None, 'SINCE', since)
else: typ, data = imap.search(None, 'ALL')
if not typ=='OK': raise Exception(typ)
bodyPeek_works = True # will set to False if it doesn't
dList = data[0].split(); count = 1
lastTime = time.time()
for msgID in dList:
if not quiet and time.time() > lastTime + 2:
sys.stdout.write(" %d%%\r" % (100*count/len(dList))) ; sys.stdout.flush()
lastTime = time.time()
count += 1
typ, data = imap.fetch(msgID, '(FLAGS)')
if not typ=='OK': continue
flags = data[0]
if not flags: flags = ""
if '\\Deleted' in flags: continue # we don't mark messages deleted until they're processed; if imapfix was interrupted in the middle of a run, then don't process this message a second time
if '(' in flags: flags=flags[flags.rindex('('):flags.index(')')+1] # so it's suitable for imap.store below
if bodyPeek_works:
typ, data = imap.fetch(msgID, '(BODY.PEEK[])')
if not typ=='OK': bodyPeek_works = False
if not bodyPeek_works: # fall back to older RFC822:
typ, data = imap.fetch(msgID, '(RFC822)')
if not "seen" in flags.lower(): # fetching RFC822 will set 'seen' flag, so we'll need to clear it again
try: imap.store(msgID, 'FLAGS', flags)
except: imap.store(msgID, 'FLAGS', "()") # gmail can give a \Recent flag but not accept setting it
if not typ=='OK': continue
try: yield msgID, flags, data[0][1] # data[0][0] is e.g. '1 (RFC822 {1015}'
except: continue # data==None or data[0]==None
def rewrite_deliveryfail(msg):
if not failed_address_to_subject: return
subj = msg.get("Subject","")
if not subj.lower().startswith("mail delivery") or not msg.get("From","").lower().startswith("mail delivery"): return
fr = msg.get("X-Failed-Recipients","")
if not fr: return
del msg['Subject'] ; msg['Subject']=fr+' '+subj
return True
for k in forced_names.keys():
if not k==k.lower():
forced_names[k.lower()]=forced_names[k]
del forced_names[k]
def forced_from(msg):
def f(fr):
if fr.lower() in forced_names:
del msg['From']
msg["From"] = forced_names[fr.lower()] + ' <'+fr.lower()+'>'
return True
fr = msg.get("From","").strip()
if f(fr): return True
if fr.endswith('>') and '<' in fr and f(fr[fr.rindex('<')+1:-1]): return True
# TODO: any other formats to check?
def quote_display_name_if_needed(msg):
# 'From: Testing @ ZOE COVID Study <...>'
# gave 'Unknown Sender' in K9, please quote it
def ensureQuoted(dnMatch):
displayName = dnMatch.group()
if re.match("^[A-Za-z0-9 .]*$",displayName):
# should be OK to leave unchanged if it didn't use @ etc
return displayName
else: return '"'+displayName+'"'
f = msg.get("From","").strip()
f2 = re.sub(r'^[^<"]*(?= <[^>]*>$)',ensureQuoted,f)
if not f==f2:
del msg['From'] ; msg['From'] = f2 ; return True
def body_text(msg):
"Returns a representation of the message's body text (all parts), for rules"
if msg.is_multipart(): return "\n".join(body_text(p) for p in msg.get_payload())
if not msg.get_content_type().startswith("text/"): return ""
return msg.get_payload(decode=True).strip()
def rewrite_importance(msg):
if not important_regexps: return
changed = False
for h in ['Priority', # e.g. Urgent
'X-Priority', # 1 is highest
'X-MSMail-Priority', # e.g. High
'Importance']: # e.g. High; clients that recognise the previous 3 tend to recognise this one as well, and other clients recognise only this one, so we'll clear all but set only this one
if h in msg:
del msg[h] ; changed = True
bTxt = body_text(msg)
for r in important_regexps:
if re.search(r,bTxt):
msg['Importance'] = 'High'; changed=True; break
return changed
def rewrite_return_path(msg):
if not rewrite_return_path_SRS or not 'Return-Path' in msg: return
rp = msg["Return-Path"]
rp2 = re.sub(r"(?i)^<SRS(?:0|(?:1.*=))=[0-9a-z]+=[0-9a-z]+=([^=]+)=([^@]+)@.*$",r"\2@\1",rp)
if not (rp2.startswith('<') and rp2.endswith('>')): rp2='<'+rp2+'>' # even if not SRS (TODO: document that this is also controlled by rewrite_return_path_SRS; helps with rule portability, e.g. between gmail and outlook servers)
if not rp==rp2:
del msg["Return-Path"]
msg["Return-Path"] = rp2 ; return True
def process_imap_inbox():
make_sure_logged_in()
doneSomething = True
while doneSomething:
doneSomething = False # if we do something on 1st loop, we'll loop again before handing control back to the delay or IMAP-event wait. This is to catch the case where new mail comes in WHILE we are processing the last batch of mail (e.g. another imapfix is running with --multinote and some of the processing calls send_mail() with a callSMTP_time delay: we don't want the rest of the notes to be delayed 29 minutes for the next IMAP-wait to time out)
for is_additional in [False,True]:
if is_additional:
if not additional_inbox: break
if additional_inbox_might_not_exist:
typ, data = imap.select(additional_inbox)
if not typ=='OK': break
else: check_ok(imap.select(additional_inbox))
else: check_ok(imap.select()) # the inbox
imapMsgid = None ; newMail = False
for msgID,flags,message in yield_all_messages():
if not is_additional and leave_note_in_inbox and imap==saveImap and isImapfixNote(message):
if imapMsgid: # somehow ended up with 2, delete one
imap.store(imapMsgid, '+FLAGS', '\\Deleted')
imapMsgid = msgID ; continue
doneSomething = True
msg = email.message_from_string(message)
box,message,changed,changed0,seenFlag = handleMsg(msg,message,is_additional)
if box:
if seenFlag: copyWorked = False # unless we can get copy_to to set the Seen flag on the copy, which means the IMAP server must have an extension that causes COPY to return the new ID unless we want to search for it
elif type(box)==tuple: copyWorked = False # e.g. imap to maildir
elif not changed and saveImap == imap and (not imap_to_maildirs or box in imap_maildir_exceptions):
debug("Copying unchanged message to ",box)
copyWorked = copy_to(box, msgID)
if not copyWorked: debug("... failed; falling back to re-upload")
else: copyWorked = False
if not copyWorked:
debug("Saving message to ",box)
try: save_to(box, message, seenFlag, changed0 and saveImap==imap)
except: # uh-oh
debug("Save FAIL: writing error and halting")
save_to(filtered_inbox,"From: "+from_line+"\r\nSubject: imapfix save error, HALTED\r\nDate: %s\r\n\r\nsave_to(%s) FAILED.\nThis can happen when you have quota issues with a very large message.\nFor safety, imapfix will now HALT processing.\nYou should terminate pid %s once you've sorted out the inbox.\n" % (email.utils.formatdate(localtime=True),box,str(os.getpid()))) # TODO: could try saving without attachments and quarantine the attachments somewhere; could try leaving it as 'seen' and checking for 'seen' on future loop iterations
imap.expunge() # the ones we did before this
make_sure_logged_out()
if sync_command: os.system(sync_command)
while True: time.sleep(60)
if box==filtered_inbox: newMail = True
else: debug("Deleting message")
imap.store(msgID, '+FLAGS', '\\Deleted')
if not is_additional and leave_note_in_inbox and imap==saveImap:
if newMail:
if imapMsgid: # delete the old one
imap.store(imapMsgid, '+FLAGS', '\\Deleted')
save_to("", imapfixNote())
elif imapMsgid:
# un-"seen" it (in case the IMAP server treats our fetching it as "seen"); TODO what if the client really read it but didn't delete?
imap.store(imapMsgid, '-FLAGS', '\\Seen')
check_ok(imap.expunge())
if (not quiet) and imap==saveImap and not type(filtered_inbox)==tuple: debug("Quota ",repr(imap.getquotaroot(filtered_inbox)[1])) # RFC 2087 "All mailboxes that share the same named quota root share the resource limits of the quota root" - so if the IMAP server has been set up in a typical way with just one limit, this command should print the current and max values for that shared limit. (STORAGE = size in Kb, MESSAGE = number)
def handleMsg(msg,message=None,is_additional=True):
box = changed = changed0 = bypass_spamprobe = False ; seenFlag=""
if authenticates(msg):
# do auth'd-msgs processing before any convert-to-attachment etc
debug("Message authenticates")
bypass_spamprobe = True
box,newSubj = authenticated_wrapper(re.sub(header_charset_regex,header_to_u8,msg.get("Subject",""),flags=re.DOTALL),getFirstPart(msg).lstrip(),get_attachments(msg))
if newSubj: # for postponed_foldercheck
del msg["Subject"]
msg["Subject"] = utf8_to_header(newSubj)
changed = True
if box and box[0]=='*': seenFlag="\\Seen"
box=rename_folder(box,False) # don't set newmail markers for authenticated
if not box==None: # authenticates didn't say delete
# globalise charsets BEFORE the filtering rules
# (especially if they've been developed based on
# post-charset-conversion saved messages) - but
# no point doing preview images or Office
# conversions for spam or to-delete messages
changed = globalise_charsets(msg,imap_8bit) or changed
changed = remove_blank_inline_parts(msg) or changed
changed = rewrite_deliveryfail(msg) or changed
changed = forced_from(msg) or changed
changed = quote_display_name_if_needed(msg) or changed
changed = rewrite_importance(msg) or changed
changed = rewrite_return_path(msg) or changed
if max_size_of_first_part and size_of_first_part(msg) > max_size_of_first_part: msg,changed = turn_into_attachment(msg),True
if changed or not message: message = myAsString(msg)
changed0 = changed # for change_message_id
if box==False: # authenticates didn't decide a box
header = message[:message.find("\r\n\r\n")]
if add_return_path and not "\nReturn-Path" in header and "From" in msg:
header += "\r\nReturn-Path: <"+re.sub(">.*","",re.sub("^[^<]*<","",msg["From"]))+">" # to help processing rules depending on that, for locally-delivered messages without Return-Path or unixfrom
message = header+message[message.find("\r\n\r\n"):]
box = process_header_rules(header)
if box==False:
try: box = rename_folder(extra_rules(message))
except:
if not catch_extraRules_errors: raise
o = StringIO() ; traceback.print_exc(None,o)
save_to(filtered_inbox,"From: "+from_line+"\r\nSubject: imapfix_config exception in extra_rules (message has been saved to '%s')\r\nDate: %s\r\n\r\n%s\n" % (filtered_inbox,email.utils.formatdate(localtime=True),o.getvalue()))
box = filtered_inbox
if box==False:
if bypass_spamprobe: box = filtered_inbox
else: box = spamprobe_rules(message,is_additional and additional_inbox_train_spam)
if box and not box==spam_folder:
if headers_to_delete and delete_headers(msg):
changed0 = changed = True # for change_message_id
added = True # so myAsString happens
else: added = False
if use_tnef: added = add_tnef(msg) or added
if image_size: added = add_previews(msg) or added
if office_convert: added = add_office(msg) or added
if pdf_convert: added = add_pdf(msg) or added
changed = added or changed # don't need to alter changed0 here, since if the only change is adding parts then change_message_id does not need to take effect (at least, not with Gmail January 2022)
if added: message = myAsString(msg)
return box,message,changed,changed0,seenFlag
def process_maildir_inbox():
if not os.path.exists(maildir_to_process+os.sep+"cur"):
return # no error if it's not currently a maildir
mailbox.Maildir.colon = maildir_colon
m = get_maildir(maildir_to_process)
said = False
for k,msg in m.items():
if not said:
debug("Processing messages from ",maildir_to_process)
said = True
box,message,_,_,seenFlag = handleMsg(msg)
if box:
debug("Saving message to ",box)
save_to(box, message, seenFlag, False)
else: debug("Deleting message")
del m[k]
m.clean()
def authenticates(msg):
return trusted_domain and smtps_auth and authenticates0(msg,trusted_domain,smtps_auth) or super_trusted_domain and authenticates0(msg,super_trusted_domain,None)
def authenticates0(msg,trusted_domain,smtps_auth):
rxd = msg.get_all("Received",[])