Merge pull request #29 from perrette/dev

Flexible key and file format

README.md

@@ -66,13 +66,52 @@ For the sake of the example, one of my owns: https://www.earth-syst-dynam.net/4/
     	INFO:papers:found doi:10.5194/esd-4-11-2013
     	INFO:papers:new entry: perrette_2013
     	INFO:papers:create directory: files/2013
-    	INFO:papers:mv /home/perrette/playground/papers/esd-4-11-2013.pdf files/2013/Perrette_2013.pdf
+    	INFO:papers:mv /home/perrette/playground/papers/esd-4-11-2013.pdf files/perrette_et_al_2013_a-scaling-approach-to-project-regional-sea-level-rise-and-its-uncertainties.pdf
     	INFO:papers:renamed file(s): 1
 
 (the `--info` argument asks for the above output information to be printed out to the terminal)
 
-In the common case where the bibtex (`--bibtex`) and files directory  (`--filesdir`) do not change,
-it is convenient to *install* `papers`.
+
+- control fields when renaming file
+
+        $> papers add --rename --info --name-template "{AuthorX}{year}-{Title}" --name-title-sep '' --name-author-sep '' esd-4-11-2013
+        INFO:papers:found doi:10.5194/esd-4-11-2013
+        INFO:papers:new entry: perrette2013scaling
+        INFO:papers:create directory: files/2013
+        INFO:papers:mv /home/perrette/playground/papers/esd-4-11-2013.pdf files/PerretteEtAl2013-AScalingApproachToProjectRegionalSeaLevelRiseAndItsUncertainties.pdf
+        INFO:papers:renamed file(s): 1
+
+where '--name-template' is a python template (will be formated via .format() method) with valid fields being any field available in the bibtex. Fields not in the bibtex will remain untouched.
+
+To rename `esd-4-11-2013.pdf` as `perrette_2013.pdf`, the template should be `--name-template {author}_{year} --name-author-num 1`
+If that happens to be the entry ID, `ID` also works.
+
+To `rename esd-4-11-2013.pdf` as `2013/Perrette2013-AScalingApproachToProjectRegionalSeaLevelRiseAndItsUncertainties.pdf`,
+name-template should be `--name-template {year}/{Author}{year}-{Title} --name-title-sep ''` (note the case).
+
+Entries are case-sensitive, and a few more fields are added, so that:
+- 'author' generates 'perrette'
+- 'Author' generates 'Perrette'
+- 'AUTHOR' generates 'PERRETTE'
+- 'authorX' generates 'perrette', 'perrette_and_landerer' or 'perrette_et_al' dependening on the number of authors
+- 'AuthorX' same as authorX but capitalized
+
+The modifiers are:
+
+- `--name-title-sep` : separator for title words
+- `--name-title-length` : max title length
+- `--name-title-word-size` : min size to be considered a word
+- `--name-title-word-num` : max number of title words
+
+and similarly:
+
+- `--name-author-sep` : separator for authors
+- `--name-author-num` : number of authors to  (not relevant for `{authorX}`)
+
+The same template and modifiers system applies to the bibtex key generation by replacing the prefix `--name-` with `--key-`, e.g. `--key-template`
+
+
+In the common case where the bibtex (`--bibtex`), files directory  (`--filesdir`), and name and key formats (e.g. `--name-template`) do not change, it is convenient to *install* `papers`.
 Install comes with the option to git-track any change to the bibtex file (`--git`) options.
 
 - setup git-tracked library (optional)

papers/bib.py

@@ -29,9 +29,6 @@
 
 # KEY GENERATION
 # ==============
-NAUTHOR = 2
-NTITLE = 0
-
 
 def append_abc(key, keys=[]):
     """
@@ -60,24 +57,6 @@ def append_abc(key, keys=[]):
     return Key
 
 
-def generate_key(entry, nauthor=NAUTHOR, ntitle=NTITLE, minwordlen=3, mintitlen=4, keys=None):
-    # names = bibtexparser.customization.getnames(entry.get('author','unknown').lower().split(' and '))
-    names = family_names(entry.get('author','unknown').lower())
-    authortag = '_'.join([nm for nm in names[:nauthor]])
-    yeartag = entry.get('year','0000')
-    if not ntitle or not entry.get('title',''):
-        titletag = ''
-    else:
-        words = [word for word in entry['title'].lower().strip().split() if len(word) >= minwordlen]
-        while len(''.join(words[:ntitle])) < mintitlen and ntitle < len(words):
-            ntitle += 1
-        titletag = '_'.join(words[:ntitle])
-    key = authortag + yeartag + titletag
-    if keys and key in keys: # and not isinstance(keys, set):
-        key = append_abc(key, keys)
-    return key
-
-
 # DUPLICATE DEFINITION
 # ====================
 
@@ -114,12 +93,11 @@ def entry_id(e):
     return (e.get('doi','').lower(), authortitle)
 
 
-
-
 FUZZY_RATIO = 80
 
 # should be conservative (used in papers add)
 DEFAULT_SIMILARITY = 'FAIR'
+# DEFAULT_SIMILARITY = 'PARTIAL'
 
 EXACT_DUPLICATES = 104
 GOOD_DUPLICATES = 103
@@ -172,7 +150,7 @@ def are_duplicates(e1, e2, similarity=DEFAULT_SIMILARITY, fuzzy_ratio=FUZZY_RATI
     except KeyError:
         raise ValueError('similarity must be one of EXACT, GOOD, FAIR, PARTIAL, FUZZY')
 
-    score = compare_entries(e1, e2, fuzzy=level==FUZZY_DUPLICATES)
+    score = compare_entries(e1, e2, fuzzy=target==FUZZY_DUPLICATES)
     logger.debug('score: {}, target: {}, similarity: {}'.format(score, target, similarity))
     return score >= target
 
@@ -205,8 +183,6 @@ def read_entry_dir(self, direc, update_files=True):
     return entry
 
 
-
-
 def backupfile(bibtex):
     return os.path.join(os.path.dirname(bibtex), '.'+os.path.basename(bibtex)+'.backup')
 
@@ -216,7 +192,7 @@ class DuplicateKeyError(ValueError):
 class Biblio:
     """main config
     """
-    def __init__(self, db=None, filesdir=None, key_field='ID', nauthor=NAUTHOR, ntitle=NTITLE, similarity=DEFAULT_SIMILARITY):
+    def __init__(self, db=None, filesdir=None, key_field='ID', nameformat=None, keyformat=None, similarity=DEFAULT_SIMILARITY):
         self.filesdir = filesdir
         # assume an already sorted list
         self.key_field = key_field
@@ -225,10 +201,10 @@ def __init__(self, db=None, filesdir=None, key_field='ID', nauthor=NAUTHOR, ntit
         elif not isinstance(db, bibtexparser.bibdatabase.BibDatabase):
             raise TypeError('db must of type BibDatabase')
         self.db = db
-        self.sort()
-        self.nauthor = nauthor
-        self.ntitle = ntitle
+        self.nameformat = nameformat or config.nameformat
+        self.keyformat = keyformat or config.keyformat
         self.similarity = similarity
+        self.sort()
 
     @property
     def entries(self):
@@ -356,7 +332,10 @@ def insert_entry_check(self, entry, update_key=False, mergefiles=True, on_confli
     def generate_key(self, entry):
         " generate a unique key not yet present in the record "
         keys = {self.key(e) for e in self.db.entries}
-        return generate_key(entry, keys=keys, nauthor=self.nauthor, ntitle=self.ntitle)
+        key = self.keyformat(entry)
+        if keys and key in keys: # and not isinstance(keys, set):
+            key = append_abc(key, keys)
+        return key
 
     def append_abc_to_key(self, entry):
         return append_abc(entry['ID'], keys={self.key(e) for e in self.entries})
@@ -378,13 +357,19 @@ def fetch_doi(self, doi, **kw):
         self.add_bibtex(bibtex, **kw)
 
 
-    def add_pdf(self, pdf, attachments=None, rename=False, copy=False, search_doi=True, search_fulltext=True, scholar=False, **kw):
+    def add_pdf(self, pdf, attachments=None, rename=False, copy=False, search_doi=True, search_fulltext=True, scholar=False, doi=None, **kw):
 
-        bibtex = extract_pdf_metadata(pdf, search_doi, search_fulltext, scholar=scholar)
+        if doi:
+            bibtex = fetch_bibtex_by_doi(doi)
+        else:
+            bibtex = extract_pdf_metadata(pdf, search_doi, search_fulltext, scholar=scholar)
 
         bib = bibtexparser.loads(bibtex)
         entry = bib.entries[0]
 
+        # convert curly brackets to unicode
+        bibtexparser.customization.convert_to_unicode(entry)
+
         files = [pdf]
         if attachments:
             files += attachments
@@ -449,26 +434,45 @@ def check_duplicates(self, key=None, eq=None, mode='i'):
         self.sort() # keep sorted
 
 
-    def rename_entry_files(self, e, copy=False):
+    def rename_entry_files(self, e, copy=False, formatter=None):
+        """ Rename files
+
+        See `confog.Format` class
+
+
+            To rename esd-4-11-2013.pdf as perrette_2013.pdf, nameformat should be '{author}_{year}' with --name-nauthor 1.
+            If that happens to be the entry ID, 'ID' also works.
+
+            To rename esd-4-11-2013.pdf as
+            2013/Perrette2013-AScalingApproachToProjectRegionalSeaLevelRiseAndItsUncertainties.pdf,
+            nameformat should be '{year}/{Author}{year}-{Title}' with --name-nauthor 1 (note the case).
+
+            Entries are case-sensitive, so that:
+                'author' generates 'perrette'
+                'Author' generates 'Perrette'
+                'AUTHOR' generates 'PERRETTE'
+            any other case, like 'AuTHoR', will retrieve the field from 'e' with unaltered case.
+        """
 
         if self.filesdir is None:
             raise ValueError('filesdir is None, cannot rename entries')
 
         files = parse_file(e.get('file',''))
         # newname = entrydir(e, root)
-        direc = os.path.join(self.filesdir, e.get('year','0000'))
+
+        direc = self.filesdir
 
         if not files:
             logger.info('no files to rename')
             return
 
-        autoname = lambda e: e['ID'].replace(':','-').replace(';','-') # ':' and ';' are forbidden in file name
+        newname = (formatter or self.nameformat)(e)
 
         count = 0
         if len(files) == 1:
             file = files[0]
             base, ext = os.path.splitext(file)
-            newfile = os.path.join(direc, autoname(e)+ext)
+            newfile = os.path.join(direc, newname+ext)
             if not os.path.exists(file):
                 raise ValueError(file+': original file link is broken')
             elif file != newfile:
@@ -480,7 +484,7 @@ def rename_entry_files(self, e, copy=False):
 
         # several files: only rename container
         else:
-            newdir = os.path.join(direc, autoname(e))
+            newdir = os.path.join(direc, newname)
             newfiles = []
             for file in files:
                 newfile = os.path.join(newdir, os.path.basename(file))
@@ -769,6 +773,52 @@ def main():
     grp.add_argument('--dry-run', action='store_true',
         help='no PDF renaming/copying, no bibtex writing on disk (for testing)')
 
+    grp.add_argument('--key-template', default=config.keyformat.template,
+        help='python template for generating keys (default:%(default)s)')
+    grp.add_argument('--key-author-num', type=int, default=config.keyformat.author_num,
+        help='number of authors to include in key (default:%(default)s)')
+    grp.add_argument('--key-author-sep', default=config.keyformat.author_sep,
+        help='separator for authors in key (default:%(default)s)')
+    grp.add_argument('--key-title-word-num', type=int, default=config.keyformat.title_word_num,
+        help='number of title words to include in key (default:%(default)s)')
+    grp.add_argument('--key-title-word-size', type=int, default=config.keyformat.title_word_size,
+        help='number of title words to include in key (default:%(default)s)')
+    grp.add_argument('--key-title-sep', default=config.keyformat.title_sep,
+        help='separator for title words in key (default:%(default)s)')
+
+    grp.add_argument('--name-template', default=config.nameformat.template,
+        help='python template for renaming files (default:%(default)s)')
+    grp.add_argument('--name-author-num', type=int, default=config.nameformat.author_num,
+        help='number of authors to include in filename (default:%(default)s)')
+    grp.add_argument('--name-author-sep', default=config.nameformat.author_sep,
+        help='separator for authors in filename (default:%(default)s)')
+    grp.add_argument('--name-title-word-num', type=int, default=config.nameformat.title_word_num,
+        help='number of title words to include in filename (default:%(default)s)')
+    grp.add_argument('--name-title-word-size', type=int, default=config.nameformat.title_word_size,
+        help='min size of title words to include in filename (default:%(default)s)')
+    grp.add_argument('--name-title-length', type=int, default=config.nameformat.title_length,
+        help='title length to include in filename (default:%(default)s)')
+    grp.add_argument('--name-title-sep', default=config.nameformat.title_sep,
+        help='separator for title words in filename (default:%(default)s)')
+
+
+    def set_format_config_from_cmd(o):
+        config.keyformat.template = o.key_template
+        config.keyformat.author_num = o.key_author_num
+        config.keyformat.author_sep = o.key_author_sep
+        config.keyformat.title_word_num = o.key_title_word_num
+        config.keyformat.title_word_size = o.key_title_word_size
+        config.keyformat.title_sep = o.key_title_sep
+
+        config.nameformat.template = o.name_template
+        config.nameformat.author_num = o.name_author_num
+        config.nameformat.author_sep = o.name_author_sep
+        config.nameformat.title_length = o.name_title_length
+        config.nameformat.title_word_num = o.name_title_word_num
+        config.nameformat.title_word_size = o.name_title_word_size
+        config.nameformat.title_sep = o.name_title_sep
+
+
     # status
     # ======
     statusp = subparsers.add_parser('status',
@@ -922,6 +972,7 @@ def savebib(my, o):
         help='ignore errors when adding multiple files')
 
     grp = addp.add_argument_group('pdf metadata')
+    grp.add_argument('--doi', help='provide DOI -- skip parsing PDF')
     grp.add_argument('--no-query-doi', action='store_true', help='do not attempt to parse and query doi')
     grp.add_argument('--no-query-fulltext', action='store_true', help='do not attempt to query fulltext in case doi query fails')
     grp.add_argument('--scholar', action='store_true', help='use google scholar instead of crossref')
@@ -936,7 +987,6 @@ def savebib(my, o):
 
 
     def addcmd(o):
-
         if os.path.exists(o.bibtex):
             my = Biblio.load(o.bibtex, o.filesdir)
         else:
@@ -964,7 +1014,7 @@ def addcmd(o):
                     my.add_pdf(file, attachments=o.attachment, rename=o.rename, copy=o.copy,
                             search_doi=not o.no_query_doi,
                             search_fulltext=not o.no_query_fulltext,
-                            scholar=o.scholar,
+                            scholar=o.scholar, doi=o.doi,
                             **kw)
 
                 else: # file.endswith('.bib'):
@@ -994,8 +1044,8 @@ def addcmd(o):
     grp.add_argument('--fix-key', action='store_true', help='fix key based on author name and date (in case misssing or digit)')
     grp.add_argument('--key-ascii', action='store_true', help='replace keys unicode character with ascii')
     grp.add_argument('--auto-key', action='store_true', help='new, auto-generated key for all entries')
-    grp.add_argument('--nauthor', type=int, default=NAUTHOR, help='number of authors to include in key (default:%(default)s)')
-    grp.add_argument('--ntitle', type=int, default=NTITLE, help='number of title words to include in key (default:%(default)s)')
+#     grp.add_argument('--nauthor', type=int, default=config.nauthor, help='number of authors to include in key (default:%(default)s)')
+#     grp.add_argument('--ntitle', type=int, default=config.ntitle, help='number of title words to include in key (default:%(default)s)')
     # grp.add_argument('--ascii-key', action='store_true', help='replace unicode characters with closest ascii')
 
     grp = checkp.add_argument_group('crossref fetch and fix')
@@ -1201,7 +1251,11 @@ def longmatch(word, target):
         if o.duplicates_tit:
             entries = list_dup(entries, key=title_id)
         if o.duplicates:
-            eq = lambda a, b: a['ID'] == b['ID'] or are_duplicates(a, b, similarity=level, fuzzy_ratio=o.fuzzy_ratio)
+            # QUESTION MARK: in latest HEAD before merge with @malfatti's PR, I used hard-coded "PARTIAL".
+            # I think that's because we might need to be inclusive here, whereas the default is conservative (parameter used for several functions with possibly differing requirements).
+            # (otherwise we'd have used the command-line option o.similarity, or possibly DEFAULT_SIMILARITY)
+            # Might need to revise later (the question mark is from a review after a long time without use)
+            eq = lambda a, b: a['ID'] == b['ID'] or are_duplicates(a, b, similarity="PARTIAL", fuzzy_ratio=o.fuzzy_ratio)
             entries = list_dup(entries, eq=eq)
 
         def nfiles(e):
@@ -1243,7 +1297,7 @@ def nfiles(e):
                 print(e['ID'].encode('utf-8'))
         elif o.one_liner:
             for e in entries:
-                tit = e['title'][:60]+ ('...' if len(e['title'])>60 else '')
+                tit = e.get('title', '')[:60]+ ('...' if len(e.get('title', ''))>60 else '')
                 info = []
                 if e.get('doi',''):
                     info.append('doi:'+e['doi'])
@@ -1335,6 +1389,7 @@ def gitcmd(o):
         return statuscmd(o)
 
     def check_install():
+        set_format_config_from_cmd(o)
         if not os.path.exists(o.bibtex):
             print('papers: error: no bibtex file found, use `papers install` or `touch {}`'.format(o.bibtex))
             parser.exit(1)