Using python to clone bundles in parallel

autoload/vundle.vim

@@ -86,5 +86,7 @@ let vundle#bundles = []
 let vundle#lazy_load = 0
 let vundle#log = []
 let vundle#updated_bundles = []
+let vundle#threads = 15
+let vundle#shallow_copy = 0
 
 " vim: set expandtab sts=2 ts=2 sw=2 tw=78 norl:

autoload/vundle/installer.vim

@@ -23,18 +23,146 @@ func! vundle#installer#new(bang, ...) abort
     return
   endif
 
-  let names = vundle#scripts#bundle_names(map(copy(bundles), 'v:val.name_spec'))
-  call vundle#scripts#view('Installer',['" Installing plugins to '.expand(g:vundle#bundle_dir, 1)], names +  ['Helptags'])
-
-  " This calls 'add' as a normal mode command. This is a buffer local mapping
-  " defined in vundle#scripts#view(). The mapping will call a buffer local
-  " command InstallPlugin which in turn will call vundle#installer#run() with
-  " vundle#installer#install().
-  call s:process(a:bang, (a:bang ? 'add!' : 'add'))
+  let specs = map(copy(bundles), 'v:val.name_spec')
+  let names = vundle#scripts#bundle_names(specs)
+  let headers = [
+        \  '" Installing plugins to '.expand(g:vundle#bundle_dir, 1)
+        \ ]
+  call vundle#scripts#view('Installer', headers, names +  ['Helptags'])
+
+  if (has('python') || has('python3')) && g:vundle#threads > 1
+    " This runs a Pool of threads to syncronize the bundles
+    call s:process_parallel(a:bang, specs, len(headers), g:vundle#threads)
+
+    " goto the last line and update the docs
+    exec ':+1'
+    call vundle#installer#run('vundle#installer#docs', 'helptags', [])
+  else
+    " This calls 'add' as a normal mode command. This is a buffer local mapping
+    " defined in vundle#scripts#view(). The mapping will call a buffer local
+    " command InstallPlugin which in turn will call vundle#installer#run() with
+    " vundle#installer#install().
+    call s:process(a:bang, (a:bang ? 'add!' : 'add'))
+  endif
 
   call vundle#config#require(bundles)
 endf
 
+func! s:process_parallel(bang, specs, headers, threads) abort
+  let cmds = map(copy(a:specs), 's:make_sync_command(a:bang, s:bundle_from_name(v:val))')
+  let python = has('python3') ? 'python3' : 'python'
+
+  execute python "<< EOF"
+import os
+import subprocess
+import sys
+from multiprocessing import Pool
+
+import vim
+
+# bytes vs str
+if sys.version_info < (3,):
+  def to_str(data):
+    return data
+else:
+  def to_str(data):
+    return data.decode('utf8')  # TODO: make sure utf8 is the right encoding
+
+def main(cmds, shas, threads):
+  ''' execute cmds in parallel and update the ui '''
+  iterable = Pool(threads).imap(sync, cmds)
+  iterable = ui(iterable, cmds, shas, len(cmds), threads)
+
+  for result in iterable:
+    yield result
+
+def sync(cmd):
+  ''' run cmd discarding the output '''
+  devnull = os.open(os.devnull, os.O_RDWR)
+
+  if sys.platform == 'win32':
+    import msvcrt
+    devnull = msvcrt.get_osfhandle(os.devnull)
+
+  # py3 output is bytes, convert it before returning
+
+  try:
+    out = subprocess.check_output(
+      cmd,
+      shell=True,
+      stdin=devnull,
+      stderr=subprocess.STDOUT,
+    )
+  except subprocess.CalledProcessError as error:
+    return (error.returncode, to_str(error.output))
+  except Excpetion as error:
+    return (-1, to_str(error.message))
+
+  return (0, to_str(out))
+
+def ui(iterable, cmds, shas, total, threads):
+  ''' compose the current running and the finished marks '''
+  vim.command('redraw')
+  for result in status(active(iterable, threads, total), cmds, shas):
+    vim.command('redraw')
+    yield result
+  vim.command('redraw')
+
+def active(iterable, start, total):
+  ''' mark the  bundles that are being downloaded '''
+  for running in range(start):
+    mark('active', running)
+
+  for result in iterable:
+    if running < total:
+      running += 1
+      mark('active', running)
+
+    yield result
+
+def status(iterable, cmds, shas):
+  ''' mark the status of the bundle '''
+  for current, (code, out) in enumerate(iterable):
+    if not len(cmds[current]):
+      mark('todate', current)
+    elif code != 0:
+      mark('error', current)
+    elif not len(shas[current]):
+      mark('new', current)
+    # elif not len(shas[current]):
+    else:
+      mark('updated', current)
+
+    yield (code, out)
+
+def mark(status, line):
+  ''' mark status on line '''
+
+  vim.command('''
+    exec ':{line}'
+    call s:sign('{status}')
+  '''.format(line=1 + line + headers, status=status))
+
+def clean(data):
+  ''' remove single quotes from the data to avoid escaping problems when using eval'''
+  return data.replace("'", '')
+
+cmd_sha = vim.eval('cmds')
+cmds = [cmd for cmd, sha in cmd_sha]
+shas = [sha for cmd, sha in cmd_sha]
+
+specs = vim.eval('a:specs')
+threads = int(vim.eval('a:threads'))
+headers = int(vim.eval('a:headers'))
+
+for pos, (result, out) in enumerate(main(cmds, shas, threads)):
+  vim.command("call s:log('Plugin {spec}')".format(spec=clean(specs[pos])))
+  vim.command("call s:log('{cmd}', '$ ')".format(cmd=clean(cmds[pos])))
+  vim.command("call s:log('{out}', '> ')".format(out=clean(out)))
+
+EOF
+endf
+
 
 " ---------------------------------------------------------------------------
 " Iterate over all lines in a Vundle window and execute the given command for
@@ -164,21 +292,32 @@ endf
 " return -- the return value from s:sync()
 " ---------------------------------------------------------------------------
 func! vundle#installer#install(bang, name) abort
+  let bundle = call s:bundle_from_name(name)
+  return s:sync(a:bang, bundle)
+endf
+
+
+" ---------------------------------------------------------------------------
+" Creates bundle detiny directory and return a bundle dictionary.
+"
+" name   -- the name_spec for the bundle
+" return -- 'error' if an error occurred, else return 'helptags'
+" ---------------------------------------------------------------------------
+func! s:bundle_from_name(name)
   if !isdirectory(g:vundle#bundle_dir) | call mkdir(g:vundle#bundle_dir, 'p') | endif
 
   let n = substitute(a:name,"['".'"]\+','','g')
   let matched = filter(copy(g:vundle#bundles), 'v:val.name_spec == n')
 
   if len(matched) > 0
-    let b = matched[0]
+    let bundle = matched[0]
   else
-    let b = vundle#config#init_bundle(a:name, {})
+    let bundle = vundle#config#init_bundle(a:name, {})
   endif
 
-  return s:sync(a:bang, b)
+  return bundle
 endf
 
-
 " ---------------------------------------------------------------------------
 " Call :helptags for all bundles in g:vundle#bundles.
 "
@@ -390,7 +529,7 @@ func! s:make_sync_command(bang, bundle) abort
       let cmd_parts = [
                   \ 'cd '.vundle#installer#shellesc(a:bundle.path()) ,
                   \ 'git remote set-url origin ' . vundle#installer#shellesc(a:bundle.uri),
-                  \ 'git fetch',
+                  \ 'git fetch' . (g:vundle#shallow_copy == 1 ? ' --depth 1' : ''),
                   \ 'git reset --hard origin/HEAD',
                   \ 'git submodule update --init --recursive',
                   \ ]
@@ -406,16 +545,19 @@ func! s:make_sync_command(bang, bundle) abort
     endif
 
     let cmd_parts = [
-                \ 'cd '.vundle#installer#shellesc(a:bundle.path()),
-                \ 'git pull',
+                \ 'cd ' . vundle#installer#shellesc(a:bundle.path()),
+                \ 'git pull' . (g:vundle#shallow_copy == 1 ? ' --depth 1' : ''),
                 \ 'git submodule update --init --recursive',
                 \ ]
     let cmd = join(cmd_parts, ' && ')
     let cmd = vundle#installer#shellesc_cd(cmd)
 
     let initial_sha = s:get_current_sha(a:bundle)
   else
-    let cmd = 'git clone --recursive '.vundle#installer#shellesc(a:bundle.uri).' '.vundle#installer#shellesc(a:bundle.path())
+    let cmd = 'git clone --recursive ' .
+          \ (g:vundle#shallow_copy == 1 ? '--depth 1 ' : '') .
+          \ vundle#installer#shellesc(a:bundle.uri) . ' ' .
+          \ vundle#installer#shellesc(a:bundle.path())
     let initial_sha = ''
   endif
   return [cmd, initial_sha]