From 6e85d49b86b8c030c14b04cb6e1daa6cf4d98cd9 Mon Sep 17 00:00:00 2001 From: "Augusto F. Hack" Date: Sun, 26 Jul 2015 21:37:36 -0300 Subject: [PATCH 1/6] parallel download --- autoload/vundle.vim | 2 + autoload/vundle/installer.vim | 143 ++++++++++++++++++++++++++++++---- 2 files changed, 130 insertions(+), 15 deletions(-) diff --git a/autoload/vundle.vim b/autoload/vundle.vim index 148ff3ef..4e2dbd78 100644 --- a/autoload/vundle.vim +++ b/autoload/vundle.vim @@ -86,5 +86,7 @@ let vundle#bundles = [] let vundle#lazy_load = 0 let vundle#log = [] let vundle#updated_bundles = [] +let vundle#threads = 15 +let vundle#shallow_copy = 1 " vim: set expandtab sts=2 ts=2 sw=2 tw=78 norl: diff --git a/autoload/vundle/installer.vim b/autoload/vundle/installer.vim index 472271a3..fbf5e81b 100644 --- a/autoload/vundle/installer.vim +++ b/autoload/vundle/installer.vim @@ -23,18 +23,117 @@ func! vundle#installer#new(bang, ...) abort return endif - let names = vundle#scripts#bundle_names(map(copy(bundles), 'v:val.name_spec')) - call vundle#scripts#view('Installer',['" Installing plugins to '.expand(g:vundle#bundle_dir, 1)], names + ['Helptags']) - - " This calls 'add' as a normal mode command. This is a buffer local mapping - " defined in vundle#scripts#view(). The mapping will call a buffer local - " command InstallPlugin which in turn will call vundle#installer#run() with - " vundle#installer#install(). - call s:process(a:bang, (a:bang ? 'add!' : 'add')) + let specs = map(copy(bundles), 'v:val.name_spec') + let names = vundle#scripts#bundle_names(specs) + let headers = [ + \ '" Installing plugins to '.expand(g:vundle#bundle_dir, 1) + \ ] + call vundle#scripts#view('Installer', headers, names + ['Helptags']) + + if (has('python') || has('python3')) && g:vundle#threads > 1 + " This runs a Pool of threads to syncronize the bundles + call s:process_parallel(a:bang, specs, len(headers), g:vundle#threads) + else + " This calls 'add' as a normal mode command. This is a buffer local mapping + " defined in vundle#scripts#view(). The mapping will call a buffer local + " command InstallPlugin which in turn will call vundle#installer#run() with + " vundle#installer#install(). + call s:process(a:bang, (a:bang ? 'add!' : 'add')) + endif call vundle#config#require(bundles) endf +func! s:process_parallel(bang, specs, headers, threads) abort + let cmds = map(a:specs, 's:make_sync_command(a:bang, s:bundle_from_name(v:val))') + let python = has('python3') ? 'python3' : 'python' + + execute python "<< EOF" +import os +import subprocess +from multiprocessing import Pool + +import vim + +def main(cmds, shas, threads): + ''' execute cmds in parallel and update the ui ''' + iterable = Pool(threads).imap(sync, cmds) + iterable = ui(iterable, cmds, shas, len(cmds), threads) + + for result in iterable: + yield result + +def sync(cmd): + ''' run cmd discarding the output ''' + devnull = os.open(os.devnull, os.O_RDWR) + + if subprocess.mswindows: + import msvcrt + devnull = msvcrt.get_osfhandle(os.devnull) + + return subprocess.call( + cmd, + shell=True, + stdin=devnull, + stdout=devnull, + stderr=devnull, + ) + +def ui(iterable, cmds, shas, total, threads): + ''' compose the current running and the finished marks ''' + vim.command('redraw') + for result in status(active(iterable, threads, total), cmds, shas): + vim.command('redraw') + yield result + vim.command('redraw') + +def active(iterable, start, total): + ''' mark the bundles that are being downloaded ''' + for running in range(start): + mark('active', running) + + for result in iterable: + if running < total: + running += 1 + mark('active', running) + + yield result + +def status(iterable, cmds, shas): + ''' mark the status of the bundle ''' + for current, result in enumerate(iterable): + if not len(cmds[current]): + mark('todate', current) + elif result != 0: + mark('error', current) + elif not len(shas[current]): + mark('new', current) + # elif not len(shas[current]): + else: + mark('updated', current) + + yield result + +def mark(status, line): + ''' mark status on line ''' + + vim.command(''' + exec ':{line}' + call s:sign('{status}') + '''.format(line=1 + line + headers, status=status)) + +cmd_sha = vim.eval('cmds') +cmds = [cmd for cmd, sha in cmd_sha] +shas = [sha for cmd, sha in cmd_sha] + +threads = int(vim.eval('a:threads')) +headers = int(vim.eval('a:headers')) + +list(main(cmds, shas, threads)) + +EOF +endf + " --------------------------------------------------------------------------- " Iterate over all lines in a Vundle window and execute the given command for @@ -164,21 +263,32 @@ endf " return -- the return value from s:sync() " --------------------------------------------------------------------------- func! vundle#installer#install(bang, name) abort + let bundle = call s:bundle_from_name(name) + return s:sync(a:bang, bundle) +endf + + +" --------------------------------------------------------------------------- +" Creates bundle detiny directory and return a bundle dictionary. +" +" name -- the name_spec for the bundle +" return -- 'error' if an error occurred, else return 'helptags' +" --------------------------------------------------------------------------- +func! s:bundle_from_name(name) if !isdirectory(g:vundle#bundle_dir) | call mkdir(g:vundle#bundle_dir, 'p') | endif let n = substitute(a:name,"['".'"]\+','','g') let matched = filter(copy(g:vundle#bundles), 'v:val.name_spec == n') if len(matched) > 0 - let b = matched[0] + let bundle = matched[0] else - let b = vundle#config#init_bundle(a:name, {}) + let bundle = vundle#config#init_bundle(a:name, {}) endif - return s:sync(a:bang, b) + return bundle endf - " --------------------------------------------------------------------------- " Call :helptags for all bundles in g:vundle#bundles. " @@ -390,7 +500,7 @@ func! s:make_sync_command(bang, bundle) abort let cmd_parts = [ \ 'cd '.vundle#installer#shellesc(a:bundle.path()) , \ 'git remote set-url origin ' . vundle#installer#shellesc(a:bundle.uri), - \ 'git fetch', + \ 'git fetch' . g:vundle#shallow_copy == 1 ? '--depth 1' : '' \ 'git reset --hard origin/HEAD', \ 'git submodule update --init --recursive', \ ] @@ -407,7 +517,7 @@ func! s:make_sync_command(bang, bundle) abort let cmd_parts = [ \ 'cd '.vundle#installer#shellesc(a:bundle.path()), - \ 'git pull', + \ 'git pull' . g:vundle#shallow_copy == 1 ? '--depth 1' : '', \ 'git submodule update --init --recursive', \ ] let cmd = join(cmd_parts, ' && ') @@ -415,7 +525,10 @@ func! s:make_sync_command(bang, bundle) abort let initial_sha = s:get_current_sha(a:bundle) else - let cmd = 'git clone --recursive '.vundle#installer#shellesc(a:bundle.uri).' '.vundle#installer#shellesc(a:bundle.path()) + let cmd = 'git clone --recursive ' . + \ g:vundle#shallow_copy == 1 ? '--depth 1' : '' . + \ vundle#installer#shellesc(a:bundle.uri) . ' ' . + \ vundle#installer#shellesc(a:bundle.path()) let initial_sha = '' endif return [cmd, initial_sha] From 03d823854ca75f80abadd717e0a7d86211ae3750 Mon Sep 17 00:00:00 2001 From: "Augusto F. Hack" Date: Sun, 26 Jul 2015 22:43:53 -0300 Subject: [PATCH 2/6] added logging --- autoload/vundle/installer.vim | 45 ++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/autoload/vundle/installer.vim b/autoload/vundle/installer.vim index fbf5e81b..53b049b5 100644 --- a/autoload/vundle/installer.vim +++ b/autoload/vundle/installer.vim @@ -45,7 +45,7 @@ func! vundle#installer#new(bang, ...) abort endf func! s:process_parallel(bang, specs, headers, threads) abort - let cmds = map(a:specs, 's:make_sync_command(a:bang, s:bundle_from_name(v:val))') + let cmds = map(copy(a:specs), 's:make_sync_command(a:bang, s:bundle_from_name(v:val))') let python = has('python3') ? 'python3' : 'python' execute python "<< EOF" @@ -71,13 +71,19 @@ def sync(cmd): import msvcrt devnull = msvcrt.get_osfhandle(os.devnull) - return subprocess.call( - cmd, - shell=True, - stdin=devnull, - stdout=devnull, - stderr=devnull, - ) + try: + out = subprocess.check_output( + cmd, + shell=True, + stdin=devnull, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as error: + return (error.returncode, error.output) + except Excpetion as error: + return (-1, error.message) + + return (0, out) def ui(iterable, cmds, shas, total, threads): ''' compose the current running and the finished marks ''' @@ -101,10 +107,10 @@ def active(iterable, start, total): def status(iterable, cmds, shas): ''' mark the status of the bundle ''' - for current, result in enumerate(iterable): + for current, (code, out) in enumerate(iterable): if not len(cmds[current]): mark('todate', current) - elif result != 0: + elif code != 0: mark('error', current) elif not len(shas[current]): mark('new', current) @@ -112,7 +118,7 @@ def status(iterable, cmds, shas): else: mark('updated', current) - yield result + yield (code, out) def mark(status, line): ''' mark status on line ''' @@ -126,10 +132,17 @@ cmd_sha = vim.eval('cmds') cmds = [cmd for cmd, sha in cmd_sha] shas = [sha for cmd, sha in cmd_sha] +specs = vim.eval('a:specs') threads = int(vim.eval('a:threads')) headers = int(vim.eval('a:headers')) -list(main(cmds, shas, threads)) +for pos, (result, out) in enumerate(main(cmds, shas, threads)): + vim.command(''' + call s:log('') + call s:log('Plugin {spec}') + call s:log("{cmd}", '$ ') + call s:log("{out}", '> ') + '''.format(spec=specs[pos], cmd=cmds[pos], out=out)) EOF endf @@ -500,7 +513,7 @@ func! s:make_sync_command(bang, bundle) abort let cmd_parts = [ \ 'cd '.vundle#installer#shellesc(a:bundle.path()) , \ 'git remote set-url origin ' . vundle#installer#shellesc(a:bundle.uri), - \ 'git fetch' . g:vundle#shallow_copy == 1 ? '--depth 1' : '' + \ 'git fetch' . (g:vundle#shallow_copy == 1 ? ' --depth 1' : ''), \ 'git reset --hard origin/HEAD', \ 'git submodule update --init --recursive', \ ] @@ -516,8 +529,8 @@ func! s:make_sync_command(bang, bundle) abort endif let cmd_parts = [ - \ 'cd '.vundle#installer#shellesc(a:bundle.path()), - \ 'git pull' . g:vundle#shallow_copy == 1 ? '--depth 1' : '', + \ 'cd ' . vundle#installer#shellesc(a:bundle.path()), + \ 'git pull' . (g:vundle#shallow_copy == 1 ? ' --depth 1' : ''), \ 'git submodule update --init --recursive', \ ] let cmd = join(cmd_parts, ' && ') @@ -526,7 +539,7 @@ func! s:make_sync_command(bang, bundle) abort let initial_sha = s:get_current_sha(a:bundle) else let cmd = 'git clone --recursive ' . - \ g:vundle#shallow_copy == 1 ? '--depth 1' : '' . + \ (g:vundle#shallow_copy == 1 ? '--depth 1 ' : '') . \ vundle#installer#shellesc(a:bundle.uri) . ' ' . \ vundle#installer#shellesc(a:bundle.path()) let initial_sha = '' From c8a6537600284e42533393f3249479aa996a3dcf Mon Sep 17 00:00:00 2001 From: "Augusto F. Hack" Date: Tue, 11 Aug 2015 16:50:02 -0300 Subject: [PATCH 3/6] by default dont use shallow-copy --- autoload/vundle.vim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoload/vundle.vim b/autoload/vundle.vim index 4e2dbd78..fb8dd51b 100644 --- a/autoload/vundle.vim +++ b/autoload/vundle.vim @@ -87,6 +87,6 @@ let vundle#lazy_load = 0 let vundle#log = [] let vundle#updated_bundles = [] let vundle#threads = 15 -let vundle#shallow_copy = 1 +let vundle#shallow_copy = 0 " vim: set expandtab sts=2 ts=2 sw=2 tw=78 norl: From c84b871d8f3d056ae0b6fb7b95121f2449a30cf6 Mon Sep 17 00:00:00 2001 From: "Augusto F. Hack" Date: Sat, 29 Aug 2015 16:57:28 -0300 Subject: [PATCH 4/6] generating the docs --- autoload/vundle/installer.vim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/autoload/vundle/installer.vim b/autoload/vundle/installer.vim index 53b049b5..7e4a7928 100644 --- a/autoload/vundle/installer.vim +++ b/autoload/vundle/installer.vim @@ -33,6 +33,10 @@ func! vundle#installer#new(bang, ...) abort if (has('python') || has('python3')) && g:vundle#threads > 1 " This runs a Pool of threads to syncronize the bundles call s:process_parallel(a:bang, specs, len(headers), g:vundle#threads) + + " goto the last line and update the docs + exec ':+1' + call vundle#installer#run('vundle#installer#docs', 'helptags', []) else " This calls 'add' as a normal mode command. This is a buffer local mapping " defined in vundle#scripts#view(). The mapping will call a buffer local From dc561b4d8a78261d1508514efafc38d441043941 Mon Sep 17 00:00:00 2001 From: "Augusto F. Hack" Date: Wed, 10 Feb 2016 01:57:52 -0200 Subject: [PATCH 5/6] py3 compat --- autoload/vundle/installer.vim | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/autoload/vundle/installer.vim b/autoload/vundle/installer.vim index 7e4a7928..b98dd005 100644 --- a/autoload/vundle/installer.vim +++ b/autoload/vundle/installer.vim @@ -55,10 +55,19 @@ func! s:process_parallel(bang, specs, headers, threads) abort execute python "<< EOF" import os import subprocess +import sys from multiprocessing import Pool import vim +# bytes vs str +if sys.version_info < (3,): + def to_str(data): + return data +else: + def to_str(data): + return data.decode('utf8') # TODO: make sure utf8 is the right encoding + def main(cmds, shas, threads): ''' execute cmds in parallel and update the ui ''' iterable = Pool(threads).imap(sync, cmds) @@ -71,7 +80,7 @@ def sync(cmd): ''' run cmd discarding the output ''' devnull = os.open(os.devnull, os.O_RDWR) - if subprocess.mswindows: + if sys.platform == 'win32': import msvcrt devnull = msvcrt.get_osfhandle(os.devnull) @@ -87,7 +96,8 @@ def sync(cmd): except Excpetion as error: return (-1, error.message) - return (0, out) + # py3 returns bytes + return (0, to_str(out)) def ui(iterable, cmds, shas, total, threads): ''' compose the current running and the finished marks ''' @@ -132,6 +142,10 @@ def mark(status, line): call s:sign('{status}') '''.format(line=1 + line + headers, status=status)) +def clean(data): + ''' remove single quotes from the data to avoid escaping problems when using eval''' + return data.replace("'", '') + cmd_sha = vim.eval('cmds') cmds = [cmd for cmd, sha in cmd_sha] shas = [sha for cmd, sha in cmd_sha] @@ -141,12 +155,9 @@ threads = int(vim.eval('a:threads')) headers = int(vim.eval('a:headers')) for pos, (result, out) in enumerate(main(cmds, shas, threads)): - vim.command(''' - call s:log('') - call s:log('Plugin {spec}') - call s:log("{cmd}", '$ ') - call s:log("{out}", '> ') - '''.format(spec=specs[pos], cmd=cmds[pos], out=out)) + vim.command("call s:log('Plugin {spec}')".format(spec=clean(specs[pos]))) + vim.command("call s:log('{cmd}', '$ ')".format(cmd=clean(cmds[pos]))) + vim.command("call s:log('{out}', '> ')".format(out=clean(out))) EOF endf From 2d9d619a0dcf15c729ba62c7da6e56fed24efaaa Mon Sep 17 00:00:00 2001 From: "Augusto F. Hack" Date: Sat, 16 Jul 2016 00:23:16 -0300 Subject: [PATCH 6/6] fixed py3, subprocess error output also needs to be converted to str --- autoload/vundle/installer.vim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/autoload/vundle/installer.vim b/autoload/vundle/installer.vim index b98dd005..8eaca90a 100644 --- a/autoload/vundle/installer.vim +++ b/autoload/vundle/installer.vim @@ -84,6 +84,8 @@ def sync(cmd): import msvcrt devnull = msvcrt.get_osfhandle(os.devnull) + # py3 output is bytes, convert it before returning + try: out = subprocess.check_output( cmd, @@ -92,11 +94,10 @@ def sync(cmd): stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as error: - return (error.returncode, error.output) + return (error.returncode, to_str(error.output)) except Excpetion as error: - return (-1, error.message) + return (-1, to_str(error.message)) - # py3 returns bytes return (0, to_str(out)) def ui(iterable, cmds, shas, total, threads):