release

#!/usr/bin/env sos-runner
#fileformat=SOS1.0
import os, sys, glob

parameter: root = path('docs')
parameter: pip = 'pip'
parameter: overwrite_rmd = False
parameter: hide_code = False
sys.path.append(f'{root:a}')

[default_1 (Configure website): shared = ['conf', 'binders', 'notebook_files', 'rmd_files', 'index_files', 'pipeline_files', 'generated_files', 'password', 'status']]
depends: Py_Module('bs4', autoinstall=True), Py_Module('date_parser', autoinstall=True)
DEFAULT_CONF = {
           '__version__': '0.9.7',
           '__sos_version__': '0.19.12',
           '__about_commit__': 'https://stephenslab.github.io/ipynb-website/notes.html#Note-about-commit-ids',
           'name': 'A Pet Project',
           'repo': 'http://github.com/vatlab/jnbinder',
           'footer': "&copy 2016-2017 Gao Wang at Stephens Lab, University of Chicago",
           'include_dir': [],
           'exclude_file': [],
           'hide_navbar': [],
           'disqus': None,
           'report_style': False,
           'release_message': None,
           'post_release_actions': None,
           'homepage_label': "Overview",
           'source_label': '<span class="fa fa-github"></span>',
           'theme': 'cosmo',
           'homepage': 'Overview.ipynb',
           'skip_homepage': False,
           'font': "Droid Sans",
           'notebook_toc': True,
           'add_commit_info': True,
           'long_autoindex': True,
           'reverse_autoindex': [],
           'remove_whiteline': False,
           'auto_highlight': None,
           'password_file': None,
           'site_url': None
           }
status = 0
try:
  from jnbinder import compare_versions, update_gitignore, get_base_link
except Exception as e:
  status = 1
  if 'jnbinder' in str(e):
     sos_run('setup-jnbinder')
     fail_if(status, msg = f"``jnbinder`` program has just been installed to ``{root}`` but you need to configure ``config.yml`` and rerun this program.")
  else:
     raise RuntimeError('{}. You can try `pip install` to install the missing module.'.format(e))
if os.path.isdir(".git"):
   update_gitignore()
if compare_versions(SOS_VERSION, DEFAULT_CONF["__sos_version__"]) < 0:
  sos_run("upgrade-sos")
  status = 1
  fail_if(status, msg = 'Cannot generate documents due to obsolete SoS software. Though we have attempted to upgrade SoS (see above) you need to rerun this script to load the updates.')
#
err_msg = '''
No valid path found in ``include_dir``
Please properly set ``"include_dir"`` in ``config.yml``
'''
import yaml
with open('config.yml', 'r') as f:
    conf = yaml.safe_load(f)
fail_if(conf is None, msg = err_msg)
conf.update({k:v for k,v in DEFAULT_CONF.items() if k not in conf})
conf['jt_theme'] = conf['theme'] if os.path.isfile(f"{root}/css/{conf['theme']}.css") else None
for k in ['include_dir', 'hide_navbar', 'exclude_file', 'reverse_autoindex']:
    conf[k] = [os.path.normpath(x) for x in conf[k]]
for item in conf['include_dir']:
    if os.path.basename(item) in ['css', 'fonts', 'js', 'site_libs', root]:
       status = 1
       fail_if(stats, msg = f'Folder name ``{item}`` conflicts with website library paths. Please use a different name.')
conf['remove_whiteline'] = 1 if conf['remove_whiteline'] else 0
if conf['site_url'] is None:
   conf['site_url'] = f"http://{path(conf['repo']):db}.github.io/{path(conf['repo']):b}"
if conf['auto_highlight'] is None:
   conf['auto_highlight'] = ('pack', 'null')
else:
   if os.path.isfile(f'{root}/site_libs/highlightjs/highlight.{conf["auto_highlight"]}.js'):
      conf['auto_highlight'] = (conf['auto_highlight'], 'jnbinder')
   else:
      conf['auto_highlight'] = ('pack', 'jnbinder')
binders = [x for x in conf['include_dir'] if os.path.isdir(x)]
# binders = [x for x in next(os.walk('./'))[1] if x not in conf['exclude_dir'] + [root] and not x.startswith('.')]
fail_if(len(binders) == 0, msg = err_msg)
rmd_files = sum([[y for y in glob.glob(f"{x}/**/*.Rmd", recursive=True) if y not in conf['exclude_file']] for x in binders], [])
if conf['homepage'].endswith('.Rmd') or conf['homepage'].endswith('.md'):
  rmd_files.append(conf['homepage'])
  conf['homepage']= os.path.splitext(conf['homepage'])[0] + '.ipynb' 
notebook_files = list(set(sum([[y for y in glob.glob(f"{x}/**/*.ipynb", recursive=True) if y not in [f"{x}/index.ipynb", f"{x}/_index.ipynb", conf['homepage']] + conf['exclude_file']] for x in binders], []) + [os.path.splitext(x)[0] + '.ipynb' for x in rmd_files if x.endswith('Rmd')]))
index_files = [f'{x}/index.ipynb' if os.path.isfile(f'{x}/index.ipynb') else f'{x}/_index.ipynb' for x in binders if x not in conf['hide_navbar']]
pipeline_files = sum([[y for y in glob.glob(f"{x}/**/*.sos", recursive=True) if y not in conf['exclude_file']] for x in binders], [])
generated_files = [f'{root}/{x}.html' for x in binders if x not in conf['hide_navbar']] + ([f'{root}/index.html'] if not conf['skip_homepage'] else []) + [f'{root}/{get_base_link(x)}' for x in notebook_files] + [f'{root}/{get_base_link(x, "pipeline.html")}' for x in pipeline_files] + [f'{root}/js/docs.js', f'{root}/index.tpl'] + [f"{root}/{x}.tpl" for x in binders]
if conf['password_file'] is not None and os.path.isfile(conf['password_file']):
   with open(conf['password_file']) as f:
        password = {os.path.normpath(k): v for k, v in yaml.safe_load(f).items()}
   for key in password:
       if not os.path.isdir(key) and not os.path.isfile(key):
          status = 1
          fail_if(status, msg = f"Cannot find ``{key}``. Please comment out or remove it from ``{conf['password_file']}``")
else:
   password = None
conflicting_notebooks = [os.path.splitext(item)[0] + '.ipynb' for item in rmd_files if os.path.isfile(os.path.splitext(item)[0] + '.ipynb')]
fail_if(len(conflicting_notebooks) and not overwrite_rmd, msg = f"Rmarkdown files conflicts with existing notebooks ``{' '.join(conflicting_notebooks)}``. If the notebook was automatically generated you can either remove them, or use ``--overwrite-rmd`` to allow overwritting them.")
stop_if(status)
err_msg = '''
To properly add in version info, all changes made to source notebooks must be committed.
Please use ``git commit`` to commit changes and run this command again. Or you may
set ``add_commit_info: False`` in ``config.yml`` if you want to disable this feature.
'''
changed_list = get_output('git status --untracked-files=no --porcelain')
fail_if(os.path.isdir('.git') and ('ipynb' in changed_list or 'Rmd' in changed_list) and conf['add_commit_info'], msg = err_msg)

[download-jnbinder: provides = 'jnbinder-master/README.md']
download: decompress = True, dest_file = 'jnbinder-master.zip'
  https://github.com/vatlab/jnbinder/archive/master.zip

[setup-jnbinder]
depends: 'jnbinder-master/README.md'
run: expand = True
  mkdir -p {root}
  yes | cp -rf jnbinder-master/docs/* {root}/
  cp jnbinder-master/config.yml {'config.default.yml' if os.path.isfile('config.yml') else 'config.yml'}
  rm -rf jnbinder-master*

[upgrade-jnbinder]
depends: 'jnbinder-master/README.md'
bash: expand = "${ }"
  mkdir -p ${root}
  rm -rf ${root}/{css,fonts,jnbinder.py,js,site_libs}
  cp -r jnbinder-master/${root}/{css,fonts,jnbinder.py,js,site_libs} ${root}
  cp jnbinder-master/jnbinder_docker.sh ./
  rm -rf jnbinder-master*

[upgrade-sos (SoS upgrader)]
# using latest pip release
try:
  get_output(f'{pip} install --upgrade --upgrade-strategy only-if-needed --no-cache-dir sos')
except:
  get_output(f'{pip} install --upgrade --upgrade-strategy only-if-needed --no-cache-dir --user sos')

[check-link (Broken link checker)]
depends: executable('linkchecker'), sos_variable('conf')
run: expand = True
  linkchecker {conf['site_url']}

[update-tpl (Jupyter HTML template updater)]
parameter: binders = list
parameter: conf = dict
input: None
output: [f"{root}/{x}.tpl" for x in binders] + [f'{root}/index.tpl']
from jnbinder import make_template
make_template(conf, binders, root)

[update-index (Index notebook generator)]
parameter: binders = list
parameter: notebook_files = list
parameter: conf = dict
stop_if(len(notebook_files) == 0)
input: notebook_files
output: [binder + '/_index.ipynb' for binder in binders]
from jnbinder import make_index_nb, make_empty_nb
for binder in binders:
  content = make_index_nb(binder, conf['exclude_file'] + [conf['homepage']], conf['long_autoindex'], binder in conf['reverse_autoindex'])
  with open(f'{binder}/_index.ipynb', 'w') as f:
    f.write(content)
if not os.path.isfile(conf['homepage']):
  with open(conf['homepage'], 'w') as f:
    f.write(make_empty_nb(conf['name']))

[update-hp (Index HTML updater)]
parameter: index_files = list
parameter: homepage = str
parameter: skip_homepage = bool
parameter: hide_navbar = bool
parameter: remove_whiteline = int
parameter: password = str
depends: executable('jupyter'), executable('sed'), executable('perl'), Py_Module('jupyter_contrib_nbextensions', autoinstall=True), path(homepage).with_suffix('.ipynb')
from jnbinder import get_sha1_files
input_files = index_files + ([path(homepage).with_suffix('.ipynb')] if not skip_homepage else [])
output_files = [f'{root}/{x}.html' for x in binders if x not in hide_navbar] + ([f'{root}/index.html'] if not skip_homepage else [])
input: dynamic(input_files), group_by = 1
output: output_files, group_by = 1
bash: expand = "${ }", stderr = False
  jupyter nbconvert ${_input:e} --output ${_output:ae} --template ${root}/index.tpl ${"--no-input" if hide_code else ""}
  # sed -i 's/<div class="prompt input_prompt">In&nbsp;\[[0-9]\]:<\/div>//g' ${_output:e}
  if [[ ${remove_whiteline} -gt 0 ]]; then
     perl -i -ne 'print if /\S/' ${_output:e}
  fi

get_sha1_files([(_input, _output)], [], password, write=True, docs_dir=root)

[update-nb (Notebook to HTML converter)]
parameter: notebook_files = list
parameter: nb_info = list
parameter: remove_whiteline = int
parameter: footer = str
parameter: password = str
stop_if(len(notebook_files) == 0)
depends: executable('jupyter'), executable('sed'), executable('perl'), Py_Module('jupyter_contrib_nbextensions', autoinstall=True), paths(notebook_files)
from jnbinder import get_base_link, get_sha1_files
input: notebook_files, group_by = 1, paired_with = ['nb_info']
output: [f"{root}/{get_base_link(x)}" for x in notebook_files], group_by = 1
bash: expand = "${ }", stderr = False
  jupyter nbconvert ${_input:e} --output ${_output:ae} --template ${root}/${str(_input).split(os.path.sep)[0]}.tpl ${"--no-input" if hide_code else ""}
  num=`grep -n -Fx '''${footer}''' ${_output:e} | tail -1 | sed 's/\([0-9]*\).*/\1/'`
  perl -i -lpe 'print "${_nb_info[0]}" if $. == '"$num"'+1' ${_output:e}
  if [[ ${remove_whiteline} -gt 0 ]]; then
     perl -i -ne 'print if /\S/' ${_output:e}
  fi

get_sha1_files([], [(_input, _output)], password, write=True, docs_dir=root)

[update-wf (Pipeline to HTML converter)]
parameter: pipeline_files = list
stop_if(len(pipeline_files) == 0)
from jnbinder import get_base_link
input: pipeline_files, group_by = 1
output: [f"{root}/{get_base_link(x, 'pipeline.html')}" for x in pipeline_files], group_by = 1
run: expand = "${ }", stdout = False, stderr = False
  sos convert ${_input} ${_output}

[update-toc (TOC Javascript updater)]
parameter: notebook_files = list
parameter: pipeline_files = list
parameter: index_files = list
parameter: binders = list
parameter: exclude_file = list
parameter: homepage = str
input: dynamic(notebook_files + pipeline_files + index_files)
output: f'{root}/js/docs.js'
from jnbinder import get_toc
out = [get_toc(x, exclude_file + [homepage]) for x in binders]
with open(_output, 'w') as f:
  f.write('\n'.join(['\n'.join(x) for x in out]))

[update-search (Search Javescript updater)]
parameter: generated_files = list
parameter: url = str
input: generated_files
output: f'{root}/site_libs/tipuesearch/tipuesearch_content.js'
from jnbinder import generate_tipue_content
generate_tipue_content(generated_files, url, f'{root}/')
with open(f'{root}/search.html', 'w') as f:
  f.write(open(f'{root}/site_libs/tipuesearch/search.html').read())

[remove-obsolete (Obsolete files remover)]
parameter: conf = dict()
parameter: index_files = list
parameter: notebook_files = list
parameter: pipeline_files = list
parameter: binders = list
parameter: generated_files = list
parameter: password = str
output: f'{root}/MANIFEST'
from jnbinder import get_sha1_files, get_base_link
if os.path.isfile(_output):
  old_files = [x.strip() for x in open(_output).readlines()]
  all_f1 = [(x, y) for x, y in zip(index_files + [conf['homepage']], [f'{root}/{x}.html' for x in binders if x not in conf['hide_navbar']] + ([f'{root}/index.html'] if not conf['skip_homepage'] else []))]
  all_f2 = [(x, y) for x, y in zip(notebook_files,  [f'{root}/{get_base_link(x)}' for x in notebook_files])]
  all_f3 = [(x, y) for x, y in zip(pipeline_files, [f'{root}/{get_base_link(x, "pipeline.html")}' for x in notebook_files])]

  sha1_files = [f"{root}/{item.rsplit('_', 1)[-1][:-5]}/{path(item.rsplit('_', 1)[0]):b}.html" if item.rsplit('_', 1)[-1][:-5] != root else f"{root}/{path(item.rsplit('_', 1)[0]):b}.html" for item in get_sha1_files(all_f1, all_f2 + all_f3, password, docs_dir=root)]
  for item in [x for x in old_files if not x in generated_files + sha1_files]:
    if item == f'{root}/index.html':
      continue
    try:
      os.remove(item)
    except OSError:
      pass
else:
  sha1_files = []
with open(_output, 'w') as f:
  f.write('\n'.join(sorted(generated_files + sha1_files)))

[release-website (gh-pages uploader)]
parameter: conf = dict()
stop_if(conf['release_message'] is None)
try:
  get_output(f'''cd {root} && git add . && git commit --no-verify -m "{conf['release_message']}" && git push --no-verify''')
  get_output(conf['post_release_actions'])
except:
  pass

[clean]
# remove generated files
depends: f'{root}/MANIFEST'
for item in [x.strip() for x in open(f'{root}/MANIFEST').readlines()]:
  try:
    os.remove(item)
  except OSError:
    pass

[default_2 (Rmarkdown to ipynb)]
depends: Py_Module("ipyrmd", autoinstall=True), R_library('IRkernel'), R_library('IRdisplay')
skip_if(len(rmd_files)==0)
input: rmd_files, group_by=1, concurrent=True
output:f'{_input:n}.ipynb'
run: expand = "${ }", stdout = False, stderr = False
  ipyrmd --to ipynb --from Rmd -y -o ${_output} ${_input}
  jupyter nbconvert --to notebook --execute ${_output} --ExecutePreprocessor.timeout=360 ${"--no-input" if hide_code else ""}
  mv ${_output:nn}.nbconvert.ipynb ${_output} 

[default_3 (Build website)]
input: group_by = 'all'
from jnbinder import get_commit_info
sos_run('update-index', conf=conf, binders=binders, notebook_files=notebook_files)
sos_run('update-tpl', conf=conf, binders=binders)
sos_run('update-hp', binders=binders, homepage=conf['homepage'], skip_homepage=conf['skip_homepage'], remove_whiteline=conf['remove_whiteline'], hide_navbar=conf['hide_navbar'], index_files=index_files, password=password)
sos_run('update-nb', nb_info=[get_commit_info(x[:-5] + 'Rmd' if path(x).with_suffix(".Rmd").exists() else x, conf) for x in notebook_files], remove_whiteline=conf['remove_whiteline'], footer=conf['footer'], notebook_files=notebook_files, password=password)
sos_run('update-wf', pipeline_files=pipeline_files)
sos_run('update-toc', binders=binders, notebook_files=notebook_files, index_files=index_files, pipeline_files=pipeline_files, homepage=conf['homepage'], exclude_file=conf['exclude_file'])
sos_run('update-search', url=conf['site_url'], generated_files=generated_files)
sos_run('remove-obsolete', conf=conf, binders=binders, notebook_files=notebook_files, pipeline_files=pipeline_files, index_files=index_files, generated_files=generated_files, password=password)
sos_run('release-website', conf=conf)

[default_4 (Clean up)]
stop_if(status)
hp = [path(conf["homepage"]).with_suffix(".ipynb")] if conf["homepage"].endswith(".Rmd") or conf["homepage"].endswith(".md") else []
for item in [binder + '/_index.ipynb' for binder in binders] + hp:
  try:
    os.remove(item)
  except OSError:
    pass