Merge pull request tompollard#100 from JamesOwers/users/jfowers/fix-p…

…andoc-citeproc

Fix citations and travis build

.gitignore

@@ -1,3 +1,5 @@
+.vscode/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.travis.yml

@@ -9,35 +9,117 @@ branches:
 # Set the language
 language: bash
 
-os: osx
+os:
+  - osx
+  # - linux  # https://tex.stackexchange.com/questions/313768/why-getting-this-error-tlmgr-unknown-directive
 
-env:
-  global:
-    - PATH=$TRAVIS_BUILD_DIR/usr/bin/:$PATH
-    - PATH=/Library/TeX/texbin:$PATH
+# env:
+  # global:
+    # - PATH=$TRAVIS_BUILD_DIR/usr/bin/:$PATH
+    # - PATH=/Library/TeX/texbin:$PATH
 
+python:
+  # We don't actually use the Travis Python, but this keeps it organized.
+  # - "2.7"
+  # - "3.5"
+  - "3.6"
+  - "3.7"
+  - "3.8"
+
+addons:
+  apt:
+    packages:
+    - texlive
+    - xzdec
+    update: true
+  homebrew:
+    casks:
+    - mactex
+    update: true
 
 before_install:
-  # # Fetch pandoc and pandoc-citeproc binaries
-  # - wget https://github.com/jgm/pandoc/releases/download/1.15.0.5/pandoc-1.15.0.5-1-amd64.deb && ar p pandoc-1.15.0.5-1-amd64.deb data.tar.gz | tar zx
-  # # Install TeX Live
-  # - sudo add-apt-repository -y ppa:texlive-backports/ppa
-  # - sudo apt-get update -qq
-  # - sudo apt-get install texlive-xetex texlive-latex-recommended texlive-latex-extra
-  # - sudo apt-get install texlive-fonts-recommended texlive-fonts-extra
-  # - sudo apt-get install texlive-science
-  # # Install fonts
-  # - sudo apt-get install lmodern
-  - brew install pandoc
-  - brew install pandoc-citeproc
-  - brew cask install mactex
-  - tlmgr info fontspec
-  - sudo pip3 install pandoc-fignos pandoc-eqnos pandoc-tablenos \
-            pandoc-secnos pandoc-shortcaption
-
-# Attempt to create a PDF
+  # For debug
+  - echo $PATH
+  - |
+    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+      eval "$(/usr/libexec/path_helper)"
+      sudo tlmgr update --self
+    elif [ "$TRAVIS_OS_NAME" = "linux" ]; then
+      tlmgr init-usertree
+      echo MSG FROM JO:
+      echo on linux we cannot update apt installed texlive
+      echo next steps to fix see commit msg: https://github.com/tompollard/phd_thesis_markdown/pull/100/commits/85fd9e7ac413a34066b79f1e49b3e1d3efdeac00
+      echo c.f. https://tex.stackexchange.com/questions/551383/cant-run-tex-lives-tlmgr-in-a-github-action
+      echo and https://tex.stackexchange.com/questions/1092/how-to-install-vanilla-texlive-on-debian-or-ubuntu
+
+    fi
+  # Install conda
+  - |
+    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+      OS_NAME=MacOSX
+    elif [ "$TRAVIS_OS_NAME" = "linux" ]; then
+      OS_NAME=Linux
+    fi
+  - |
+    if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+      wget https://repo.continuum.io/miniconda/Miniconda2-latest-${OS_NAME}-x86_64.sh -O miniconda.sh
+    else
+      wget https://repo.continuum.io/miniconda/Miniconda3-latest-${OS_NAME}-x86_64.sh -O miniconda.sh
+    fi
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - source "$HOME/miniconda/etc/profile.d/conda.sh"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  # Useful for debugging any issues with conda
+  - conda info -a
+  # The install of mactex...takes a long time: ~20mins
+  # investigate using a docker image to save time?:
+  #     https://tex.stackexchange.com/questions/398830/how-to-build-my-latex-automatically-using-travis-ci
+  # This is now done in the 'addons:' travis config above
+  # - |
+  #   if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+  #     brew update && brew upgrade
+  #     brew install --cask mactex
+  #     sudo tlmgr update --self
+  #   elif [ "$TRAVIS_OS_NAME" = "linux" ]; then
+  #     sudo apt-get install texlive
+  #   fi
+  # For debug
+  - echo $PATH
+
+install:
+  # Make python venv
+  - conda create -n phd python=${PYTHON_VERSION} pandoc
+  - conda activate phd
+  # Installs both python and texlive dependencies
+  - make install
+  # For debug
+  - cat /usr/local/texlive/2020/texmf-var/web2c/tlmgr.log
+  - conda list
+  - pip list
+  - python --version
+  - pandoc --version
+  - tlmgr --version
+  - latex --version
+  - echo $PATH
+  - which tlmgr
+  - which latex
+  - find /usr/local/texlive/2020/ -type f -name "l3backend*"
+
+# Attempt to create all document output types
 script:
+  - echo ========================== beginning testing script ==========================
   - make help
+  - make tex
+  - cat pandoc.tex.log
   - make pdf
-  - cat pandoc.log
+  - cat pandoc.pdf.log
+  # Debugging the mac error
+  - find /usr/local/texlive/2020/ -type f -name "l3backend*"
   - make html
+  - cat pandoc.html.log
+  - make docx
+  - cat pandoc.docx.log
+  - make all
+  - cat pandoc.*.log

Makefile

@@ -25,68 +25,96 @@ help:
 	@echo 'get local templates with: pandoc -D latex/html/etc'
 	@echo 'or generic ones from: https://github.com/jgm/pandoc-templates'
 
+ifeq ($(OS),Windows_NT) 
+	detected_OS=Windows
+else
+	detected_OS=$(shell sh -c 'uname 2>/dev/null || echo Unknown')
+endif
+
+UNAME := $(shell uname)
+ifeq ($(UNAME), Linux)
+install:
+	bash $(BASEDIR)/install_linux.sh
+else ifeq ($(shell uname), Darwin)
 install:
-	sh $(BASEDIR)/install.sh
+	bash $(BASEDIR)/install_mac.sh
+endif
 
 pdf:
 	pandoc  \
-	--filter=pandoc-shortcaption \
-	--filter=pandoc-xnos \
-	--template="$(STYLEDIR)/template.tex" \
-	"$(INPUTDIR)"/*.md \
-	"$(INPUTDIR)/metadata.yml" \
-	-o "$(OUTPUTDIR)/thesis.pdf" \
-	-H "$(STYLEDIR)/preamble.tex" \
-	--bibliography="$(BIBFILE)" 2>pandoc.log \
-	--csl="$(STYLEDIR)/ref_format.csl" \
-	-V fontsize=12pt \
-	-V papersize=a4paper \
-	-V documentclass=report \
-	--pdf-engine=xelatex \
-	--verbose
+		--output "$(OUTPUTDIR)/thesis.pdf" \
+		--template="$(STYLEDIR)/template.tex" \
+		--include-in-header="$(STYLEDIR)/preamble.tex" \
+		--variable=fontsize:12pt \
+		--variable=papersize:a4paper \
+		--variable=documentclass:report \
+		--pdf-engine=xelatex \
+		"$(INPUTDIR)"/*.md \
+		"$(INPUTDIR)/metadata.yml" \
+		--filter=pandoc-shortcaption \
+		--filter=pandoc-xnos \
+		--bibliography="$(BIBFILE)" \
+		--citeproc \
+		--csl="$(STYLEDIR)/ref_format.csl" \
+		--number-sections \
+		--verbose \
+		2>pandoc.pdf.log
 
 tex:
-	pandoc "$(INPUTDIR)"/*.md \
-	--filter=pandoc-shortcaption \
-	--filter=pandoc-xnos \
-	--template="$(STYLEDIR)/template.tex" \
-	"$(INPUTDIR)/metadata.yml" \
-	-o "$(OUTPUTDIR)/thesis.tex" \
-	-H "$(STYLEDIR)/preamble.tex" \
-	--bibliography="$(BIBFILE)" \
-	-V fontsize=12pt \
-	-V papersize=a4paper \
-	-V documentclass=report \
-	-N \
-	--csl="$(STYLEDIR)/ref_format.csl" \
-
-docx:
-	pandoc \
-	--filter=pandoc-shortcaption \
-	--filter=pandoc-xnos \
-	"$(INPUTDIR)"/*.md \
-	"$(INPUTDIR)/metadata.yml" \
-	-o "$(OUTPUTDIR)/thesis.docx" \
-	--bibliography="$(BIBFILE)" \
-	--csl="$(STYLEDIR)/ref_format.csl" \
-	--toc
+	pandoc  \
+		--output "$(OUTPUTDIR)/thesis.tex" \
+		--template="$(STYLEDIR)/template.tex" \
+		--include-in-header="$(STYLEDIR)/preamble.tex" \
+		--variable=fontsize:12pt \
+		--variable=papersize:a4paper \
+		--variable=documentclass:report \
+		--pdf-engine=xelatex \
+		"$(INPUTDIR)"/*.md \
+		"$(INPUTDIR)/metadata.yml" \
+		--filter=pandoc-shortcaption \
+		--filter=pandoc-xnos \
+		--bibliography="$(BIBFILE)" \
+		--citeproc \
+		--csl="$(STYLEDIR)/ref_format.csl" \
+		--number-sections \
+		--verbose \
+		2>pandoc.tex.log
 
 html:
-	pandoc "$(INPUTDIR)"/*.md \
-	"$(INPUTDIR)/metadata.yml" \
-	-o "$(OUTPUTDIR)/thesis.html" \
-	--filter=pandoc-shortcaption \
-	--filter=pandoc-xnos \
-	--standalone \
-	--template="$(STYLEDIR)/template.html" \
-	--bibliography="$(BIBFILE)" \
-	--csl="$(STYLEDIR)/ref_format.csl" \
-	--include-in-header="$(STYLEDIR)/style.css" \
-	--toc \
-	--number-sections
+	pandoc  \
+		--output "$(OUTPUTDIR)/thesis.html" \
+		--template="$(STYLEDIR)/template.html" \
+		--include-in-header="$(STYLEDIR)/style.css" \
+		--toc \
+		"$(INPUTDIR)"/*.md \
+		"$(INPUTDIR)/metadata.yml" \
+		--filter=pandoc-shortcaption \
+		--filter=pandoc-xnos \
+		--bibliography="$(BIBFILE)" \
+		--citeproc \
+		--csl="$(STYLEDIR)/ref_format.csl" \
+		--number-sections \
+		--verbose \
+		2>pandoc.html.log
 	rm -rf "$(OUTPUTDIR)/source"
 	mkdir "$(OUTPUTDIR)/source"
 	cp -r "$(INPUTDIR)/figures" "$(OUTPUTDIR)/source/figures"
 
+docx:
+	pandoc  \
+		--output "$(OUTPUTDIR)/thesis.docx" \
+		--toc \
+		"$(INPUTDIR)"/*.md \
+		"$(INPUTDIR)/metadata.yml" \
+		--filter=pandoc-shortcaption \
+		--filter=pandoc-xnos \
+		--bibliography="$(BIBFILE)" \
+		--citeproc \
+		--csl="$(STYLEDIR)/ref_format.csl" \
+		--number-sections \
+		--verbose \
+		2>pandoc.docx.log
+
+all: pdf tex html docx
 
 .PHONY: help install pdf docx html tex

README.md

@@ -10,6 +10,23 @@ If you have used this template in your work, please cite the following publicati
 
 > Tom Pollard et al. (2016). Template for writing a PhD thesis in Markdown. Zenodo. http://dx.doi.org/10.5281/zenodo.58490
 
+## Quickstart
+If you're a mac user and you have conda and brew installed, run the following in your terminal to install and generate the example outputs:
+```bash
+# get texlive
+brew install --cask mactex
+
+# update tlmgr and packages
+sudo tlmgr update --self
+
+# make python venv and install pandoc
+conda create -n phd -y python=3.7 pandoc
+conda activate phd
+
+# Install required python and texlive packages
+make install
+```
+
 ## Why write my thesis in Markdown?
 
 Markdown is a super-friendly plain text format that can be easily converted to a bunch of other formats like PDF, Word and LaTeX. You'll enjoy working in Markdown because:
@@ -51,15 +68,48 @@ There are some minor annoyances:
     - Pandoc plugins by running ```make install```
     - Git, for version control.
 2. [Fork the repository](https://github.com/tompollard/phd_thesis_markdown/fork) on Github  
-3. Clone the repository onto your local computer (or [download the Zip file](https://github.com/tompollard/phd_thesis_markdown/archive/master.zip)).  
-4. Navigate to the directory that contains the Makefile and type "make pdf" (or "make html") at the command line to update the PDF (or HTML) in the output directory.  
+3. Clone the repository onto your local computer (or [download the Zip file](https://github.com/tompollard/phd_thesis_markdown/archive/master.zip)).
+4. (Skip this step to use default UCL style) Configure style for your institution - see instructions below
+5. Navigate to the directory that contains the Makefile and type "make pdf" (or "make html") at the command line to update the PDF (or HTML) in the output directory.  
 **In case of an error** (e.g. `make: *** [pdf] Error 43`), consult [this article](https://dalwilliams.info/lessons-learned-from-writing-a-phd-dissertation-in-markdown.html) for possible fixes. Most importantly, make sure tlmgr is properly installed, then run ```install.sh``
-
-5. Edit the files in the 'source' directory, then goto step 4.  
+6. Edit the files in the 'source' directory, then goto step 5.
+
+## How does it work?
+The universal document converter [`pandoc`](https://pandoc.org/) does all the heavy lifting. For example:
+
+1. `make pdf` (the code under `pdf: ...` in [`Makefile`](./Makefile)) runs `pandoc` which takes as input
+    1. the markdown files which contain the writing content: `input/*.md`
+    1. a yaml file with metadata: `input/metadata.yml`
+    1. a LaTeX template: `style/template.tex`
+    1. a LaTeX header: `style/preamble.tex`
+    1. a BibTeX file of your references: `input/references.bib`
+    1. a csl style file for citations: `style/ref_format.csl`
+    1. a bunch of options which change the output e.g. `--number-sections`
+1. the output produced is:
+    1. the generated pdf: [`output/thesis.pdf`](./output/thesis.pdf)
+    1. logs (which contain the `.tex` which was compiled): `pandoc.pdf.log`    
+
+Put simply, `pandoc` uses the latex template provided to create a `.tex` file, then compiles it. In detail, `pandoc` processes the input files in the following way (the file names in quotes aren't visible to you, but are named for the purpose of understanding):
+1. Make replacements within the markdown files `input/*.md` e.g.:
+    * references to figures, captions, and sections are handled: `@fig:my_fig` -> `\ref{fig:my_fig}`
+    * equations are converted to LaTeX and numbered: `$f(x) = ax^3 + bx^2 + cx + d$ {#eq:my_equation}` -> `\begin{equation}f(x) = ax^3 + bx^2 + cx + d\label{eq:my_equation}\end{equation}`
+    * citations are handled: `[@Cousteau1963]` -> `(Cousteau Jacques & Dugan James 1963)`
+    * see `input/*.md` for more examples!
+1. Create "`body.tex`" by:
+    * converting all the `*.md` files **in the order that they were stated** in the `pandoc` call
+1. Create "`main.tex`" from `style/template.tex` by running code wrapped in `$` signs. The important things to note are:
+    * this populates `style/template.tex` with metadata from `input/metadata.yml` and the arguments from the `pandoc` call
+    * "`body.tex`" is pasted in verbatim in place of `$body$`
+1. Create "`references.tex`" by converting `./input/references.bib`
+1. Concatenate files together to create the final `thesis.tex` = `style/preamble.tex` + "`main.tex`" + "`references.tex`"
+1. Compile `thesis.tex` (you can see the logs for this process, and what "`thesis.tex`" would look like in `pandoc.pdf.log`)
+    * **TIP**: You can also generate and view `output/thesis.tex` by running `make tex` - this follows all the above steps, bar the final compilation
 
 ## What else do I need to know?
 
 Some useful points, in a random order:
+- if you only care about generating `theis.pdf` you can always fall back on writing LaTeX within the markdown files (but note that `theis.html` and other outputs will not be able to render this)
+- the markdown files you write (i.e. your chapters) will be compiled in alphabetical order, so keep the filenames sorted in the order you want them to appear e.g. `01_chapter_1.md`, `02_chapter_2.md`, etc. This is required because of the way we have written `make pdf`. You can change this behaviour by writing a custom `pandoc` command instead of using `make pdf`.
 - each chapter must finish with at least one blank line, otherwise the header of the following chapter may not be picked up.
 - add two spaces at the end of a line to force a line break.
 - the template uses [John Macfarlane's Pandoc](http://johnmacfarlane.net/pandoc/README.html) to generate the output documents. Refer to this page for Markdown formatting guidelines.
@@ -68,6 +118,22 @@ Some useful points, in a random order:
 - For fellow web developers, there is a Grunt task file (Gruntfile.js) which can be used to 'watch' the markdown files. By running `$ npm install` and then `$ npm run watch` the PDF and HTML export is done automatically when saving a Markdown file.
 - You can automatically reload the HTML page on your browser using LiveReload with the command `$ npm run livereload`. The HTML page will automatically reload when saving a Markdown file after the export is done.
 
+# Troubleshooting
+1. The first thing to try if the `make *` command fails is a simpler build, e.g. if `make pdf` failed, try `make tex` to see if that fails too.
+2. If tex compilation is failing (i.e. `make tex` works but `make pdf` fails), try updating tex live and/or packages. For example, if you get the error `make: *** [pdf] Error 43`, have a look in `pandoc.pdf.log` for the error. If it is something like
+```
+`l3backend-xdvipdfmx.def' not found
+```
+then try:
+```bash
+sudo tlmgr update --self
+sudo tlmgr l3backend
+# Full nuclear option - update *all* the packages! (takes about 10m)
+# sudo tlmgr update --all
+```
+3. Try reinstalling everything from scratch (tip: check out [`.travis.yml`](./.travis.yml))
+3. Search the [github issues](https://github.com/tompollard/phd_thesis_markdown/issues) and [pull requests](https://github.com/tompollard/phd_thesis_markdown/pulls) in this repo
+
 # Contributing
 
 Contributions to the template are encouraged! There are lots of things that could be improved, like: