Adds server-side code block formatting

Wyam.sln

@@ -112,6 +112,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "integration", "integration"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wyam.Docs", "src\extensions\Wyam.Docs\Wyam.Docs.csproj", "{0EB71E59-8D5C-45F3-BF1D-8649BA76D139}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wyam.Highlight", "src\extensions\Wyam.Highlight\Wyam.Highlight.csproj", "{5FE758A2-1221-48FE-8A3C-8B3B9A32AFAB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wyam.Highlight.Tests", "tests\extensions\Wyam.Highlight.Tests\Wyam.Highlight.Tests.csproj", "{A6DE4586-1398-4071-90AB-5CCC1C949720}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -294,6 +298,14 @@ Global
 		{0EB71E59-8D5C-45F3-BF1D-8649BA76D139}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0EB71E59-8D5C-45F3-BF1D-8649BA76D139}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0EB71E59-8D5C-45F3-BF1D-8649BA76D139}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5FE758A2-1221-48FE-8A3C-8B3B9A32AFAB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5FE758A2-1221-48FE-8A3C-8B3B9A32AFAB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5FE758A2-1221-48FE-8A3C-8B3B9A32AFAB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5FE758A2-1221-48FE-8A3C-8B3B9A32AFAB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A6DE4586-1398-4071-90AB-5CCC1C949720}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A6DE4586-1398-4071-90AB-5CCC1C949720}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A6DE4586-1398-4071-90AB-5CCC1C949720}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A6DE4586-1398-4071-90AB-5CCC1C949720}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -350,5 +362,7 @@ Global
 		{2E5842F3-4797-4BB5-9A55-F3A005B434F8} = {596452BD-AE34-4989-A36F-88FF72B2DFD8}
 		{C5AED397-4463-4DE7-8A7E-74A4C949E417} = {596452BD-AE34-4989-A36F-88FF72B2DFD8}
 		{0EB71E59-8D5C-45F3-BF1D-8649BA76D139} = {5A431149-2B88-40C3-9717-CA6CCF214317}
+		{5FE758A2-1221-48FE-8A3C-8B3B9A32AFAB} = {5A431149-2B88-40C3-9717-CA6CCF214317}
+		{A6DE4586-1398-4071-90AB-5CCC1C949720} = {2E5842F3-4797-4BB5-9A55-F3A005B434F8}
 	EndGlobalSection
 EndGlobal

src/core/Wyam.Configuration/KnownExtensionGenerated.cs

@@ -14,6 +14,7 @@ public partial class KnownExtension
 		public static readonly KnownExtension Feeds = new KnownExtension("Wyam.Feeds");
 		public static readonly KnownExtension Git = new KnownExtension("Wyam.Git");
 		public static readonly KnownExtension GitHub = new KnownExtension("Wyam.GitHub");
+		public static readonly KnownExtension Highlight = new KnownExtension("Wyam.Highlight");
 		public static readonly KnownExtension Html = new KnownExtension("Wyam.Html");
 		public static readonly KnownExtension Images = new KnownExtension("Wyam.Images");
 		public static readonly KnownExtension Json = new KnownExtension("Wyam.Json");

src/extensions/Wyam.Highlight/Highlight.cs

@@ -0,0 +1,182 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using AngleSharp.Dom.Html;
+using AngleSharp.Extensions;
+using AngleSharp.Parser.Html;
+using MsieJavaScriptEngine;
+using Wyam.Common.Documents;
+using Wyam.Common.Execution;
+using Wyam.Common.Modules;
+using Wyam.Common.Tracing;
+using Wyam.Common.Util;
+
+namespace Wyam.Highlight
+{
+    /// <summary>
+    /// Applies syntax highlighting to code blocks
+    /// </summary>
+    /// <remarks>
+    /// <para>This module finds all &lt;pre&gt; &lt;code&gt; blocks and applies HighlightJs's syntax highlighting.</para>
+    /// </remarks>
+    ///  <example>
+    /// <code>
+    /// Pipelines.Add("Highlight",
+    ///     ReadFiles("*.html"),
+    ///     Highlight(),
+    ///     WriteFiles(".html")
+    /// );
+    /// </code>
+    /// </example>
+    /// <category>Content</category>
+    public class Highlight : IModule
+    {
+        private string _codeQuerySelector = "pre code";
+        private string _highlightJsFile;
+        private bool _escapeAt = true;
+        private bool _warnOnMissingLanguage = true;
+
+        /// <summary>
+        /// Sets the query selector to use to find code blocks.
+        /// </summary>
+        /// <param name="querySelector">The query selector to use to select code blocks. The default value is pre code</param>
+        /// <returns>The current instance.</returns>
+        public Highlight WithCodeQuerySelector(string querySelector)
+        {
+            _codeQuerySelector = querySelector;
+            return this;
+        }
+
+        /// <summary>
+        /// Sets whether a warning should be raised if a missing language is detected in a code block.
+        /// </summary>
+        /// <param name="warnOnMissingLanguage">if set to <c>true</c> [warn on missing].</param>
+        /// <returns>The current instance.</returns>
+        public Highlight WithMissingLanguageWarning(bool warnOnMissingLanguage = true)
+        {
+            _warnOnMissingLanguage = warnOnMissingLanguage;
+            return this;
+        }
+
+        /// <summary>
+        /// Sets the file path to a custom highlight.js file. If not set the embeded version will be used.
+        /// </summary>
+        /// <param name="filePath">The file path.</param>
+        /// <returns>The current instance.</returns>
+        public Highlight WithCustomHighlightJs(string filePath)
+        {
+            _highlightJsFile = filePath;
+            return this;
+        }
+
+        /// <summary>
+        /// Specifies whether the <c>@</c> symbol should be escaped (the default is <c>true</c>).
+        /// This is important if the highlighted documents are going to be passed to the Razor module,
+        /// otherwise the Razor processor will interpret the unescaped <c>@</c> symbols as code
+        /// directives.
+        /// </summary>
+        /// <param name="escapeAt">If set to <c>true</c>, <c>@</c> symbols are HTML escaped.</param>
+        /// <returns>The current instance.</returns>
+        public Highlight WithEscapedAt(bool escapeAt = true)
+        {
+            _escapeAt = escapeAt;
+            return this;
+        }
+
+        public IEnumerable<IDocument> Execute(IReadOnlyList<IDocument> inputs, IExecutionContext context)
+        {
+            HtmlParser parser = new HtmlParser();
+            ThreadLocal<MsieJsEngine> jsEngines = new ThreadLocal<MsieJsEngine>(BuildJsEngineWithHighlight, true);
+
+            return inputs.AsParallel().Select(context, input =>
+            {
+                MsieJsEngine engine = jsEngines.Value;
+
+                try
+                {
+                    using (Stream stream = input.GetStream())
+                    using (IHtmlDocument htmlDocument = parser.Parse(stream))
+                    {
+                        foreach (AngleSharp.Dom.IElement element in htmlDocument.QuerySelectorAll(_codeQuerySelector))
+                        {
+                            // don't highlight anything that potentially is already highlighted
+                            if (element.ClassList.Contains("hljs"))
+                            {
+                                continue;
+                            }
+
+                            try
+                            {
+                                // make sure to use TextContent, otherwise you'll get escaped html which 
+                                // highlightjs won't parse
+                                engine.SetVariableValue("input", element.TextContent);
+
+                                // check if they specified a language in their code block
+                                string language = element.ClassList.FirstOrDefault(i => i.StartsWith("language"));
+                                if (language != null)
+                                {
+                                    engine.SetVariableValue("language", language.Replace("language-", ""));
+                                    engine.Execute("result = hljs.highlight(language, input)");
+                                }
+                                else
+                                {
+                                    engine.Execute("result = hljs.highlightAuto(input)");
+                                    string detectedLanguage = engine.Evaluate<string>("result.language");
+                                    if (string.IsNullOrWhiteSpace(detectedLanguage) == false)
+                                    {
+                                        element.ClassList.Add("language-" + detectedLanguage);
+                                    }
+                                }
+                            }
+                            catch (JsRuntimeException jsRuntimeException)
+                            {
+                                if (_warnOnMissingLanguage || !jsRuntimeException.Message.StartsWith("Script threw an exception: Unknown language: "))
+                                {
+                                    // not a missing language exception so rethrow and let our parent handle it
+                                    throw;
+                                }
+
+                                // missing language but warning is disabled. Log the info and go to the next element
+                                Trace.Information("Exception while highlighting source code for {0} : {1}", input.SourceString(), jsRuntimeException.Message);
+                                continue;
+                            }
+
+                            element.ClassList.Add("hljs");
+                            string formatted = engine.Evaluate<string>("result.value");
+                            if (_escapeAt)
+                            {
+                                // without this razor has the potential to blow up parsing our code block
+                                formatted = formatted.Replace("@", "&#64;");
+                            }
+
+                            element.InnerHtml = formatted;
+                        }
+
+                        return context.GetDocument(input, htmlDocument.ToHtml());
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Trace.Warning("Exception while highlighting source code for {0}: {1}", input.SourceString(), ex.Message);
+                    return input;
+                }
+            });
+        }
+
+        private MsieJsEngine BuildJsEngineWithHighlight()
+        {
+            MsieJsEngine engine = new MsieJsEngine();
+            if (string.IsNullOrWhiteSpace(_highlightJsFile))
+            {
+                engine.ExecuteResource("highlight-all.js", typeof(Highlight));
+            }
+            else
+            {
+                engine.ExecuteFile(_highlightJsFile);
+            }
+            return engine;
+        }
+    }
+}

src/extensions/Wyam.Highlight/Properties/AssemblyInfo.cs

@@ -0,0 +1,8 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+[assembly: AssemblyTitle("Wyam.Highlight")]
+[assembly: AssemblyDescription("")]
+[assembly: ComVisible(false)]
+[assembly: Guid("5fe758a2-1221-48fe-8a3c-8b3b9a32afab")]

src/extensions/Wyam.Highlight/Wyam.Highlight.csproj

@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{5FE758A2-1221-48FE-8A3C-8B3B9A32AFAB}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>Wyam.Highlight</RootNamespace>
+    <AssemblyName>Wyam.Highlight</AssemblyName>
+    <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="AngleSharp, Version=0.9.9.0, Culture=neutral, PublicKeyToken=e83494dcdc6d31ea, processorArchitecture=MSIL">
+      <HintPath>..\..\..\packages\AngleSharp.0.9.9\lib\net45\AngleSharp.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="MsieJavaScriptEngine, Version=2.1.1.0, Culture=neutral, PublicKeyToken=a3a2846a37ac0d3e, processorArchitecture=MSIL">
+      <HintPath>..\..\..\packages\MsieJavaScriptEngine.2.1.1\lib\net45\MsieJavaScriptEngine.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="..\..\..\SolutionInfo.cs">
+      <Link>Properties\SolutionInfo.cs</Link>
+    </Compile>
+    <Compile Include="Highlight.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="highlight-all.js" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\core\Wyam.Common\Wyam.Common.csproj">
+      <Project>{F40B73E9-C0CC-465C-925E-B51E686C1D5C}</Project>
+      <Name>Wyam.Common</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="regenerating-highlight-all.js.txt" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>

src/extensions/Wyam.Highlight/Wyam.Highlight.nuspec

@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<package >
+  <metadata>
+    <id>Wyam.Highlight</id>
+    <version>$version$</version>
+    <title>Wyam.Highlight</title>
+    <authors>Dave Glick</authors>
+    <owners>Dave Glick</owners>
+    <licenseUrl>https://github.com/Wyamio/Wyam/blob/master/LICENSE</licenseUrl>
+    <iconUrl>https://wyam.io/Content/images/logo-square-64.png</iconUrl>
+    <projectUrl>https://wyam.io</projectUrl>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <description>Wyam is a simple to use, highly modular, and extremely configurable static content generator. This library performs server side highlighting of code blocks.</description>
+    <copyright>Copyright 2016</copyright>
+    <tags>Wyam Static StaticContent StaticSite Blog BlogEngine Highlight.js</tags>    
+  </metadata>
+  <files></files>
+</package>