Automate repository docs inclusion (#1762)

So far every prometheus/alertmanater/... release branch had to be
manually configured in the nanoc.yaml config file. With this change the
most recent release branches will be checked out automatically if a
corresponding semver tag exists.

As the Prometheus git repository includes several hundreds of megabytes
of vendored assets, the repository is cloned bare and all blobs are
filtered by default. Each version is then checked out in an individual
working tree and git's spare-chekcout feature is used to reduce the
checkout to the `docs/` folder. The git data is cached in
`tmp/repo_docs/` and will be recreated automatically if removed.

Signed-off-by: Tobias Schmidt <[email protected]>

Author: grobie

.gitignore

@@ -5,8 +5,7 @@ output/
 
 # Temporary file directory
 tmp/
-/downloads/
-/repositories/
+downloads/
 
 # Crash Log
 crash.log

Makefile

@@ -9,7 +9,7 @@ bundle:
 	bundle install --path vendor
 
 clean:
-	rm -rf output downloads repositories
+	rm -rf output downloads
 
 compile:
 	$(NANOC)

Rules

@@ -14,21 +14,8 @@
 #   because “*” matches zero or more characters.
 
 passthrough '/assets/**'
-
-compile '/_redirects/' do
-end
-
-route '/_redirects/' do
-  '/_redirects'
-end
-
-# TODO(ts): Remove these hacks once the nanoc4 upgrade is done.
-compile '*/images/*' do
-end
-
-route '*/images/*' do
-  item.identifier.chop + '.' + item[:extension]
-end
+passthrough '/_redirects'
+passthrough '*/images/*'
 
 # RSS Feed
 compile '/blog/feed/' do
@@ -45,8 +32,7 @@ compile '*' do
   if item[:extension] == 'md'
     filter :redcarpet, options: {filter_html: true, autolink: true, no_intraemphasis: true, fenced_code_blocks: true, gh_blockcode: true, tables: true}, renderer_options: {with_toc_data: true}
     filter :normalize_links, item[:repo_docs] if item[:repo_docs]
-    filter :outdated_content, item[:repo_docs] if item[:repo_docs] && item[:repo_docs][:outdated]
-    filter :prerelease_content, item[:repo_docs] if item[:repo_docs] && item[:repo_docs][:prerelease]
+    filter :version_warning, item[:repo_docs] if item[:repo_docs]
     filter :add_anchors
     filter :bootstrappify
     filter :admonition

layouts/header.html

@@ -25,7 +25,7 @@
     <link rel="icon" type="image/png" href="/assets/favicons/favicon-96x96.png" sizes="96x96">
     <link rel="icon" type="image/png" href="/assets/favicons/favicon-16x16.png" sizes="16x16">
     <link rel="manifest" href="/assets/favicons/android-chrome-manifest.json">
-    <% if (c = @item[:repo_docs]) && c[:canonical] %><link rel="canonical" href="<%= @item.path.sub(c[:items_root], c[:canonical]) %>" /><% end %>
+    <% if (c = @item[:repo_docs]) && c[:canonical_root] %><link rel="canonical" href="<%= @item.path.sub(c[:items_root], c[:canonical_root]) %>" /><% end %>
     <!-- Meta tag for indexing that enables faceted search in Algolia,
       see https://docsearch.algolia.com/docs/required-configuration/#introduce-global-information-as-meta-tags -->
     <meta name="docsearch:prometheus-version" content="<%= @item[:repo_docs] && @item[:repo_docs][:name] || 'none' %>" />

lib/data_sources/repo_docs.rb

@@ -1,60 +1,158 @@
-# TODO(ts): Rewrite data source and use one single instance to combine all
-# different versions for a given path.
-class RepoDocsDataSource < ::Nanoc::DataSources::Filesystem
+require 'uri'
+
+# The RepoDocs data source provides items sourced from other Git repositories.
+# For a given repository_url, all git version tags are fetched and for the most
+# recent (in order to save compilation time) tags the `docs/` folder in the
+# respective `release-<version>` is checked out and its content mounted under
+# the given `items_root`.
+#
+# As the Prometheus git repository includes several hundreds of megabytes of
+# vendored assets, the repository is cloned bare and all blobs are filtered by
+# default. Each version is then checked out in an individual working tree and
+# git's spare-checkout feature is used to reduce the checkout to the `docs/`
+# folder. The git data is cached in `tmp/repo_docs/`.
+class RepoDocsDataSource < ::Nanoc::DataSource
   identifier :repo_docs
 
-  PATH = "repositories"
+  DOCS_DIRECTORY = 'docs'.freeze
+  BRANCH_PATTERN = 'release-*'.freeze
+  VERSION_REGEXP = /\Av\d+\.\d+\.\d+(?:-[a-z0-9.]+)?\z/.freeze
+  TMPDIR = 'tmp/repo_docs/'.freeze
 
   def up
-    c = config[:config]
-
-    %x(
-      scripts/checkout.sh \
-        -d "#{docs_root}" \
-        -t "#{repo_path}" \
-        "#{c[:repository]}" "#{c[:refspec]}"
-    )
-    if $?.exitstatus != 0
-      raise "Couldn't checkout repository #{c.inspect}"
-    end
-
-    super
+    validate
+    sync_repository
   end
 
   def items
-    c = config.fetch(:config)
-    super.map do |item|
-      attrs = item.attributes.dup
-      attrs[:repo_docs] = c
-      attrs[:repo_docs][:items_root] = config.fetch(:items_root)
-      # TODO(ts): Remove assumptions about the path layout, rewrite datasource.
-      attrs[:repo_docs][:version_root] = config.fetch(:items_root).sub(%r{(.+/)[^/]+/\Z}, '\\1')
-      # TODO(ts): Document that repo doc index.md will be ignored.
-      if item.identifier == '/'
-        attrs[:nav] = { strip: true }
+    items_root = config.fetch(:items_root, '/')
+    latest = latest_version
+
+    versions.inject([]) do |list, version|
+      branch = "release-#{version}"
+      dir = git_checkout(branch, DOCS_DIRECTORY)
+      fs_config = { content_dir: dir, encoding: 'utf-8', identifier_type: 'legacy' }
+      fs = ::Nanoc::DataSources::Filesystem.new(@site_config, '/', '/', fs_config)
+
+      fs.items.each do |item|
+        attrs = item.attributes.dup
+        attrs[:nav] = { strip: true } if item.identifier == '/'
+        attrs[:repo_docs] = {
+          name: version,
+          refspec: branch,
+          version: version,
+          latest: latest,
+          items_root: items_root,
+          version_root: File.join(items_root, version, '/'),
+          canonical_root: File.join(items_root, 'latest', '/'),
+          repository_url: git_remote,
+          entrypoint: config[:config][:entrypoint],
+        }
+
+        if version == latest
+          lattrs = attrs.dup
+          lattrs[:repo_docs] = attrs[:repo_docs].dup
+          lattrs[:repo_docs][:name] = "latest (#{version})"
+          lattrs[:repo_docs][:version_root] = lattrs[:repo_docs][:canonical_root]
+          list << new_item(item.content, lattrs, item.identifier.prefix('/latest'))
+        end
+
+        list << new_item(item.content, attrs, item.identifier.prefix('/' + version))
       end
-      new_item(item.content, attrs, item.identifier)
+
+      list
     end
   end
 
-  def content_dir_name
-    File.join(repo_path, docs_root)
+  private
+
+  def validate
+    if !config[:config].has_key?(:entrypoint)
+      fail ArgumentError, 'entrypoint config option must be set'
+    end
+    if !config[:config].has_key?(:repository_url)
+      fail ArgumentError, 'repository config option must be set'
+    end
+    URI(config[:config][:repository_url]) # raises an exception if invalid
   end
 
-  def layouts_dir_name
-    'unsupported'
+  def git_remote
+    config[:config][:repository_url]
   end
 
-  private
+  def git_dir
+    basename = File.basename(git_remote)
+    basename += '.git' unless basename.end_with?('.git')
+    File.join(TMPDIR, basename)
+  end
+
+  def git_branches
+    output = `cd #{git_dir} && git branch --format='%(refname:short)' --list '#{BRANCH_PATTERN}'`
+    fail "Could not list git branches" if $?.exitstatus != 0
+    output.split("\n")
+  end
+
+  def git_tags
+    output = `cd #{git_dir} && git tag`
+    fail "Could not list git tags" if $?.exitstatus != 0
+    output.split("\n")
+  end
+
+  # git_checkout checks out the directory in the specified branch using git's
+  # sparse checkout and returns the path to the location in the workingtree.
+  def git_checkout(branch, directory)
+    worktree = File.absolute_path(File.join(git_dir.delete_suffix('.git'), branch))
+    if !File.exist?(File.join(worktree, '.git'))
+      run_command("cd #{git_dir} && git worktree add --no-checkout #{worktree} #{branch}")
+    end
+
+    worktree_info = File.join(git_dir, 'worktrees', branch, 'info')
+    Dir.mkdir(worktree_info) if !Dir.exist?(worktree_info)
+    File.write(File.join(worktree_info, 'sparse-checkout'), "/#{directory}\n")
+
+    run_command("cd #{worktree} && git reset --hard --quiet && git clean --force")
+    File.join(worktree, directory)
+  end
+
+  # sync_repository clones or updates a bare git repository and enables the
+  # sparse checkout feature.
+  def sync_repository
+    if !Dir.exist?(git_dir)
+      run_command("git clone --bare --filter=blob:none #{git_remote} #{git_dir}")
+      run_command("cd #{git_dir} && git config core.sparseCheckout true")
+    else
+      run_command("cd #{git_dir} && git fetch --quiet")
+    end
+  end
+
+  # versions returns an ordered list of major.minor version names for which
+  # documentation should be published. Only the most recent versions for which a
+  # corresponding release-* branch exists are returned.
+  def versions
+    branches = git_branches
+    all = git_tags
+      .select { |v| v.match(VERSION_REGEXP) }
+      .map { |v| v.delete_prefix('v').split('.')[0, 2].join('.') }
+      .uniq
+      .select { |v| branches.include?('release-' + v) }
+      .sort_by { |v| v.split('.').map(&:to_i) }
+      .reverse
+
+    # Number of versions is reduced to speed up site compilation time.
+    grouped = all.group_by { |v| v.split('.').first }
+    grouped.inject([]) do |list, (major, versions)|
+      size = major == grouped.keys.first ? 10 : 1
+      list += versions[0, size]
+    end
+  end
 
-  def docs_root
-    c = config.fetch(:config)
-    c.fetch(:root, 'docs/')
+  # latest_version returns the latest released version.
+  def latest_version
+    tags = git_tags
+    versions.find { |v| tags.any? { |t| t.start_with?('v' + v) && !t.include?('-') } }
   end
 
-  def repo_path
-    c = config.fetch(:config)
-    base = c.fetch(:repo_base, 'repositories')
-    File.join(base, File.basename(c[:repository]), c[:name])
+  def run_command(cmd)
+    fail "Running command '#{cmd}' failed" if !system(cmd)
   end
 end

lib/filters/normalize_links.rb

@@ -35,10 +35,7 @@ def run(content, config = {})
   end
 
   def github_link_to(file, config)
-    base = config[:repository]
-    if base.end_with?('.git')
-      base = base[0..-5]
-    end
+    base = config[:repository_url].delete_suffix('.git')
     File.join(base, 'blob', config[:refspec], file)
   end
 

lib/filters/outdated_content.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+require 'nokogiri'
+
+# VersionWarning adds a warning to the top of pre-release or outdated versioned
+# documentation pages.
+class VersionWarning < ::Nanoc::Filter
+  identifier :version_warning
+
+  def run(content, params = {})
+    case version_compare(params[:version], params[:latest])
+    when 1
+      type = 'a pre-release version'
+    when 0
+      return content
+    when -1
+      type = 'an old version'
+    end
+
+    href = File.join(params[:canonical_root], params[:entrypoint])
+    repo = File.basename(params[:repository_url], '.git').capitalize
+    warning = %(<p>CAUTION: This page documents #{type} of #{repo}.
+      Check out the <a href="#{href}">latest stable version</a>.</p>)
+
+    prepend_warning(content, warning)
+  end
+
+  private
+
+  def prepend_warning(content, warning)
+    doc = Nokogiri::HTML(content)
+    body = doc.css('body')
+    if first = body.children.first
+      first.add_previous_sibling(warning)
+    else
+      body << Nokogiri::HTML::DocumentFragment.parse(warning)
+    end
+    doc.to_s
+  end
+
+  def version_compare(a, b)
+    a.split('.').map(&:to_i) <=> b.split('.').map(&:to_i)
+  end
+end

lib/filters/prerelease_content.rb

@@ -90,21 +90,16 @@ def self.versioned?(item)
     !item[:repo_docs].nil?
   end
 
-  # latest? returns true if the item is part of the version group "latest".
-  def self.latest?(opts)
-    opts[:name].include?('latest')
-  end
-
   # current? returns true if the item is part of the selected version group. If
   # no group is selected (e.g. when a page outside of the versioned docs is
   # viewed), the latest version will be shown.
   def self.current?(opts, page)
     return false if opts.nil? || !page.respond_to?(:path)
 
-    if page.path.start_with?(opts[:version_root])
-      page.path.start_with?(opts[:items_root])
+    if page.path.start_with?(opts[:items_root])
+      page.path.start_with?(opts[:version_root])
     else
-      latest?(opts)
+      opts[:version_root] == opts[:canonical_root]
     end
   end
 
@@ -115,7 +110,7 @@ def self.picker(items, page, active)
       selected = current?(v, page) ? 'selected="selected"' : ''
       # TODO(ts): Refactor and think about linking directly to the page of the same version.
       first = items
-        .find { |i| i.path.start_with?(v[:items_root]) }
+        .find { |i| i.path.start_with?(v[:version_root]) }
         .children.sort_by { |c| c[:sort_rank] || 0 }.first
       %(<option value="#{first.path}" #{selected}>#{v[:name]}</option>)
     end