Merge branch 'galaxyproject:main' into single-cell-data-ingest

.github/workflows/ci-main.yml

@@ -70,4 +70,5 @@ jobs:
         with:
           rubocop_version: gemfile
           reporter: github-pr-review
+          rubocop_flags: --except Layout/LineLength,Lint/RedundantCopDisableDirective
           fail_on_error: true

.github/workflows/matrix.yml

@@ -40,7 +40,7 @@ jobs:
 
       - name: Send announcements to matrix
         run: |
-            commit=$(git log --since "24 hours ago" --format=%H | tail -n 1)
+            commit=$(git log --since "24 hours ago" --format=%H --first-parent main| tail -n 1)
             bundle exec ruby bin/news.rb -p "${commit}~1" --matrix-post
         env:
             MATRIX_ACCESS_TOKEN: ${{ secrets.matrix_access_token }}

_layouts/base.html

@@ -32,10 +32,10 @@
             {% assign og_title = page.title %}
         {% endif %}
         {% assign og_desc = page.description | default:topic.summary | default: "Collection of tutorials developed and maintained by the worldwide Galaxy community" %}
-        <meta name="description" content="{{ og_desc | strip_html | truncate: 120}}">
+        <meta name="description" content="{{ og_desc | strip_html}}">
         <meta property="og:site_name" content="Galaxy Training Network">
-        <meta property="og:title" content="Galaxy Training{% if og_title %}: {{ og_title | truncate: 60}}{% endif %}">
-        <meta property="og:description" content="{{ og_desc | strip_html | truncate: 120}}">
+	<meta property="og:title" content="[{% if topic.title %}{{ topic.title }}{% endif %}]{% if page.layout == "faq" %}FAQS {% elsif page.layout == "topic" %}Topic: {% endif %}{{ og_title }} {% if page.layout == "workflow-list" %}Workflows{% endif %}">
+        <meta property="og:description" content="{{ og_desc | strip_html}}">
 
         {%- if page.cover %}{% assign coverimage = page.cover %}
         {%- elsif page.tags contains "cofest" %}{% assign coverimage = "/assets/images/cofest.png" %}

_layouts/base_slides.html

@@ -30,10 +30,10 @@
             {% assign og_title = page.title %}
         {% endif %}
         {% assign og_desc = page.description | default:topic.summary | default: "Collection of tutorials developed and maintained by the worldwide Galaxy community" %}
-        <meta name="description" content="{{ og_desc | strip_html | truncate: 120}}">
+        <meta name="description" content="{{ og_desc | strip_html }}">
         <meta property="og:site_name" content="Galaxy Training Network">
-        <meta property="og:title" content="Galaxy Training{% if og_title %}: {{ og_title | truncate: 60}}{% endif %}">
-        <meta property="og:description" content="{{ og_desc | strip_html | truncate: 120}}">
+        <meta property="og:title" content="Slide Deck: {{ og_title }}">
+        <meta property="og:description" content="{{ og_desc | strip_html }}">
 
         {%- if page.cover %}{% assign coverimage = page.cover %}
         {%- elsif page.tags contains "cofest" %}{% assign coverimage = "/assets/images/cofest.png" %}

_layouts/slides-plain.html

@@ -34,7 +34,8 @@ <h2>Requirements</h2>
 		</ul>
 		{% endif %}
 
-		<div><strong> {% icon last_modification %} Last modification:</strong> {{ page | last_modified_at | date: "%b %-d, %Y"}} </div>
+		<div><strong> {% icon last_modification %} Published:</strong> {{ page | gtn_pub_date | date: "%b %-d, %Y"}} </div>
+		<div><strong> {% icon last_modification %} Last Updated:</strong> {{ page | last_modified_at | date: "%b %-d, %Y"}} </div>
 
 	<hr />
 

_layouts/tutorial_hands_on.html

@@ -149,6 +149,7 @@ <h4 class="alert-heading">Under Development!</h4>
             {% endif %}
         </ul>
 
+        <div><strong>{% icon last_modification aria=false %} {{ locale['published'] | default: "Published" }}:</strong> {{ page | gtn_pub_date | date: "%b %-d, %Y"}} </div>
         <div><strong>{% icon last_modification aria=false %} {{ locale['last-modification'] | default: "Last modification" }}:</strong> {{ page | last_modified_at | date: "%b %-d, %Y"}} </div>
         <div><strong>{% icon license aria=false %} {{ locale['license'] | default: "License" }}:</strong>
 		{% if page.copyright %}

_layouts/workflow-list.html

@@ -12,7 +12,8 @@ <h1 data-toc-skip>Workflows</h1>
 
     <div class="row">
     {% for workflow in material.workflows %}
-	<div class="card col-md-6">
+	<div class="col-md-6">
+		<div class="card">
 		<div class="card-body">
 			<h5 class="card-title">{{ workflow.title }}</h5>
 			<h6 class="card-subtitle mb-2 text-muted">
@@ -55,10 +56,11 @@ <h6 class="card-subtitle mb-2 text-muted">
   </div>
 </div>
 
-	</div>
 	<pre class="mermaid">{{ workflow.mermaid }}
 	</pre>
-</div>
+		</div><!-- card-body -->
+	</div><!-- card -->
+</div><!-- col -->
 {% endfor %}
 
 </div>

_plugins/gtn.rb

@@ -122,6 +122,20 @@ def replace_newline_doublespace(text)
       text.gsub(/\n/, "\n  ")
     end
 
+    ##
+    # Returns the publication date of a page, when it was merged into `main`
+    # Params:
+    # +page+:: The page to get the publication date of
+    # Returns:
+    # +String+:: The publication date of the page
+    #
+    def gtn_pub_date(path)
+      # if it's not a string then log a warning
+      path = path['path'] if !path.is_a?(String)
+      # Automatically strips any leading slashes.
+      Gtn::PublicationTimes.obtain_time(path.gsub(%r{^/}, ''))
+    end
+
     ##
     # Returns the last modified date of a page
     # Params:

_plugins/gtn/mod.rb

@@ -13,7 +13,7 @@ def self.init_cache
 
       @@TIME_CACHE = {}
       @@COMMIT_COUNT_CACHE = Hash.new(0)
-      puts '[GTN/MOD] Filling Time Cache'
+      puts '[GTN/Time/Mod] Filling Time Cache'
       `git log --name-only --pretty='GTN_GTN:%ct'`
         .split('GTN_GTN:')
         .map { |x| x.split("\n\n") }
@@ -58,9 +58,76 @@ def self.obtain_time(f)
       end
     end
   end
+
+  # Module for obtaining original publication times of files.
+  # It walks the git history to record the last time a file was modified.
+  # This is faster than talking to the file system.
+  module PublicationTimes
+    @@TIME_CACHE = nil
+
+    def self.chase_rename(renames, path)
+      if renames.key? path
+        chase_rename(renames, renames[path])
+      else
+        path
+      end
+    end
+
+    def self.init_cache
+      return unless @@TIME_CACHE.nil?
+
+      @@TIME_CACHE = {}
+      renames = {}
+
+      puts '[GTN/Time/Pub] Filling Publication Time Cache'
+      `git log --first-parent --name-status --diff-filter=AR --pretty='GTN_GTN:%ct' main`
+        .split('GTN_GTN:')
+        .map { |x| x.split("\n\n") }
+        .select { |x| x.length > 1 }
+        .each do |date, files|
+        files.split("\n").grep(/\.(md|html)$/).each do |f|
+          modification_type, path = f.split("\t")
+          if modification_type == 'A'
+            # Chase the renames.
+            final_filename = chase_rename(renames, path)
+            @@TIME_CACHE[final_filename] = Time.at(date.to_i)
+          elsif modification_type[0] == 'R'
+            _, moved_from, moved_to = f.split("\t")
+            renames[moved_from] = moved_to # Point from the 'older' version to the newer.
+          end
+        end
+      end
+      # pp renames
+    end
+
+    def self.time_cache
+      @@TIME_CACHE
+    end
+
+    def self.obtain_time(f)
+      init_cache
+      if @@TIME_CACHE.key? f
+        @@TIME_CACHE[f]
+      else
+        begin
+          # Non git file.
+          @@TIME_CACHE[f] = File.mtime(f)
+          @@TIME_CACHE[f]
+        rescue StandardError
+          Time.at(0)
+        end
+      end
+    end
+  end
 end
 
 if $PROGRAM_NAME == __FILE__
-  Gtn::ModificationTimes.init_cache
-  pp Gtn::ModificationTimes.commit_count_cache
+  # Gtn::ModificationTimes.init_cache
+  # pp Gtn::ModificationTimes.commit_count_cache
+
+  Gtn::PublicationTimes.init_cache
+  Gtn::PublicationTimes.time_cache.select do |_, v|
+    # Things in last 6 months
+    v > Time.now - (6 * 30 * 24 * 60 * 60)
+  end.map { |k, v| puts "#{v} #{k}" }
 end

_plugins/jekyll-bundler.rb

@@ -63,7 +63,7 @@ def bundle_preloads_prod
         bundle['preload'] == true
       end
 
-      bundles.map do |name, bundle|
+      bundles.map do |_name, bundle|
         bundle_path = "#{baseurl}#{bundle['path']}?v=#{bundle['timestamp']}"
         "<link rel='preload' href='#{bundle_path}' as='script'>"
       end.join("\n")

_plugins/jekyll-jsonld.rb

@@ -307,6 +307,7 @@ def to_jsonld(material, topic, site)
         # "creator":,
         # "dateCreated":,
         dateModified: Gtn::ModificationTimes.obtain_time(material['path']),
+        datePublished: Gtn::PublicationTimes.obtain_time(material['path']),
         # "datePublished":,
         discussionUrl: site['gitter_url'],
         # "editor":,

_plugins/jekyll-scholar.rb

@@ -99,14 +99,14 @@ def render(context)
       site.config['citation_cache'][source_page].push(@text)
 
       begin
-          doi = site.config['cached_citeproc'].items[@text].doi
-          url = site.config['cached_citeproc'].items[@text].url
-          if !doi.nil?
-            "https://doi.org/#{doi}"
-          elsif !url.nil?
-            url
-          end
-          res = url
+        doi = site.config['cached_citeproc'].items[@text].doi
+        url = site.config['cached_citeproc'].items[@text].url
+        if !doi.nil?
+          "https://doi.org/#{doi}"
+        elsif !url.nil?
+          url
+        end
+        res = url
       rescue StandardError => e
         puts "[GTN/scholar] Could not render #{@text} from #{source_page} (#{e})"
         res = %(<span>https://example.com/ERROR+INVALID+CITATION+#{@text}</span>)

_plugins/jekyll-topic-filter.rb

@@ -373,13 +373,9 @@ def self.mermaid(wf)
     #     B -- No ----> E[End]
 
     output = "flowchart TD\n"
-    wf['steps'].keys.each do |id|
+    wf['steps'].each_key do |id|
       step = wf['steps'][id]
       output += "  #{id}[\"#{step['name']}\"];\n"
-    end
-
-    wf['steps'].keys.each do |id|
-      # Look at the 'input connections' to this step
       step = wf['steps'][id]
       step['input_connections'].each do |_, v|
         # if v is a list
@@ -844,16 +840,41 @@ def most_recent_contributors(contributors, count)
     # Find the most recently modified tutorials
     # Parameters:
     # +site+:: The +Jekyll::Site+ object, used to get the list of pages.
+    # +exclude_recently_published+:: Do not include ones that were recently
+    #                                published in the slice, to make it look a bit nicer.
     # Returns:
     # +Array+:: An array of the 10 most recently modified pages
     # Example:
     #  {% assign latest_tutorials = site | recently_modified_tutorials %}
-    def recently_modified_tutorials(site)
+    def recently_modified_tutorials(site, exclude_recently_published: true)
       tutorials = site.pages.select { |page| page.data['layout'] == 'tutorial_hands_on' }
 
       latest = tutorials.sort do |x, y|
         Gtn::ModificationTimes.obtain_time(y.path) <=> Gtn::ModificationTimes.obtain_time(x.path)
       end
+
+      latest_published = recently_published_tutorials(site)
+      latest = latest.reject { |x| latest_published.include?(x) } if exclude_recently_published
+
+      latest.slice(0, 10)
+    end
+
+    ##
+    # Find the most recently published tutorials
+    # Parameters:
+    # +site+:: The +Jekyll::Site+ object, used to get the list of pages.
+    # Returns:
+    # +Array+:: An array of the 10 most recently published modified pages
+    # Example:
+    #  {% assign latest_tutorials = site | recently_modified_tutorials %}
+    def recently_published_tutorials(site)
+      tutorials = site.pages.select { |page| page.data['layout'] == 'tutorial_hands_on' }
+
+      latest = tutorials.sort do |x, y|
+        Gtn::PublicationTimes.obtain_time(y.path) <=> Gtn::PublicationTimes.obtain_time(x.path)
+      end
+
+      latest.each { |x| puts [x.path, Gtn::PublicationTimes.obtain_time(x.path)] }
       latest.slice(0, 10)
     end
 

assets/css/main.scss

@@ -698,13 +698,22 @@ blockquote {
 		border-bottom: 1px solid var(--border-light);
 		padding: 1.25rem;
 		color: var(--brand-color-contrast);
-		margin-left: -15px; // BS has a 15px margin on cards, this makes the card title fill the card.
-		margin-right: -15px;
+		// margin-left: -15px; // BS has a 15px margin on cards, this makes the card title fill the card.
+		// margin-right: -15px;
 	}
 
 	.card-body {
 		padding: 0em;
 		flex: 0 1 auto; // Override BS for the workflow listing.
+
+		// We stripped the padding from the card body in the above line
+		// Thus we need to add it back via margins to each element.
+		// We have to manually exclude the card titles
+		// And also canvases because they apparently calculate their area before the CSS has finished.
+		& > :not(.card-title,canvas){
+			margin-left: 1em;
+			margin-right: 1em;
+		}
 	}
 }
 

bin/lint.rb

@@ -984,8 +984,8 @@ def self.fix_file(path)
       # Check if there's a missing workflow test
       folder = File.dirname(path)
       basename = File.basename(path).gsub(/.ga$/, '')
-      possible_tests = Dir.glob("#{folder}/#{basename}*ym*")
-      possible_tests = possible_tests.grep(/#{basename}[_-]tests?.ya?ml/)
+      possible_tests = Dir.glob("#{folder}/#{Regexp.escape(basename)}*ym*")
+      possible_tests = possible_tests.grep(/#{Regexp.escape(basename)}[_-]tests?.ya?ml/)
 
       if possible_tests.empty?
         results += [
@@ -999,10 +999,11 @@ def self.fix_file(path)
       else
         # Load tests and run some quick checks:
         possible_tests.each do |test_file|
-          if !test_file.match(/-test.yml/)
+          if !test_file.match(/-tests?.yml/)
             results += [
               ReviewDogEmitter.file_error(path: path,
-                                          message: 'Please use the extension -test.yml for this test file.',
+                                          message: 'Please use the extension -test.yml ' \
+                                                   'or -tests.yml for this test file.',
                                           code: 'GTN:032')
             ]
           end

search-tools.html

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: Pan Galactic Tool Search
+title: GTN Pan-Galactic Tool Search
 ---
 
 
@@ -78,6 +78,7 @@
 		return
 	}
 
+	document.getElementsByTagName("title")[0].innerText = `${textQuery} | GTN Pan-Galactic Tool Search`
 	console.log(`Searching! ${textQuery}`)
 
 	// Which should be hidden

search.md

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: Search Tutorials
+title: GTN Tutorial Search
 ---
 
 <script src="https://cdnjs.cloudflare.com/ajax/libs/lunr.js/2.3.9/lunr.min.js" integrity="sha512-4xUl/d6D6THrAnXAwGajXkoWaeMNwEKK4iNfq5DotEbLPAfk6FSxSP3ydNxqDgCw1c/0Z1Jg6L8h2j+++9BZmg==" crossorigin="anonymous"></script>
@@ -29,6 +29,9 @@ function search(idx, q, includeFaqs, includeTutorials){
             results_exact = idx.search(`${q}`),
             results_fuzzy = idx.search(`${q}~3`);
 
+	// Include search term in page title
+		document.getElementsByTagName("title")[0].innerText = `${q} | GTN Tutorial Search`
+
         thereMap  = Object.assign({}, ...results_partial.map((x) => ({[x.ref]: x.score})));
 
         results_exact.forEach(x => {

search2.html

@@ -1,6 +1,6 @@
 ---
 layout: page
-title: Search GTN Materials
+title: GTN Materials Search
 ---
 
 <p>
@@ -120,6 +120,8 @@
 		return
 	}
 
+	// Include search term in page title
+	document.getElementsByTagName("title")[0].innerText = `${textQuery} | GTN Materials Search`
 	console.log(`Searching! ${textQuery} ${typeFilter} ${levelFilter}`)
 
 	// Which should be hidden