Merge pull request #4758 from galaxyproject/tamandua-ostrich

No more full depth clones for deployment

.github/workflows/cron-commit-cache.yml

@@ -0,0 +1,62 @@
+name: "[Cron] Update Commit Cache"
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '45 0 * * 1'
+jobs:
+  runner-job:
+    runs-on: ubuntu-latest
+    # Only run on main repo on and PRs that match the main repo.
+    if: |
+      github.repository == 'galaxyproject/training-material' &&
+      (github.event_name != 'pull_request' ||
+       github.event.pull_request.head.repo.full_name == github.repository)
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1000
+
+      # BEGIN Dependencies
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.0"
+      - uses: actions/cache@v2
+        with:
+          path: |
+            vendor/bundle
+          key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-
+      - name: Install dependencies
+        run: |
+          gem install bundler
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+          bundle pristine ffi
+      # END Dependencies
+
+      - name: Update commit cache
+        id: generate
+        run: |
+          # Check if the oldest commit of the commit cache is known in this history
+          last_cached_commit=$(ls metadata/git-mod-*.txt | cut -d- -f 3 | cut -d. -f 1)
+          git show $last_cached_commit
+          ec=$?
+          if (( ec != 0 )); then
+            # Worst case, we need to fetch more. just fetch everything to be safe.
+            git fetch --unshallow
+          fi
+
+          ruby bin/commit-cache-update.rb >> $GITHUB_OUTPUT
+
+      - name: Create Pull Request
+        # If it's not a Pull Request then commit any changes as a new PR.
+        if: |
+          github.event_name != 'pull_request' &&
+          steps.generate.outputs.new_ids != ''
+        uses: peter-evans/create-pull-request@v3
+        with:
+          title: Update Cached Commit Data
+          branch-suffix: commitcache
+          commit-message: Update Cached Commit Data
+          add-paths: metadata/git-*.txt

.github/workflows/deploy.yml

@@ -16,7 +16,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          fetch-depth: 0
+          # Avg commits between successive monday:
+          #  Min.   :  2.0
+          #  1st Qu.: 25.0
+          #  Median : 54.5
+          #  Mean   : 65.2
+          #  3rd Qu.: 94.0
+          #  Max.   :282.0
+          fetch-depth: 400
 
       # BEGIN Dependencies
       - uses: actions/setup-python@v2

.github/workflows/matrix.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          fetch-depth: 0
+          fetch-depth: 200
 
       # BEGIN Dependencies
       - uses: ruby/setup-ruby@v1

.github/workflows/monthly-release.yml

@@ -32,7 +32,8 @@ jobs:
 
       - uses: actions/checkout@v2
         with:
-          fetch-depth: 0
+          # Based on logic in deploy.yml
+          fetch-depth: 1200
 
       # BEGIN Dependencies
       - uses: actions/setup-python@v2

.github/workflows/video-dry.yml

@@ -12,15 +12,15 @@ jobs:
     runs-on: ubuntu-latest
     services:
       # Label used to access the service container
-      mozillatts:
-        image: synesthesiam/mozillatts
-        ports:
-          - 5002:5002
+      # mozillatts:
+      #   image: synesthesiam/mozillatts
+      #   ports:
+      #     - 5002:5002
     steps:
       # Shallow should be fine for video
       - uses: actions/checkout@v2
         with:
-          fetch-depth: 0
+          fetch-depth: 1
 
       # BEGIN Dependencies
       - uses: actions/setup-python@v2

_plugins/gtn.rb

@@ -511,10 +511,10 @@ def tutorials_over_time_bar_chart(site)
       # https://stackoverflow.com/questions/71745593/how-to-do-a-single-line-cumulative-count-for-hash-values-in-ruby
       graph
         # Turns it into an array
-        .sort_by{|k, v| k}
+        .sort_by { |k, _v| k }
         # Cumulative sum
         .each_with_object([]) { |(k, v), a| a << [k, v + a.last&.last.to_i] }.to_h
-        .map{ |k, v| { 'x' => k, 'y' => v } }
+        .map { |k, v| { 'x' => k, 'y' => v } }
         .to_json
     end
 

_plugins/gtn/mod.rb

@@ -14,7 +14,7 @@ def self.init_cache
       @@TIME_CACHE = {}
       @@COMMIT_COUNT_CACHE = Hash.new(0)
       Jekyll.logger.info '[GTN/Time/Mod] Filling Time Cache'
-      `git log --name-only --pretty='GTN_GTN:%ct'`
+      cached_command
         .split('GTN_GTN:')
         .map { |x| x.split("\n\n") }
         .select { |x| x.length > 1 }
@@ -26,6 +26,39 @@ def self.init_cache
       end
     end
 
+    def self.discover_caches
+      # Really there should only be one, but maybe someone's been silly so
+      # we'll just take the first one we find.
+      Dir.glob('metadata/git-mod-*.txt').first
+    end
+
+    def self.generate_cache
+      rev = `git rev-list -n 1 main`.strip
+
+      if discover_caches.nil?
+        File.write("metadata/git-mod-#{rev}.txt", command)
+      else
+        prev = discover_caches
+        File.write("metadata/git-mod-#{rev}.txt", cached_command)
+        File.delete(prev)
+      end
+    end
+
+    def self.cached_command
+      return command if discover_caches.nil?
+
+      Jekyll.logger.info '[GTN/Time/Mod] Using cached modification times'
+
+      previous_commit = discover_caches.split('-').last.split('.').first
+      previous = File.read(discover_caches)
+
+      `git log --name-only --pretty='GTN_GTN:%ct' #{previous_commit}..` + previous
+    end
+
+    def self.command
+      `git log --name-only --pretty='GTN_GTN:%ct'`
+    end
+
     def self.time_cache
       @@TIME_CACHE
     end
@@ -51,6 +84,7 @@ def self.obtain_time(f)
         begin
           # Non git file.
           @@TIME_CACHE[f] = File.mtime(f)
+          Jekyll.logger.warning "[GTN/Time/Mod] No git cached time available for #{f}, defaulting to checkout"
           @@TIME_CACHE[f]
         rescue StandardError
           Time.at(0)
@@ -80,7 +114,7 @@ def self.init_cache
       renames = {}
 
       Jekyll.logger.info '[GTN/Time/Pub] Filling Publication Time Cache'
-      `git log --first-parent --name-status --diff-filter=AR --pretty='GTN_GTN:%ct' main`
+      cached_command
         .split('GTN_GTN:')
         .map { |x| x.split("\n\n") }
         .select { |x| x.length > 1 }
@@ -100,6 +134,39 @@ def self.init_cache
       # pp renames
     end
 
+    def self.discover_caches
+      # Really there should only be one, but maybe someone's been silly so
+      # we'll just take the first one we find.
+      Dir.glob('metadata/git-pub-*.txt').first
+    end
+
+    def self.generate_cache
+      rev = `git rev-list -n 1 main`.strip
+
+      if discover_caches.nil?
+        File.write("metadata/git-pub-#{rev}.txt", command)
+      else
+        prev = discover_caches
+        File.write("metadata/git-pub-#{rev}.txt", cached_command)
+        File.delete(prev)
+      end
+    end
+
+    def self.cached_command
+      return command if discover_caches.nil?
+
+      Jekyll.logger.info '[GTN/Time/Pub] Using cached publication times'
+
+      previous_commit = discover_caches.split('-').last.split('.').first
+      previous = File.read(discover_caches)
+
+      `git log --first-parent --name-status --diff-filter=AR --pretty='GTN_GTN:%ct' #{previous_commit}..` + previous
+    end
+
+    def self.command
+      `git log --first-parent --name-status --diff-filter=AR --pretty='GTN_GTN:%ct' `
+    end
+
     def self.time_cache
       @@TIME_CACHE
     end
@@ -112,6 +179,7 @@ def self.obtain_time(f)
         begin
           # Non git file.
           @@TIME_CACHE[f] = File.mtime(f)
+          Jekyll.logger.warning "[GTN/Time/Pub] No git cached time available for #{f}, defaulting to checkout"
           @@TIME_CACHE[f]
         rescue StandardError
           Time.at(0)
@@ -125,7 +193,7 @@ def self.obtain_time(f)
   # Gtn::ModificationTimes.init_cache
   # pp Gtn::ModificationTimes.commit_count_cache
 
-  puts ' Moved tobin/list-recently-modified.rb'
+  puts ' Moved to bin/list-recently-modified.rb'
   # Gtn::PublicationTimes.init_cache
   # Gtn::PublicationTimes.time_cache.select do |_, v|
   #   # Things in last 6 months

_plugins/jekyll-tool-tag.rb

@@ -23,21 +23,24 @@ def render(context)
         # check if a variable was provided for the tool id
         tool = context[m[2].tr('{}', '')] || m[2]
         version = tool.split('/').last
+        # url encode
+        q = URI.encode_www_form_component("?tool_id=#{tool}")
+        tool_redirect = "https://my.galaxy.training/?path=#{q}"
 
         if tool.count('/').zero?
-          "<span class=\"tool\" data-tool=\"#{tool}\" title=\"#{m[1]} tool\" aria-role=\"button\">" \
+          "<a class=\"tool\" data-tool=\"#{tool}\" title=\"#{m[1]} tool\" href=\"#{tool_redirect}\">" \
             '<i class="fas fa-wrench" aria-hidden="true"></i> ' \
             "<strong>#{m[1]}</strong>" \
-            '</span>'
+            '</a>'
         else
-          "<span class=\"tool\" data-tool=\"#{tool}\" title=\"#{m[1]} tool\" aria-role=\"button\">" \
+          "<a class=\"tool\" data-tool=\"#{tool}\" title=\"#{m[1]} tool\" href=\"#{tool_redirect}\">" \
             '<i class="fas fa-wrench" aria-hidden="true"></i> ' \
             "<strong>#{m[1]}</strong> " \
             '(' \
             '<i class="fas fa-cubes" aria-hidden="true"></i> ' \
             "Galaxy version #{version}" \
             ')' \
-            '</span>'
+            '</a>'
         end
       else
         %(<span><strong>#{@text}</strong> <i class="fas fa-wrench" aria-hidden="true"></i></span>)

_plugins/jekyll-topic-filter.rb

@@ -364,6 +364,14 @@ def self.collate_materials(site, pages)
     interesting
   end
 
+  def self.mermaid_safe_label(label)
+    (label || '')
+      .gsub('(', '').gsub(')', '')
+      .gsub('[', '').gsub(']', '')
+      .gsub('"', '”') # We accept that this is not perfectly correct.
+      .gsub("'", '’')
+  end
+
   def self.mermaid(wf)
     # We're converting it to Mermaid.js
     # flowchart TD
@@ -373,24 +381,59 @@ def self.mermaid(wf)
     #     D --> B
     #     B -- No ----> E[End]
 
-    output = "flowchart TD\n"
+    statements = []
     wf['steps'].each_key do |id|
       step = wf['steps'][id]
-      output += "  #{id}[\"#{step['name']}\"];\n"
+      chosen_label = mermaid_safe_label(step['label'] || step['name'])
+
+      case step['type']
+      when 'data_collection_input'
+        statements.append "#{id}[\"ℹ️ Input Collection\\n#{chosen_label}\"];"
+      when 'data_input'
+        statements.append "#{id}[\"ℹ️ Input Dataset\\n#{chosen_label}\"];"
+      when 'parameter_input'
+        statements.append "#{id}[\"ℹ️ Input Parameter\\n#{chosen_label}\"];"
+      when 'subworkflow'
+        statements.append "#{id}[\"🛠️ Subworkflow\\n#{chosen_label}\"];"
+      else
+        statements.append "#{id}[\"#{chosen_label}\"];"
+      end
+
+      case step['type']
+      when 'data_collection_input', 'data_input'
+        statements.append "style #{id} stroke:#2c3143,stroke-width:4px;"
+      when 'parameter_input'
+        statements.append "style #{id} fill:#ded,stroke:#393,stroke-width:4px;"
+      when 'subworkflow'
+        statements.append "style #{id} fill:#edd,stroke:#900,stroke-width:4px;"
+      end
+
       step = wf['steps'][id]
       step['input_connections'].each do |_, v|
         # if v is a list
         if v.is_a?(Array)
           v.each do |v2|
-            output += "  #{v2['id']} -->|#{v2['output_name']}| #{id};\n"
+            statements.append "#{v2['id']} -->|#{mermaid_safe_label(v2['output_name'])}| #{id};"
           end
         else
-          output += "  #{v['id']} -->|#{v['output_name']}| #{id};\n"
+          statements.append "#{v['id']} -->|#{mermaid_safe_label(v['output_name'])}| #{id};"
+        end
+      end
+
+      (step['workflow_outputs'] || [])
+        .reject { |wo| wo['label'].nil? }
+        .map do |wo|
+          wo['uuid'] = SecureRandom.uuid.to_s if wo['uuid'].nil?
+          wo
         end
+        .each do |wo|
+        statements.append "#{wo['uuid']}[\"Output\\n#{wo['label']}\"];"
+        statements.append "#{id} --> #{wo['uuid']};"
+        statements.append "style #{wo['uuid']} stroke:#2c3143,stroke-width:4px;"
       end
     end
 
-    output
+    "flowchart TD\n" + statements.map { |q| "  #{q}" }.join("\n")
   end
 
   def self.resolve_material(site, material)

bin/commit-cache-update.rb

@@ -0,0 +1,11 @@
+require './_plugins/gtn'
+
+# Write the commit log
+Gtn::ModificationTimes.generate_cache
+Gtn::PublicationTimes.generate_cache
+
+# Gziped?
+# require 'zlib'
+# File.write("metadata/git-mod-#{rev}.txt.gz", Zlib.gzip(Gtn::ModificationTimes.command))
+# File.write("metadata/git-pub-#{rev}.txt.gz", Zlib.gzip(Gtn::PublicationTimes.command))
+# p Gtn::ModificationTimes.discover_caches

bin/list-recently-modified.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+require './_plugins/gtn'
+
+months = (ARGV[1] || 6).to_i
+
+Gtn::ModificationTimes.init_cache
+Gtn::ModificationTimes.time_cache.select do |_, v|
+  # Things in last 6 months
+  v > Time.now - (months * 30 * 24 * 60 * 60)
+end.map { |k, v| puts "#{v} #{k}" }

bin/list-recently-published.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+require './_plugins/gtn'
+
+months = (ARGV[1] || 6).to_i
+
+Gtn::PublicationTimes.init_cache
+Gtn::PublicationTimes.time_cache.select do |_, v|
+  # Things in last 6 months
+  v > Time.now - (months * 30 * 24 * 60 * 60)
+end.map { |k, v| puts "#{v} #{k}" }