added files to automate updates and ran updates

scripts/fix-lockfile.py

@@ -0,0 +1,65 @@
+import yaml
+import os
+import copy
+import argparse
+
+
+def update_file(fn, dry):
+    with open(fn, 'r') as handle:
+        unlocked = yaml.safe_load(handle)
+    # If a lock file exists, load it from that file
+    if os.path.exists(fn + '.lock'):
+        with open(fn + '.lock', 'r') as handle:
+            locked = yaml.safe_load(handle)
+    else:
+        # Otherwise just clone the "unlocked" list.
+        locked = copy.deepcopy(unlocked)
+
+    # We will place entries in a cleaned lockfile, removing defunct entries, etc.
+    clean_lockfile = copy.deepcopy(locked)
+    clean_lockfile['tools'] = []
+
+    # As here we add any new tools in.
+    for tool in unlocked['tools']:
+        # If we have an existing locked copy, we'll just use that.
+        locked_tools = [x for x in locked['tools'] if x['name'] == tool['name'] and x['owner'] == tool['owner']]
+        # If there are no copies of it seen in the lockfile, we'll just copy it
+        # over directly, without a reivision. Another script will fix that.
+        if len(locked) == 0:
+            # new tool, just add directly.
+            clean_lockfile['tools'].append(tool)
+            continue
+
+        # Otherwise we hvae one or more locked versions so we'll harmonise +
+        # reduce. Revisions are the only thing that could be variable.
+        # Name/section/owner should not be. If they are, we take original human
+        # edited .yaml file as source of truth.
+
+        revisions = []
+        for locked_tool in locked_tools:
+            for revision in locked_tool.get('revisions', []):
+                revisions.append(revision)
+
+        new_tool = {
+            'name': tool['name'],
+            'owner': tool['owner'],
+            'revisions': sorted(list(set(revisions))),  # Cast to list for yaml serialization
+        }
+        if tool.get('tool_panel_section_id'):
+            new_tool.update({'tool_panel_section_id': tool['tool_panel_section_id']})
+        if tool.get('tool_panel_section_label'):
+            new_tool.update({'tool_panel_section_label': tool['tool_panel_section_label']})
+
+        clean_lockfile['tools'].append(new_tool)
+
+    with open(fn + '.lock', 'w') as handle:
+        yaml.dump(clean_lockfile, handle, default_flow_style=False)
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('fn', type=argparse.FileType('r'), help="Tool.yaml file")
+    parser.add_argument('--dry-run', action='store_true', help="Trust all listed tools in the file, i.e. add the latest changest for them.")
+    args = parser.parse_args()
+    update_file(args.fn.name, dry=args.dry_run)

scripts/update-tool.py

@@ -0,0 +1,72 @@
+import yaml
+import os
+import glob
+import copy
+import argparse
+import logging
+
+from bioblend import toolshed
+
+ts = toolshed.ToolShedInstance(url='https://toolshed.g2.bx.psu.edu')
+
+
+def update_file(fn, owner=None, name=None, without=False):
+    with open(fn + '.lock', 'r') as handle:
+        locked = yaml.safe_load(handle)
+
+    # Update any locked tools.
+    for tool in locked['tools']:
+        # If without, then if it is lacking, we should exec.
+        logging.debug("Examining {owner}/{name}".format(**tool))
+
+        if without:
+            if 'revisions' in tool and not len(tool.get('revisions', [])) == 0:
+                continue
+
+        if not without and owner and tool['owner'] != owner:
+            continue
+
+        if not without and name and tool['name'] != name:
+            continue
+
+        logging.info("Fetching updates for {owner}/{name}".format(**tool))
+
+        try:
+            revs = ts.repositories.get_ordered_installable_revisions(tool['name'], tool['owner'])
+        except Exception as e:
+            print(e)
+            continue
+
+        logging.debug('TS revisions: %s' % ','.join(revs))
+        latest_rev = revs[-1]
+        if latest_rev in tool.get('revisions', []):
+            # The rev is already known, don't add again.
+            continue
+
+        logging.info("Found newer revision of {owner}/{name} ({rev})".format(rev=latest_rev, **tool))
+
+        # Get latest rev, if not already added, add it.
+        if 'revisions' not in tool:
+            tool['revisions'] = []
+
+        if latest_rev not in tool['revisions']:
+            # TS doesn't support utf8 and we don't want to either.
+            tool['revisions'].append(str(latest_rev))
+
+        tool['revisions'] = sorted(list(set( tool['revisions'] )))
+
+    with open(fn + '.lock', 'w') as handle:
+        yaml.dump(locked, handle, default_flow_style=False)
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('fn', type=argparse.FileType('r'), help="Tool.yaml file")
+    parser.add_argument('--owner', help="Repository owner to filter on, anything matching this will be updated")
+    parser.add_argument('--name', help="Repository name to filter on, anything matching this will be updated")
+    parser.add_argument('--without', action='store_true', help="If supplied will ignore any owner/name and just automatically add the latest hash for anything lacking one.")
+    parser.add_argument('--log', choices=('critical', 'error', 'warning', 'info', 'debug'), default='info')
+    args = parser.parse_args()
+    logging.basicConfig(level=getattr(logging, args.log.upper()))
+    update_file(args.fn.name, owner=args.owner, name=args.name, without=args.without)

umsa_tool_list.yaml

@@ -194,12 +194,6 @@ tools:
   - 4e3b2049a4d3
   tool_panel_section_label: ChemicalToolBox
   tool_shed_url: toolshed.g2.bx.psu.edu
-- name: query
-  owner: recetox
-  revisions:
-  - 0369de831b32
-  tool_panel_section_label: ChemicalToolBox
-  tool_shed_url: testtoolshed.g2.bx.psu.edu
 - name: msconvert
   owner: galaxyp
   revisions:
@@ -1196,6 +1190,12 @@ tools:
   - da15e8ea3b28
   tool_panel_section_label: matchms
   tool_shed_url: toolshed.g2.bx.psu.edu
+- name: matchms_remove_spectra
+  owner: recetox
+  revisions:
+  -  80df426e7e47
+  tool_panel_section_label: matchms
+  tool_shed_url: toolshed.g2.bx.psu.edu
 - name: matchms_split
   owner: recetox
   revisions: