From d3b81c6506f78763dc8a773c45e8b3b997219662 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 10 Jan 2021 10:37:06 -0500 Subject: [PATCH 1/2] Bring in usegalaxy-tools lockfile update script. --- setup.py | 1 + src/ephemeris/shed_lockfile_update.py | 95 +++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/ephemeris/shed_lockfile_update.py diff --git a/setup.py b/setup.py index b75a84d2..f909fc92 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ def get_var(var_name): setup-data-libraries=ephemeris.setup_data_libraries:main galaxy-wait=ephemeris.sleep:main install_tool_deps=ephemeris.install_tool_deps:main + shed-lockfile-update=ephemeris.shed_lockfile_update:main ''' PACKAGE_DATA = { # Be sure to update MANIFEST.in for source dist. diff --git a/src/ephemeris/shed_lockfile_update.py b/src/ephemeris/shed_lockfile_update.py new file mode 100644 index 00000000..dde035c0 --- /dev/null +++ b/src/ephemeris/shed_lockfile_update.py @@ -0,0 +1,95 @@ +# Script originally developed as part of the usegalaxy-tools project. +# https://github.com/galaxyproject/usegalaxy-tools/blob/master/scripts/update-tool.py +import argparse +import logging +import os +import yaml +from collections import defaultdict + +from bioblend import toolshed + +DEFAULT_TOOL_SHED_URL = 'https://toolshed.g2.bx.psu.edu' + + +class ToolSheds(defaultdict): + default_factory = toolshed.ToolShedInstance + + def __missing__(self, key): + logging.debug('Creating new ToolShedInstance for URL: %s', key) + return self.default_factory(url=key) + + +tool_sheds = ToolSheds() + + +def update_file(fn, owner=None, name=None, without=False): + locked_in_path = fn + ".lock" + if not os.path.exists(locked_in_path): + logging.info("Lockfile doesn't exist yet, starting with source as base.") + locked_in_path = fn + + with open(locked_in_path, 'r') as handle: + locked = yaml.safe_load(handle) + + # Update any locked tools. + for tool in locked['tools']: + # If without, then if it is lacking, we should exec. + logging.debug("Examining {owner}/{name}".format(**tool)) + + if without: + if 'revisions' in tool and not len(tool.get('revisions', [])) == 0: + continue + + if not without and owner and tool['owner'] not in owner: + continue + + if not without and name and tool['name'] != name: + continue + + ts_url = tool.get('tool_shed_url', DEFAULT_TOOL_SHED_URL) + if ts_url != DEFAULT_TOOL_SHED_URL: + logging.warning('Non-default Tool Shed URL for %s/%s: %s', tool['owner'], tool['name'], ts_url) + ts = tool_sheds[ts_url] + + logging.info("Fetching updates for {owner}/{name}".format(**tool)) + + try: + revs = ts.repositories.get_ordered_installable_revisions(tool['name'], tool['owner']) + except Exception as e: + print(e) + continue + + logging.debug('TS revisions: %s' % ','.join(revs)) + latest_rev = revs[-1] + if latest_rev in tool.get('revisions', []): + # The rev is already known, don't add again. + continue + + logging.info("Found newer revision of {owner}/{name} ({rev})".format(rev=latest_rev, **tool)) + + # Get latest rev, if not already added, add it. + if 'revisions' not in tool: + tool['revisions'] = [] + + if latest_rev not in tool['revisions']: + # TS doesn't support utf8 and we don't want to either. + tool['revisions'].append(str(latest_rev)) + + with open(fn + '.lock', 'w') as handle: + yaml.dump(locked, handle, default_flow_style=False) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('fn', type=argparse.FileType('r'), help="Tool.yaml file") + parser.add_argument('--owner', action='append', help="Repository owner to filter on, anything matching this will be updated. Can be specified multiple times") + parser.add_argument('--name', help="Repository name to filter on, anything matching this will be updated") + parser.add_argument('--without', action='store_true', help="If supplied will ignore any owner/name and just automatically add the latest hash for anything lacking one.") + parser.add_argument('--log', choices=('critical', 'error', 'warning', 'info', 'debug'), default='info') + args = parser.parse_args() + logging.basicConfig(level=getattr(logging, args.log.upper())) + update_file(args.fn.name, owner=args.owner, name=args.name, without=args.without) + + +if __name__ == '__main__': + main() From 215c0b6c38db5ced547ff4797523e19143dde4c1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 10 Jan 2021 11:14:49 -0500 Subject: [PATCH 2/2] Linting fixes for shed_lockfile_update.py. --- src/ephemeris/shed_lockfile_update.py | 41 ++++++++++++++++++++------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/ephemeris/shed_lockfile_update.py b/src/ephemeris/shed_lockfile_update.py index dde035c0..be1fe28c 100644 --- a/src/ephemeris/shed_lockfile_update.py +++ b/src/ephemeris/shed_lockfile_update.py @@ -3,12 +3,16 @@ import argparse import logging import os -import yaml from collections import defaultdict +import yaml from bioblend import toolshed DEFAULT_TOOL_SHED_URL = 'https://toolshed.g2.bx.psu.edu' +FOUND_NEWER_MESSAGE = "Found newer revision of {owner}/{name} ({rev})" +FETCHING_UPDATE_MESSAGE = "Fetching updates for {owner}/{name}" +INIT_LOCKFILE_MESSAGE = "Lockfile doesn't exist yet, starting with source as base." +NON_DEFAULT_TS_MESSAGE = 'Non-default Tool Shed URL for %s/%s: %s' class ToolSheds(defaultdict): @@ -25,7 +29,7 @@ def __missing__(self, key): def update_file(fn, owner=None, name=None, without=False): locked_in_path = fn + ".lock" if not os.path.exists(locked_in_path): - logging.info("Lockfile doesn't exist yet, starting with source as base.") + logging.info(INIT_LOCKFILE_MESSAGE) locked_in_path = fn with open(locked_in_path, 'r') as handle: @@ -48,10 +52,10 @@ def update_file(fn, owner=None, name=None, without=False): ts_url = tool.get('tool_shed_url', DEFAULT_TOOL_SHED_URL) if ts_url != DEFAULT_TOOL_SHED_URL: - logging.warning('Non-default Tool Shed URL for %s/%s: %s', tool['owner'], tool['name'], ts_url) + logging.warning(NON_DEFAULT_TS_MESSAGE, tool['owner'], tool['name'], ts_url) ts = tool_sheds[ts_url] - logging.info("Fetching updates for {owner}/{name}".format(**tool)) + logging.info(FETCHING_UPDATE_MESSAGE.format(**tool)) try: revs = ts.repositories.get_ordered_installable_revisions(tool['name'], tool['owner']) @@ -65,7 +69,7 @@ def update_file(fn, owner=None, name=None, without=False): # The rev is already known, don't add again. continue - logging.info("Found newer revision of {owner}/{name} ({rev})".format(rev=latest_rev, **tool)) + logging.info(FOUND_NEWER_MESSAGE.format(rev=latest_rev, **tool)) # Get latest rev, if not already added, add it. if 'revisions' not in tool: @@ -81,11 +85,28 @@ def update_file(fn, owner=None, name=None, without=False): def main(): parser = argparse.ArgumentParser() - parser.add_argument('fn', type=argparse.FileType('r'), help="Tool.yaml file") - parser.add_argument('--owner', action='append', help="Repository owner to filter on, anything matching this will be updated. Can be specified multiple times") - parser.add_argument('--name', help="Repository name to filter on, anything matching this will be updated") - parser.add_argument('--without', action='store_true', help="If supplied will ignore any owner/name and just automatically add the latest hash for anything lacking one.") - parser.add_argument('--log', choices=('critical', 'error', 'warning', 'info', 'debug'), default='info') + parser.add_argument( + 'fn', type=argparse.FileType('r'), help="Tool.yaml file" + ) + parser.add_argument( + '--owner', + action='append', + help="Repository owner to filter on, anything matching this will be updated. Can be specified multiple times" + ) + parser.add_argument( + '--name', + help="Repository name to filter on, anything matching this will be updated" + ) + parser.add_argument( + '--without', + action='store_true', + help="If supplied will ignore any owner/name and just automatically add the latest hash for anything lacking one." + ) + parser.add_argument( + '--log', + choices=('critical', 'error', 'warning', 'info', 'debug'), + default='info' + ) args = parser.parse_args() logging.basicConfig(level=getattr(logging, args.log.upper())) update_file(args.fn.name, owner=args.owner, name=args.name, without=args.without)