[WIP]: changes that accumulate during working on checkm parsers

.travis.yml

@@ -1,32 +1,44 @@
 language: python
 python:
-- '3.6'
+- '3.7'
 install: pip install -r requirements.txt
 script:
 - pip install -q -U setuptools
 - pip install -q -U planemo
 - pip install -q -U flake8
 - pip install -q -U xmldiff
 - pip install -q -r requirements.txt
-- python setup.py install
-- flake8 argparse2tool --ignore=E2,E3,E4,E5,W3,W505
-- PYTHONPATH=$(argparse2tool) python examples/example.py --generate_galaxy_xml > tmp.xml
+- python3 -m pip install --no-deps --no-cache-dir --force-reinstall .
+# temporarily install galaxyxml from branch https://github.com/hexylena/galaxyxml/pull/18
+- git clone -b topic/macros https://github.com/bernt-matthias/galaxyxml.git && cd galaxyxml && python3 -m pip install --no-deps --no-cache-dir --force-reinstall . && cd ..
+- flake8 argparse2tool --ignore=C901,E2,E3,E4,E5,W3,W505
+- PYTHONPATH=$(argparse2tool) python examples/example.py --generate_galaxy_xml -m examples/macros.xml > tmp.xml
 - xmldiff tmp.xml examples/example.xml
-- planemo lint --report_level all --fail_level error --xsd tmp.xml
+- planemo lint --skip tests --report_level all --fail_level error --xsd tmp.xml
 
 # Galaxy tool generation for example with subparsers -- generating one large (invalid) tool
 - echo '<root>' > tmp-sub.xml # wrap in extra level
-- PYTHONPATH=$(argparse2tool) python examples/example-sub.py --generate_galaxy_xml >> tmp-sub.xml
+- PYTHONPATH=$(argparse2tool) python examples/example-sub.py --generate_galaxy_xml -m examples/macros.xml >> tmp-sub.xml
 - echo '</root>' >> tmp-sub.xml
 - xmldiff tmp-sub.xml <(echo "<root>"; cat examples/example-sub.xml; echo "</root>")
 
-# Galaxy tool generation for example with subparsers -- generating separate tools
-- PYTHONPATH=$(argparse2tool) python examples/example-sub.py --generate_galaxy_xml --command foo > tmp-sub-foo.xml
-- PYTHONPATH=$(argparse2tool) python examples/example-sub.py --generate_galaxy_xml --command bar > tmp-sub-bar.xml
-- xmldiff tmp-sub-foo.xml examples/example-sub-foo.xml
-- planemo lint --report_level all --fail_level error --xsd tmp-sub-foo.xml
-- xmldiff tmp-sub-bar.xml examples/example-sub-bar.xml
-- planemo lint --report_level all --fail_level error --xsd tmp-sub-bar.xml
+# Galaxy tool generation for example with subparsers -- generating separate tools (this does not generate macro xml files automatically)
+- PYTHONPATH=$(argparse2tool) python examples/example-sub.py --generate_galaxy_xml -m examples/macros.xml --command foo > tmp-sub-foo.xml
+- PYTHONPATH=$(argparse2tool) python examples/example-sub.py --generate_galaxy_xml -m examples/macros.xml --command bar > tmp-sub-bar.xml
+- xmldiff tmp-sub-foo.xml examples/example-sub/example_sub_foo.xml
+- xmldiff tmp-sub-bar.xml examples/example-sub/example_sub_bar.xml
+
+# Galaxy tool generation for example with subparsers -- generating all tools and macros at once also include multiple macro files
+- mkdir tmp && cp examples/macros*xml tmp/
+- PYTHONPATH=$(argparse2tool) python examples/example-sub.py --generate_galaxy_xml -m examples/macros.xml -m examples/macros_tests.xml --directory tmp
+- ls tmp/
+- xmldiff tmp/example_sub_foo.xml examples/example-sub/example_sub_foo.xml
+- planemo lint --skip tests --report_level all --fail_level error --xsd  tmp/example_sub_foo.xml
+- xmldiff tmp/example_sub_bar.xml examples/example-sub/example_sub_bar.xml
+- planemo lint --skip tests --report_level all --fail_level error --xsd  tmp/example_sub_bar.xml
+- xmldiff tmp/example_sub_baz.xml examples/example-sub/example_sub_baz.xml
+- xmldiff tmp/example_sub_qux.xml examples/example-sub/example_sub_qux.xml
+
 
 - PYTHONPATH=$(argparse2tool) python examples/example.py --generate_cwl_tool > tmp.cwl
 - PYTHONPATH=$(argparse2tool) python examples/example-sub.py --generate_cwl_tool > tmp-sub.cwl
@@ -37,7 +49,6 @@ script:
 - diff tmp-click.cwl examples/example-click.cwl
 before_deploy:
 - sed -i "s/__version__.*/__version__= '${TRAVIS_TAG}'/g" argparse2tool/__init__.py
-deploy:
 deploy:
   provider: pypi
   username: __token__

argparse2tool/__init__.py

@@ -48,3 +48,11 @@ def load_conflicting_package(name, not_name, module_number):
             imp.load_module(random_name, f, pathname, desc)
             return sys.modules[random_name]
     return None
+
+
+def remove_extension(name):
+    if name is not None:
+        name = name.replace('.py', '')
+        name = name.replace('-', '_')
+        name = name.replace(' ', '_')
+    return name

argparse2tool/cmdline2gxml/__init__.py

@@ -23,5 +23,9 @@ def __init__(self):
 
     def process_arguments(self):
         self.parser.add_argument('--generate_galaxy_xml', action='store_true')
+        self.parser.add_argument('-d', '--directory',
+                                 help='Directory to store tool descriptions')
+        self.parser.add_argument('-m', '--macro', action="append",
+                                 help='A global macro file to include in all tools')
         self.parser.add_argument('--command', action='store', default="")
         return vars(self.parser.parse_args())

argparse2tool/dropins/argparse/__init__.py

@@ -1,6 +1,11 @@
+import os.path
 import re
+import shutil
 import sys
-from argparse2tool import load_argparse
+from argparse2tool import (
+    load_argparse,
+    remove_extension
+)
 from argparse2tool.cmdline2gxml import Arg2GxmlParser
 from argparse2tool.cmdline2cwl import Arg2CWLParser
 
@@ -21,6 +26,15 @@
     setattr(__selfmodule__, x, getattr(ap, x))
 
 tools = []
+# set of prog names where the argparser actually should be represented
+# as a macro, i.e. XYZ will be in the set if there is a
+# ArgumentParser(..., parents=[XYZ], ...)
+macros = set()
+
+# mapping tools to list of required macros (which are actually identical to the
+# parents parameter passed ArgumentParser. but I could not find out how this is
+# stored in the created ArgumentParser objects
+used_macros = dict()
 
 
 class ArgumentParser(ap.ArgumentParser):
@@ -37,10 +51,20 @@ def __init__(self,
                  argument_default=None,
                  conflict_handler='error',
                  add_help=True):
+        global macros
+        global used_macros
 
         self.argument_list = []
         self.argument_names = []
         tools.append(self)
+        if len(parents) > 0:
+            p = set([remove_extension(_.prog) for _ in parents])
+            macros = macros.union(p)
+            used_macros[remove_extension(prog)] = p
+
+        if '--generate_galaxy_xml' in sys.argv:
+            parents = []
+
         super(ArgumentParser, self).__init__(prog=prog,
                                              usage=usage,
                                              description=description,
@@ -109,6 +133,8 @@ def parse_args_cwl(self, *args, **kwargs):
                                 tool.inputs.append(cwlt_parameter)
                                 if isinstance(cwlt_parameter, cwlt.OutputParam):
                                     tool.outputs.append(cwlt_parameter)
+                        else:
+                            print("%s not implemented (%s)" % (argument_type, result.option_strings), file=sys.stderr)
 
                     if argp.epilog is not None:
                         tool.description += argp.epilog
@@ -128,6 +154,31 @@ def parse_args_cwl(self, *args, **kwargs):
         sys.exit(0)
 
     def parse_args_galaxy(self, *args, **kwargs):
+        global used_macros
+
+        directory = kwargs.get('directory', None)
+        macro = kwargs.get('macro', None)
+
+        # copy macros to destination dir
+        if directory and macro:
+            for i, m in enumerate(macro):
+                macro[i] = os.path.basename(m)
+                shutil.copyfile(m, os.path.join(directory, macro[i]))
+
+        # since macros can also make use of macros (i.e. the parent relation
+        # specified in the arguments can be nester) we need to extend the
+        # used macros such that really all are included
+        ext_used_macros = dict()
+        for tool, macros in used_macros.items():
+            ext_used_macros[tool] = set()
+            q = list(macros)
+            while len(q) > 0:
+                m = q.pop()
+                if m in used_macros:
+                    q.extend(used_macros[m])
+                ext_used_macros[tool].add(m)
+        used_macros = ext_used_macros
+
         for argp in tools:
             # make subparser description out of its help message
             if argp._subparsers:
@@ -139,53 +190,119 @@ def parse_args_galaxy(self, *args, **kwargs):
                         subparser.choices[choice_action.dest].description = choice_action.help
             else:
                 if kwargs.get('command', argp.prog) in argp.prog:
-                    data = self._parse_args_galaxy_argp(argp)
-                    print(data)
+                    data = self._parse_args_galaxy_argp(argp, macro)
+                    if directory:
+                        if directory[-1] != '/':
+                            directory += '/'
+                        filename = remove_extension(argp.prog) + ".xml"
+                        filename = directory + filename
+                        with open(filename, 'w') as f:
+                            f.write(data)
+                    else:
+                        print(data)
                 else:
                     continue
         sys.exit(0)
 
-    def _parse_args_galaxy_argp(self, argp):
+    def _parse_args_galaxy_argp(self, argp, macro):
+        global macros
+        global used_macros
+
         try:
             version = self.print_version() or '1.0'
         except AttributeError:  # handle the potential absence of print_version
             version = '1.0'
-        tool = gxt.Tool(argp.prog,
-                        argp.prog.replace(" ", "_"),
-                        version,
-                        argp.description,
-                        "python "+argp.prog,
-                        interpreter=None,
-                        version_command='python %s --version' % argp.prog)
+        print("_parse_args_galaxy_argp _subparsers %s" % argp._subparsers)
+        prog = remove_extension(argp.prog)
 
-        inputs = gxtp.Inputs()
-        outputs = gxtp.Outputs()
+        # tid = argp.prog.split()[-1]
+
+        # get the list of file names of the used macros
+        mx = used_macros.get(prog, [])
+        mx = sorted(["%s.xml" % _.split(" ")[-1] for _ in mx])
+
+        if prog not in macros:
+            tpe = gxt.Tool
+            if macro:
+                mx.extend(macro)
+        else:
+            tpe = gxt.MacrosTool
+
+        tool = tpe(prog,
+                   prog.replace(" ", "_"),
+                   version,
+                   argp.description,
+                   "python "+argp.prog,
+                   interpreter=None,
+                   version_command='python %s --version' % argp.prog,
+                   macros=mx)
+
+        inputs = tool.inputs
+        outputs = tool.outputs
+        sections = dict()
 
         at = agt.ArgparseGalaxyTranslation()
-        # Only build up arguments if the user actually requests it
-        for result in argp.argument_list:
+
+        for group in argp._action_groups:
+            if group in [argp._positionals, argp._optionals]:
+                continue
+            argument_type = group.__class__.__name__
+            methodToCall = getattr(at, argument_type)
+            sections[group] = methodToCall(group, tool=tool)
+
+        for action in argp._actions:
             # I am SO thankful they return the argument here. SO useful.
-            argument_type = result.__class__.__name__
+            argument_type = action.__class__.__name__
             # http://stackoverflow.com/a/3071
             if hasattr(at, argument_type):
                 methodToCall = getattr(at, argument_type)
-                gxt_parameter = methodToCall(result, tool=tool)
-                if gxt_parameter is not None:
-                    if isinstance(gxt_parameter, gxtp.InputParameter):
-                        inputs.append(gxt_parameter)
-                    else:
-                        outputs.append(gxt_parameter)
+                gxt_parameter = methodToCall(action, tool=tool)
+                ## print(action, gxt_parameter)
+                print(action.option_strings, action.container)
+                if gxt_parameter is None:  # e.g. for help and version actions
+                    continue
+                if not isinstance(gxt_parameter, gxtp.InputParameter):
+                    outputs.append(gxt_parameter)
+                elif action.container in sections:
+                    sections[action.container].append(gxt_parameter)
+                else:
+                    inputs.append(gxt_parameter)
+            else:
+                print("%s not implemented (%s)" % (argument_type, action.option_strings), file=sys.stderr)
+
+        for section in sections:
+            inputs.append(sections[section])
+
 
-        # TODO: replace with argparse-esque library to do this.
-        stdout = gxtp.OutputData('default', 'txt')
-        stdout.command_line_override = '> $default'
-        outputs.append(stdout)
+#         print("argp._action_groups %s" % argp._action_groups)
+#         # Only build up arguments if the user actually requests it
+#         for result in argp.argument_list:
+#             # I am SO thankful they return the argument here. SO useful.
+#             argument_type = result.__class__.__name__
+#             print("_parse_args_galaxy_argp %s type %s [%s]" % (getattr(result, "title", "no title"), argument_type, hasattr(at, argument_type)))
+#             # http://stackoverflow.com/a/3071
+#             if hasattr(at, argument_type):
+#                 methodToCall = getattr(at, argument_type)
+#                 gxt_parameter = methodToCall(result, tool=tool)
+#                 if gxt_parameter is not None:
+#                     if isinstance(gxt_parameter, gxtp.InputParameter):
+#                         inputs.append(gxt_parameter)
+#                     else:
+#                         outputs.append(gxt_parameter)
+
+        if prog in used_macros:
+            for m in sorted(used_macros[prog]):
+                inputs.append(gxtp.ExpandIO(m + "_inmacro"))
+                outputs.append(gxtp.ExpandIO(m + "_outmacro"))
+
+        if prog not in macros:
+            # TODO: replace with argparse-esque library to do this.
+            stdout = gxtp.OutputData('default', 'txt')
+            stdout.command_line_override = '> $default'
+            outputs.append(stdout)
 
-        tool.inputs = inputs
-        tool.outputs = outputs
         if argp.epilog is not None:
             tool.help = argp.epilog
         else:
             tool.help = "TODO: Write help"
-
         return tool.export()

argparse2tool/dropins/argparse/argparse_cwl_translation.py

@@ -77,6 +77,12 @@ def __args_from_nargs(self, param):
 
         return param
 
+    def _VersionAction(self, param, tool=None):
+        pass
+
+    def _HelpAction(self, param, tool=None):
+        pass
+
     def _StoreAction(self, param):
         param = self.__args_from_nargs(param)
         cwlparam = self.__cwl_param_from_type(param)
@@ -110,6 +116,11 @@ def __StoreBoolAction(self, param):
         cwlparam = self.__cwl_param_from_type(param)
         return cwlparam
 
+    def _StoreConstAction(self, param):
+        param.type = bool
+        cwlparam = self.__cwl_param_from_type(param)
+        return cwlparam
+
     @staticmethod
     def get_cwl_type(py_type):
         """