Merge branch 'main' into streamflow_translator

Author: jeff-yc-wong

.github/workflows/build.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
+        python-version: [ "3.11", "3.12", "3.13" ]
     steps:
       - uses: actions/checkout@v4
 
@@ -18,15 +18,26 @@ jobs:
 
       - name: Install dependencies
         run: |
+          sudo apt-get update
           sudo apt-get install -y python3-sphinx sphinx-rtd-theme-common
           pip install sphinx_rtd_theme recommonmark pytest pytest-cov
+          pip install --upgrade setuptools
+
+      - name: Install utilities for testing
+        run: |
+          sudo apt-get update
+          sudo apt-get install stress-ng
+          sudo apt-get install graphviz libgraphviz-dev
+          pip install docker
+          pip install pygraphviz
+          pip install pydot
 
       - name: Check package install
         run: |
-          pip install -e .
+          pip install .
 
       - name: Run tests
-        run: pytest -m unit --cov=wfcommons tests/
+        run: python3 -m pytest -s -v -m unit --cov=wfcommons tests/
 
       - name: Upload coverage
         if: github.ref == 'refs/heads/main'

.gitignore

@@ -55,4 +55,8 @@ bin/gpu_benchmark
 /wfcommons/wfc-1.3_to_dask/out
 /wfcommons/wfc-1.3_to_dask/htmlcov
 
+# Coverage stuff
+.coverage
+coverage.xml
+
 test/

MANIFEST.in

@@ -1,3 +1,6 @@
+graft wfcommons/wfchef/recipes/rnaseq/microstructures/**
+graft wfcommons/wfchef/recipes/rnaseq/microstructures
+graft wfcommons/wfchef/recipes/rnaseq
 graft wfcommons/wfchef/recipes/epigenomics/microstructures/**
 graft wfcommons/wfchef/recipes/epigenomics/microstructures
 graft wfcommons/wfchef/recipes/epigenomics

README.md

@@ -21,7 +21,7 @@ This Python package provides a collection of tools for:
 ## Installation
 
 WfCommons is available on [PyPI](https://pypi.org/project/wfcommons).
-WfCommons requires Python3.9+ and has been tested on Linux and macOS.
+WfCommons requires Python3.11+ and has been tested on Linux and MacOS.
 
 ### Installation using pip
 

bin/wfbench

@@ -341,7 +341,7 @@ def begin_flowcept(args):
                  bundle_exec_id=args.workflow_id,
                  start_persistence=False, save_workflow=False)
     f.start()
-    t = FlowceptTask(workflow_id=args.workflow_id, used={**args.__dict__})
+    t = FlowceptTask(task_id=f"{args.workflow_id}_{args.name}", workflow_id=args.workflow_id, used={**args.__dict__})
     return f, t
 
 
@@ -457,7 +457,11 @@ def main():
         if args.time:
             time.sleep(int(args.time))
             for proc in procs:
-                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+                if isinstance(proc, multiprocessing.Process):
+                    if proc.is_alive():
+                        proc.terminate()
+                elif isinstance(proc, subprocess.Popen):
+                    proc.terminate()
         else:
             for proc in procs:
                 if isinstance(proc, subprocess.Popen):

docs/source/dev_api_wfbench.rst

@@ -15,7 +15,7 @@ wfcommons.wfbench.bench
    :noindex:
 
 wfcommons.wfbench.translator.nextflow
-------------------------------------
+-------------------------------------
 
 .. automodule:: wfcommons.wfbench.translator.nextflow
    :members:

docs/source/generating_workflow_benchmarks.rst

@@ -88,8 +88,32 @@ description in :ref:`json-format-label`.
     to execute memory-intensive threads. Therefore, it is crucial to ensure that 
     :code:`stress-ng` is installed on all worker nodes.
 
+
+Dask
+++++++++
+`Dask <https://www.dask.org/>`_ is an open-source library for parallel computing
+in Python. It makes it possible to easily implement and execute workflows local machines, HPC cluster schedulers, and cloud-based
+and container-based environments. Below, we provide an example on how to generate
+workflow benchmark for running with Dask::
+
+    import pathlib
+
+    from wfcommons import BlastRecipe
+    from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator
+
+    # create a workflow benchmark object to generate specifications based on a recipe
+    benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=500)
+
+    # generate a specification based on performance characteristics
+    benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=0.6)
+
+    # generate a Dask workflow
+    translator = DaskTranslator(benchmark.workflow)
+    translator.translate(output_folder=pathlib.Path("./dask-wf/""))
+
 Nextflow
 ++++++++
+
 `Nextflow <https://www.nextflow.io/>`_ is a workflow management system that enables
 the development of portable and reproducible workflows. It supports deploying workflows
 on a variety of execution platforms including local, HPC schedulers, and cloud-based
@@ -117,29 +141,16 @@ workflow benchmark for running with Nextflow::
     that depend on another instance of the same abstract task. Thus, the translator
     fails when you try to translate a workflow with iterations.
 
-Dask
-++++++++
-`Dask <https://www.dask.org/>`_ is an open-source library for parallel computing
-in Python. It makes it possible to easily implement and execute workflows local machines, HPC cluster schedulers, and cloud-based
-and container-based environments. Below, we provide an example on how to generate
-workflow benchmark for running with Dask::
-
-    import pathlib
-
-    from wfcommons import BlastRecipe
-    from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator
-
-    # create a workflow benchmark object to generate specifications based on a recipe
-    benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=500)
-
-    # generate a specification based on performance characteristics
-    benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=0.6)
-
-    # generate a Dask workflow
-    translator = DaskTranslator(benchmark.workflow)
-    translator.translate(output_folder=pathlib.Path("./dask-wf/""))
-
+.. note::
+    
+    If you plan to run Nextflow on an HPC system using Slurm, we **strongly
+    recommend** using the `HyperQueue <https://github.com/It4innovations/hyperqueue>`_ 
+    executor. HyperQueue efficiently distributes workflow tasks across all allocated 
+    compute nodes, improving scalability and resource utilization.
 
+    The :class:`~wfcommons.wfbench.translator.nextflow.NextflowTranslator`
+    class includes functionality to automatically generate a Slurm script 
+    template for running the workflow on HPC systems.
 
 Pegasus
 +++++++
@@ -175,6 +186,31 @@ for running with Pegasus::
     the :code:`lock_files_folder` parameter when using 
     :meth:`~wfcommons.wfbench.bench.WorkflowBenchmark.create_benchmark`.
 
+PyCOMPSs
+++++++++
+
+`PyCOMPSs <https://compss.bsc.es/>`_ is a programming model and runtime that 
+enables the parallel execution of Python applications on distributed computing 
+infrastructures. It allows developers to define tasks using simple Python 
+decorators, automatically handling task scheduling, data dependencies, and 
+resource management.. Below, we provide an example on how to generate workflow 
+benchmark for running with PyCOMPSs::
+
+    import pathlib
+
+    from wfcommons import CyclesRecipe
+    from wfcommons.wfbench import WorkflowBenchmark, PyCompssTranslator
+
+    # create a workflow benchmark object to generate specifications based on a recipe
+    benchmark = WorkflowBenchmark(recipe=CyclesRecipe, num_tasks=200)
+
+    # generate a specification based on performance characteristics
+    benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=500, data=1000, percent_cpu=0.8)
+
+    # generate a PyCOMPSs workflow
+    translator = PyCompssTranslator(benchmark.workflow)
+    translator.translate(output_folder=pathlib.Path("./pycompss-wf/"))
+
 Swift/T
 +++++++
 

docs/source/generating_workflows.rst

@@ -25,6 +25,7 @@ synthetic workflow instances. The current list of available workflow recipes inc
 - :class:`~wfcommons.wfchef.recipes.epigenomics.recipe.EpigenomicsRecipe`: :code:`from wfcommons.wfchef.recipes import EpigenomicsRecipe`
 - :class:`~wfcommons.wfchef.recipes.genome.recipe.GenomeRecipe`: :code:`from wfcommons.wfchef.recipes import GenomeRecipe`
 - :class:`~wfcommons.wfchef.recipes.montage.recipe.MontageRecipe`: :code:`from wfcommons.wfchef.recipes import MontageRecipe`
+- :class:`~wfcommons.wfchef.recipes.rnaseq.recipe.RnaseqRecipe`: :code:`from wfcommons.wfchef.recipes import RnaseqRecipe`
 - :class:`~wfcommons.wfchef.recipes.seismology.recipe.SeismologyRecipe`: :code:`from wfcommons.wfchef.recipes import SeismologyRecipe`
 - :class:`~wfcommons.wfchef.recipes.soykb.recipe.SoykbRecipe`: :code:`from wfcommons.wfchef.recipes import SoykbRecipe`
 - :class:`~wfcommons.wfchef.recipes.srasearch.recipe.SrasearchRecipe`: :code:`from wfcommons.wfchef.recipes import SrasearchRecipe`

docs/source/images/wfcommons.png

@@ -10,7 +10,7 @@ representative synthetic workflow instances, and generating realistic
 workflow benchmark specifications.
 
 .. figure::  images/wfcommons.png
-   :scale:   80 %
+   :scale:   70 %
    :align:   center
 
    The WfCommons conceptual architecture.