Merge pull request #150 from ctmrbio/develop

Version 0.4.0

Author: boulund

.circleci/config.yml

@@ -0,0 +1,100 @@
+version: 2.0
+
+jobs:
+    setup_snakemake:
+        docker:
+            - image: continuumio/miniconda3:4.7.10
+        working_directory: ~/stag-mwc
+        steps:
+            - run:
+                name: Install Snakemake
+                command: |
+                  conda info -a 
+                  conda config --add channels defaults
+                  conda config --add channels conda-forge
+                  conda config --add channels bioconda
+                  conda create -q -y -n snakemake python=3.7 snakemake=5.6.0
+            - save_cache:
+                key: v1-snakemake-5.6.0-{{ .Environment.CIRCLE_SHA1 }}
+                paths: 
+                    - /opt/conda
+    bundle_dependencies:
+        docker:
+            - image: continuumio/miniconda3:4.7.10
+        working_directory: ~/stag-mwc
+        steps:
+            - run:
+                name: Bundle test data
+                command: |
+                    #git clone https://github.com/ctmrbio/stag-mwc_test_data input
+                    mkdir -pv input
+                    touch input/test1_1.fq.gz input/test1_2.fq.gz
+                    touch input/test2_1.fq.gz input/test2_2.fq.gz
+            - save_cache:
+                key: v1-stag-mwc_test_data-{{ .Environment.CIRCLE_SHA1 }}
+                paths:
+                    - ~/stag-mwc/input
+    checkout_code:
+        docker:
+            - image: continuumio/miniconda3:4.7.10
+        working_directory: ~/stag-mwc
+        steps:
+            - checkout
+            - run:
+                name: Placeholder database files
+                command: |
+                    mkdir -pv ~/stag-mwc/db/hg19
+                    mkdir -pv ~/stag-mwc/db/metaphlan2
+                    touch ~/stag-mwc/db/hg19/taxo.k2d
+                    touch ~/stag-mwc/db/metaphlan2/test.1.bt2
+            - save_cache:
+                key: v1-stag-mwc_with_dbs-{{ .Revision }}
+                paths:
+                    - ~/stag-mwc
+    validate_syntax_and_dag:
+        docker:
+            - image: continuumio/miniconda3:4.7.10
+        working_directory: ~/stag-mwc
+        steps:
+            - restore_cache:
+                key: v1-snakemake-5.6.0-{{ .Environment.CIRCLE_SHA1 }}
+            - restore_cache:
+                key: v1-stag-mwc_test_data-{{ .Environment.CIRCLE_SHA1 }}
+            - restore_cache:
+                key: v1-stag-mwc_with_dbs-{{ .Revision }}
+            - run:
+                name: Validate syntax and DAG
+                command: |
+                    sed -i 's/assess_depth: False/assess_depth: True/' config.yaml
+                    sed -i 's/sketch_compare: False/sketch_compare: True/' config.yaml
+                    sed -i 's/kaiju: False/kaiju: True/' config.yaml
+                    sed -i 's/kraken2: False/kraken2: True/' config.yaml
+                    sed -i 's/metaphlan2: False/metaphlan2: True/' config.yaml
+                    sed -i 's/humann2: False/humann2: True/' config.yaml
+                    sed -i 's/antibiotic_resistance: False/antibiotic_resistance: True/' config.yaml
+                    sed -i 's/assembly: False/assembly: True/' config.yaml
+                    sed -i 's/binning: False/binning: True/' config.yaml
+                    sed -i 's|db_path: \"\"|db_path: \"db/hg19\"|' config.yaml
+                    sed -i 's|db: \"\"|db: \"db\"|' config.yaml
+                    sed -i 's|bt2_db_dir: \"\"|bt2_db_dir: \"db/metaphlan2\"|' config.yaml
+                    sed -i 's|bt2_index: \"\"|bt2_index: \"test\"|' config.yaml
+                    sed -i 's|_db: \"\"|_db: \"db\"|' config.yaml
+                    sed -i 's|    index: \"\"|    index: \"db\"|' config.yaml
+                    sed -i 's|kmer_distrib: \"\"|kmer_distrib: \"db\"|' config.yaml
+                    cat config.yaml
+                    source activate snakemake
+                    snakemake --use-conda --dryrun
+
+workflows:
+    version: 2
+    validate_syntax_and_dag:
+        jobs:
+            - setup_snakemake
+            - bundle_dependencies
+            - checkout_code
+            - validate_syntax_and_dag:
+                requires:
+                    - setup_snakemake
+                    - bundle_dependencies
+                    - checkout_code
+

.editorconfig

@@ -13,6 +13,91 @@ files), and the patch version is typically incremented for any set of changes
 committed to the master branch that does not trigger any of the aforementioned
 situations.
 
+## [0.4.0] Unreleased
+### Added
+- Added resource limiter for HUMAnN2 due to its intense use of huge temporary
+  files in the output folder. Activated with --resources humann2=X, where X is
+  the max number of parallel instances of humann2 to run.
+- Added groot report parameters `covcutoff` and `lowcov` to config file.
+- Added automatic plot of proportion human reads. Included in report.
+- Added assembly and binning using MetaWRAP.
+- Added the possibility to run in Singularity with conda using 
+  `--use-singularity --use-conda`.
+- Added more MetaPhlAn2 data in output report.
+- Added HUMAnN2 summary tables to output report.
+- Added combined table and Krona plot for Kraken2 to output report.
+- Added metagenomic assembly, binning and "blobology", using MEGAHIT or SPAdes,
+  with binning using CONCOCT and MetaBat (MaxBin2 is not working), implemented
+  via MetaWrap.
+- Added KrakenTools from Jennifer Lu under the MIT license.
+- Added basic syntax and DAG validation test in CircleCI
+- Added possibility to skip read QC and/or host removal: will symlink relevant files
+  files into the relevant output directories so Snakemake can continue
+  without performing read QC and/or host removal.
+- Added MultiQC, mainly to summarize fastp logs.
+- Added Bracken abundance estimation on Kraken2 report files; added Bracken to
+  the StaG conda environment. Also added Bracken abundance filtering rules so
+  users can include/exclude certain taxa.
+- Added "all_samples" summary output files in a more common format for all
+  taxonomic profilers, called `mpa_style`. They are not identical, but very
+  similar, with full lineage listings for all detected taxa.
+- Added pigz v2.4 to the main StaG conda environment.
+- Added summary with read counts passing preprocessing steps as a table and
+  basic line plot. Only runs if both read QC and host removal are performed.
+
+### Fixed
+- Fixed bug in Slurm profile handling of cancelled/failed jobs.
+- MetaPhlAn2 rule now correctly detect if no database path has been entered in
+  the config file.
+- HUMAnN2 rule now correctly detects if no database path has been entered in
+  the config file.
+- Kraken2 rule now correctly detects if no database is available at the path
+  given in the config file.
+
+### Changed
+- Updated Python to 3.7 in main stag-mwc conda environment.
+- Updated Kaiju to 1.7.2.
+- Updated BBMap to 38.68.
+- Updated sambamba to 0.7.0.
+- Updated Kraken2 to 2.0.8_beta.
+- Updated seaborn to 0.8.1, added fastcluster to main stag-mwc conda env, installed via pip.
+- Updated MetaPhlAn2 to 2.96.1.
+- Updated HUMAnN2 to 2.8.1.
+- Updated GROOT to v0.8.5.
+- Updated plot_metaphlan2_heatmap.py to 0.3.
+- Changed output filenames ending with `.tsv` to `.txt` to avoid pretty HTML
+  representations in report.
+- Replaced BBMap-based host removal with Kraken2, substantially reducing time
+  and resources requirements. 
+- Added read length window filter before groot alignment step.
+- Change logdir of remove_human rule to LOGDIR/remove_human instead of
+  OUTDIR/logs/remove_human.
+- Improved make_count_table.py so it can use TSV annotation files with multiple
+  columns. Added config setting for which columns to include.
+- Cleaned up sketch comparison cluster heatmap plotting script, making it more 
+  robust to variations in output from different BBTools versions.
+- Changed the call to merge_metaphlan_tables.py due to undocumented
+  CLI change in latest conda version.
+- Changed Kaiju summary report output filenames.
+- Split biobakery environment into metaphlan2 and humann2 so users only
+  interested in MetaPhlAn2 do not have to download the huge HUMAnN2.
+- Replaced the outdated metaphlan_hclust_heatmap.py with a custom
+  plot_metaphlan2_heatmap.py script.
+- Defined some low-impact summary and plotting rules as localrules.
+- Reworded all rules relating to human removal to "host removal", and changed output folder
+  structure accordingly.
+- Renamed output folder and file names for quality and adapter trimming.
+- Set Kraken2 --confidence to 0.1 by default.
+- Adjusted HUMAnN2 cores to 20 (up from 8).
+- Adjusted MetaPhlAn2 cores to 5 (up from 4).
+
+
+### Removed
+- Removed outdated database download rules for Centrifuge, MetaPhlAn2, Kaiju,
+  Kraken2, HUMAnN2.
+- Replaced FastQC + BBDuk with fastp adapter trimming and quality filtering.
+- Removed support for Centrifuge
+
 
 ## [0.3.2-dev]
 ### Added
@@ -48,9 +133,9 @@ situations.
 - Substantial improvements to Rackham Slurm profile, focusing on better Slurm
   log handling.
 - A few low-impact rules that can be run locally are now declared as localrules.
-- Replaced MEGARes antibiotic resistance gene mapping with GROOT resistance
-  gene profiling using gene variation graphs, using a default database based on
-  arg-annot.
+- Replaced MEGARes antibiotic resistance gene mapping with Groot resistance gene
+  profiling using gene variation graphs.
+- Increased resource requirements for remove_human step in Rackham cluster profile.
 - Added clustered sketch comparison output heatmap.
 - Updated MetaPhlAn2 to version 2.7.8, with corresponding changes to config file.
 

.travis.yml

@@ -2,19 +2,16 @@
 
 [![DOI](https://zenodo.org/badge/125840716.svg)](https://zenodo.org/badge/latestdoi/125840716)
 [![Snakemake](https://img.shields.io/badge/snakemake-≥4.8.1-brightgreen.svg)](https://snakemake.bitbucket.io)
-<!--[![Build Status](https://travis-ci.org/snakemake-workflows/mwc.svg?branch=master)](https://travis-ci.org/snakemake-workflows/mwc) -->
+[![CircleCI](https://circleci.com/gh/ctmrbio/stag-mwc/tree/master.svg?style=svg)](https://circleci.com/gh/ctmrbio/stag-mwc/tree/master)
+
 ![StaG mwc logo](docs/source/img/stag_head_text.png "StaG mwc")
 
 This repo contains the code for a Snakemake workflow of the StaG Metagenomic
 Workflow Collaboration (mwc). Currently, the project focus is a barebones
 metagenomics analysis workflow to produce primary output files from several
 different metagenomic analysis tools. 
 
-## Authors
-
-* Fredrik Boulund (@boulund)
-* Lisa Olsson (@lis4matilda)
-* (your name here)
+Go to https://stag-mwc.readthedocs.org for the full documentation.
 
 ## Usage
 
@@ -26,7 +23,7 @@ StaG-mwc. Most people would probably want to install
 base environment. Conda will automatically install the required versions of 
 all tools required to run StaG-mwc.
 
-### Step 1: Install workflow
+### Step 1: Clone workflow
 To use StaG-mwc, you need a local copy of the workflow repository. Start by
 making a clone of the repository: 
 
@@ -62,17 +59,10 @@ documentation](https://snakemake.readthedocs.io) for further details on how to
 run Snakemake workflows on other types of cluster resources.
 
 
-### Automatic database download
-The workflow offers steps that can automatically download the required
-reference databases. Note that this step is normally only required once, as
-previously downloaded databases are reused. See the 
-[official documentation](https://stag-mwc.readthedocs.org) for more information.
-
-
 ## Testing
-Tests are currently not implemented. The ambition is that StaG-mwc will contain
-extensive tests to verify functionality. We plan to implement automated linting
-and testing on a small test data set via continuous integration.
+A very basic continuous integration test is currently in place. It merely
+validates the syntax by trying to let Snakemake build the dependency graph if
+all outputs are activated.
 
 
 ## Contributing
@@ -83,5 +73,10 @@ If you intend to modify or further develop this workflow, you are welcome to
 fork this reposity. Please consider sharing potential improvements via a pull
 request.
 
+## Citing
+If you find StaG-mwc useful in your research, please cite the Zenodo DOI:
+https://zenodo.org/record/1483891
+
+
 # Logo attribution
 <a href="https://www.freepik.com/free-photos-vectors/animal">Animal vector created by Patrickss - Freepik.com</a>