diff --git a/AlignmentAndQCWorkflows.jar b/AlignmentAndQCWorkflows.jar index 14fa7d4..d5ff0ab 100644 Binary files a/AlignmentAndQCWorkflows.jar and b/AlignmentAndQCWorkflows.jar differ diff --git a/AlignmentAndQCWorkflows_1.2.73-205.iml b/AlignmentAndQCWorkflows_1.2.73-205.iml new file mode 100644 index 0000000..992fb11 --- /dev/null +++ b/AlignmentAndQCWorkflows_1.2.73-205.iml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/AlignmentAndQCWorkflows_1.2.73.iml b/AlignmentAndQCWorkflows_1.2.73.iml deleted file mode 100644 index ede7797..0000000 --- a/AlignmentAndQCWorkflows_1.2.73.iml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/README.md b/README.md index 2f80131..23bed50 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,17 @@ The original script with a documentation of the underlying ideas can be found [h ## Change Logs +* 1.2.73-205 (branch-specific change) + - minor: Added `bwaPostAltJsK8Options` to allow setting opt K8 options for `bwa-postalt.js` + - minor: Added `SAMPESORT_MEMSIZE` to allow reducing memory for SAM sorting. Default: ~2 GiB. + - minor: Upgrade from COWorkflows 1.2.76 to COWorkflowsBasePlugin 1.4.2. + - minor: Old `FLAG_USE_EXISTING_PAIRED_BAMS` was renamed to `FLAG_USE_ONLY_EXISTING_PAIRED_BAMS` + - minor: Removed `FLAG_RUN_SLIM_WORKFLOW`. Corresponding code is unused for years. + - patch: Little refactorings and groovification of old Java code + +* 1.2.73-204 (branch-specific change) + - minor: Separate BWA from BWAKIT version. Default `BWAKIT_VERSION` to `BWA_VERSION`. Independently set `K8_VERSION` (default 0.2.5). Changed associated module-loading code in environment setup file `tbi-lsf-cluster.sh`. + * 1.2.73-203 (branch-specific change) - minor: Optional ALT-chromosome processing via bwa.kit's `bwa-postaln.js`. * Set `runBwaPostAltJs=true` to activate the ALT chromosome processing. Default: `false`. diff --git a/buildinfo.txt b/buildinfo.txt index 862e3b7..525deae 100644 --- a/buildinfo.txt +++ b/buildinfo.txt @@ -1,4 +1,4 @@ -dependson=COWorkflows:1.2.76 +dependson=COWorkflowsBasePlugin:1.4.2 JDKVersion=1.8 GroovyVersion=2.4 -RoddyAPIVersion=3.0 +RoddyAPIVersion=3.7 diff --git a/resources/analysisTools/qcPipeline/flags_isizes_PEaberrations.pl b/resources/analysisTools/qcPipeline/flags_isizes_PEaberrations.pl index d329493..9945af7 100755 --- a/resources/analysisTools/qcPipeline/flags_isizes_PEaberrations.pl +++ b/resources/analysisTools/qcPipeline/flags_isizes_PEaberrations.pl @@ -119,13 +119,13 @@ if ($_ =~ /^\@/) # there might be a SAM header {next;} $all++; - @help = split ("\t", $_); - $flag = $help[1]; + my ($qname, $flag, $rname, $pos, $mapq, $cigar, $rnext, $pnext, $tlen, $seq, $qual) = + split ("\t", $_); # not unmapped, no duplicate and no secondary/supplementary alignment, and mapqual >= X if (!($flag & 4) && !($flag & 1024) && !($flag & 256) && !($flag & 2048)) { $uniq++; - if ($help[4] >= $minmapq) + if ($mapq >= $minmapq) { # of these, with mapqual >= X $minmapuniq++; @@ -135,15 +135,15 @@ next; } # is the read itself on a wanted chromosome? - if (defined $chroms{$help[2]}) + if (defined $chroms{$rname}) { $onchr++; # and the mate on a wanted chromosome - if (defined $chroms{$help[6]} || $help[6] eq "=") # same chrom is usually indicated by "=" instead of repeating the name + if (defined $chroms{$rnext} || $rnext eq "=") # same chrom is usually indicated by "=" instead of repeating the name { $both++; # paired end aberration: mate also has to be mapped, on a different chrom - if (!($flag & 8) && $help[6] ne "=" && ($help[2] ne $help[6])) + if (!($flag & 8) && $rnext ne "=" && ($rname ne $rnext)) # keep matrix symmetrical to see whether there is a bias, e.g. more 1->10 than 10->1 { $aberrant++; @@ -151,14 +151,14 @@ # is more interesting if ($flag & 64) { - $chrompairs{$help[2]}{$help[6]}++; + $chrompairs{$rname}{$rnext}++; } } # for insert sizes, take first read of a proper pair (-f 67 = 64 (first in pair) + 2 (proper pair) + 1 (paired)); # discarding duplicates (-F 1024) is already done further up if ($flag & 64 && $flag & 2 && $flag & 1) { - $entry = abs($help[8]); # insert size + $entry = abs($tlen); # insert size if ($entry < $min) { $min = $entry; diff --git a/resources/analysisTools/qcPipeline/workflowLib.sh b/resources/analysisTools/qcPipeline/workflowLib.sh index 57fb0d6..702aa47 100755 --- a/resources/analysisTools/qcPipeline/workflowLib.sh +++ b/resources/analysisTools/qcPipeline/workflowLib.sh @@ -272,11 +272,12 @@ ALT_FILE="${ALT_FILE:-$INDEX_PREFIX.alt}" # By default assume that bwa-postalt.js and k8 are located besides bwa (like in bwakit). bwaPostAltJsPath="${bwaPostAltJsPath:-"$(dirname "$(which bwa)")"/bwa-postalt.js}" K8_BINARY="${K8_BINARY:-"$(dirname "$(which bwa)")"/k8}" +declare -a bwaPostAltJsK8Options="$bwaPostAltJsK8Options" optionalBwaPostAltJs() { local hlaPrefix="${1:-}" local minPaRatio="${2:-}" if [[ "$runBwaPostAltJs" == "true" ]]; then - $K8_BINARY "$bwaPostAltJsPath" ${hlaPrefix:+-p "$hlaPrefix"} ${minPaRatio:+-r "$minPaRatio"} "$ALT_FILE" + $K8_BINARY ${bwaPostAltJsK8Options[@]} "$bwaPostAltJsPath" ${hlaPrefix:+-p "$hlaPrefix"} ${minPaRatio:+-r "$minPaRatio"} "$ALT_FILE" else cat - fi diff --git a/resources/configurationFiles/analysisQC.xml b/resources/configurationFiles/analysisQC.xml index 091e8f3..4486f14 100644 --- a/resources/configurationFiles/analysisQC.xml +++ b/resources/configurationFiles/analysisQC.xml @@ -36,6 +36,8 @@ description="The bwakit module contains the bwa-postalt.js script. Used if runBwaPostAltJs=true"/> + @@ -81,6 +83,9 @@ + + diff --git a/src/de/dkfz/b080/co/QualityControlWorkflowPlugin.java b/src/de/dkfz/b080/co/QualityControlWorkflowPlugin.java index a841951..aa31a88 100644 --- a/src/de/dkfz/b080/co/QualityControlWorkflowPlugin.java +++ b/src/de/dkfz/b080/co/QualityControlWorkflowPlugin.java @@ -9,7 +9,7 @@ public class QualityControlWorkflowPlugin extends BasePlugin { public static final String CURRENT_VERSION_STRING = "1.2.73"; - public static final String CURRENT_VERSION_BUILD_DATE = "Wed Feb 21 15:47:18 CET 2018"; + public static final String CURRENT_VERSION_BUILD_DATE = "Wed Apr 12 10:50:50 CEST 2023"; @Override public String getVersionInfo() { diff --git a/src/de/dkfz/b080/co/common/AlignmentAndQCConfig.groovy b/src/de/dkfz/b080/co/common/AlignmentAndQCConfig.groovy index afb81da..1fe2681 100644 --- a/src/de/dkfz/b080/co/common/AlignmentAndQCConfig.groovy +++ b/src/de/dkfz/b080/co/common/AlignmentAndQCConfig.groovy @@ -20,11 +20,19 @@ class AlignmentAndQCConfig extends COConfig { public static final String CVALUE_RUN_FINGERPRINTING = "runFingerprinting" public static final String CVALUE_FINGERPRINTING_SITES_FILE="fingerprintingSitesFile" - public AlignmentAndQCConfig(ExecutionContext context) { + AlignmentAndQCConfig(ExecutionContext context) { super(context) } - public String getSingleBamParameter() { + void setUseOnlyExistingTargetBam(boolean value = true) { + setConfig("useOnlyExistingTargetBam", value.toString(), "boolean") + } + + void setExtractSamplesFromOutputFiles(boolean value = true) { + setConfig("extractSamplesFromOutputFiles", value.toString(), "boolean") + } + + String getSingleBamParameter() { return configValues.get("bam", ""); } @@ -33,22 +41,26 @@ class AlignmentAndQCConfig extends COConfig { } boolean getUseExistingLaneBams() { - return configValues.getBoolean(COConstants.FLAG_USE_EXISTING_PAIRED_BAMS, false) + return configValues.getBoolean("useExistingLaneBams", false) } - public String getIndexPrefix() { + boolean getUseOnlyExistingPairedBams() { + return configValues.getBoolean("useExistingPairedBams", false) + } + + String getIndexPrefix() { return configValues.getString(CVALUE_INDEX_PREFIX, "") } - public File getChromosomeSizesFile() { + File getChromosomeSizesFile() { return new File (configValues.getString(CVALUE_CHROMOSOME_SIZES_FILE, "")) } - public File getTargetRegionsFile() { + File getTargetRegionsFile() { return new File (configValues.getString(CVALUE_TARGET_REGIONS_FILE, "")) } - public Integer getTargetSize() { + Integer getTargetSize() { Integer returnValue = configValues.getString(CVALUE_TARGET_SIZE, null) as Integer if (null == returnValue) { returnValue = configValues.getString(CVALUE_TARGETSIZE, null) as Integer @@ -56,29 +68,54 @@ class AlignmentAndQCConfig extends COConfig { return returnValue } - public boolean getRunExomeAnalysis() { - return configValues.getBoolean(COConstants.FLAG_RUN_EXOME_ANALYSIS) + boolean getRunExomeAnalysis() { + return configValues.getBoolean("runExomeAnalysis") } - public File getCytosinePositionIndex() { + File getCytosinePositionIndex() { return new File(configValues.getString(CVALUE_CYTOSINE_POSITIONS_INDEX)) } - public File getClipIndex() { + File getClipIndex() { return new File(configValues.getString(CVALUE_CLIP_INDEX)) } - public boolean getRunFingerprinting() { + boolean getRunFingerprinting() { return configValues.getBoolean(CVALUE_RUN_FINGERPRINTING, true) } - public File getFingerprintingSitesFile() { + File getFingerprintingSitesFile() { return new File(configValues.getString(CVALUE_FINGERPRINTING_SITES_FILE)) } - public Boolean getUseOnlyExistingPairedBams() { - return configValues.getBoolean(COConstants.FLAG_USE_EXISTING_PAIRED_BAMS, false); + boolean getRunFastqcOnly() { + return configValues.getBoolean("runFastQCOnly", false) + } + + boolean getRunFastqc() { + return configValues.getBoolean("runFastQC", true) } + boolean getRunAlignmentOnly() { + return configValues.getBoolean("runAlignmentOnly", false) + } + + boolean getRunCoveragePlots() { + return configValues.getBoolean("runCoveragePlots", true) + } + + boolean getRunCollectBamFileMetrics() { + return configValues.getBoolean("runCollectBamFileMetrics", false) + } + + @Deprecated + boolean getUseCombinedAlignAndSampe() { + return true + } + + @Deprecated + boolean getRunSlimWorkflow() { + return true + } } diff --git a/src/de/dkfz/b080/co/common/COProjectsRuntimeService.groovy b/src/de/dkfz/b080/co/common/COProjectsRuntimeService.groovy index b78e0f8..0fd6b61 100644 --- a/src/de/dkfz/b080/co/common/COProjectsRuntimeService.groovy +++ b/src/de/dkfz/b080/co/common/COProjectsRuntimeService.groovy @@ -12,14 +12,13 @@ import de.dkfz.roddy.execution.io.ExecutionResult; import de.dkfz.roddy.execution.io.ExecutionService; import de.dkfz.roddy.execution.io.fs.FileSystemAccessProvider import de.dkfz.roddy.tools.LoggerWrapper +import groovy.transform.CompileStatic import java.util.function.Consumer; -/** - * Created by heinold on 15.01.16. - */ -@groovy.transform.CompileStatic -public class COProjectsRuntimeService extends BasicCOProjectsRuntimeService { + +@CompileStatic +class COProjectsRuntimeService extends BasicCOProjectsRuntimeService { private static LoggerWrapper logger = LoggerWrapper.getLogger(BasicCOProjectsRuntimeService.class.getName()); protected static void getFileCompression(ExecutionContext run, List allLaneFiles) { @@ -174,6 +173,14 @@ public class COProjectsRuntimeService extends BasicCOProjectsRuntimeService { return laneFiles } + protected static int indexOfPathElement(String pathnamePattern, String element) { + int index = pathnamePattern.split(StringConstants.SPLIT_SLASH).findIndexOf { it -> it == element } + if (index < 0) { + throw new RuntimeException("Couldn't match '${element}' in '${pathnamePattern}") + } + return index + } + public List getLaneFileGroupsFromFastqList(ExecutionContext context, Sample sample, String libraryID) { COConfig coConfig = new COConfig(context) List fastqFiles = coConfig.getFastqList().collect { String it -> new File(it); } @@ -238,11 +245,22 @@ public class COProjectsRuntimeService extends BasicCOProjectsRuntimeService { runIndex = 2; sampleName = split[0..1].join(StringConstants.UNDERSCORE); } - String run = split[runIndex..-2].join(StringConstants.UNDERSCORE); - String lane = String.format("L%03d", laneID); - - - BamFile bamFile = COBaseFile.constructSourceFile(BamFile, f, context, new COFileStageSettings(lane, run, sample, context.getDataSet())) as BamFile + RunID run = new RunID(split[runIndex..-2].join(StringConstants.UNDERSCORE)) + LaneID lane = new LaneID(String.format("L%03d", laneID)) + + + BamFile bamFile = + COBaseFile.constructSourceFile( + BamFile, + f, + context, + new COFileStageSettings( + lane, + run, + (LibraryID) null, + sample, + context.dataSet) + ) as BamFile return bamFile; }) BamFileGroup bamFileGroup = new BamFileGroup(bamFiles); diff --git a/src/de/dkfz/b080/co/files/AlignedSequenceFileGroup.java b/src/de/dkfz/b080/co/files/AlignedSequenceFileGroup.java index d96d0e9..811e664 100644 --- a/src/de/dkfz/b080/co/files/AlignedSequenceFileGroup.java +++ b/src/de/dkfz/b080/co/files/AlignedSequenceFileGroup.java @@ -1,5 +1,7 @@ package de.dkfz.b080.co.files; +import de.dkfz.b080.co.common.LaneID; +import de.dkfz.b080.co.common.RunID; import de.dkfz.roddy.config.Configuration; import de.dkfz.roddy.core.ExecutionContext; import de.dkfz.roddy.execution.jobs.BEJobResult; @@ -35,15 +37,16 @@ public BamFile pairAndSortSlim() { String libString = configuration.getConfigurationValues().getString(COConstants.PRM_CVAL_LIBRARY); String sampleName = laneFile0.getSample().getName(); String pid = context.getDataSet().getId(); - String run = laneFile0.getRunID(); - String lane = laneFile0.getLaneId(); + RunID run = laneFile0.getRunID(); + LaneID lane = laneFile0.getLaneId(); String lb = sampleName + "_" + pid + (libString.equals("addToOldLib") ? "" : "_lib2"); String laneId0 = "RAW_SEQ_FILE_1_INDEX=" + ((COFileStageSettings) laneFile0.getFileStage()).getNumericIndex(); String laneId1 = "RAW_SEQ_FILE_2_INDEX=" + ((COFileStageSettings) laneFile1.getFileStage()).getNumericIndex(); final String TOOL = "sampesortSlim"; - BamFile bamFile = GenericMethod.callGenericTool(TOOL, seqFile0, seqFile1, laneFile0, laneFile1, "SAMPLE=" + sampleName, "RUN=" + run, "LANE=" + lane, "LB=" + lb, laneId0, laneId1); + BamFile bamFile = GenericMethod.callGenericTool(TOOL, seqFile0, seqFile1, laneFile0, laneFile1, + "SAMPLE=" + sampleName, "RUN=" + run.toString(), "LANE=" + lane.toString(), "LB=" + lb, laneId0, laneId1); return bamFile; } @@ -78,7 +81,7 @@ public BamFile pairAndSort() { parameters.put(COConstants.PRM_RAW_SEQ_2, laneFile1.getAbsolutePath()); // Could be moved to the config / scripts - parameters.put("ID", parentFile.getRunID() + "_" + parentFile.getLaneId()); + parameters.put("ID", parentFile.getRunID().toString() + "_" + parentFile.getLaneId().toString()); parameters.put("SM", "sample_" + parentFile.getSample().getName() + "_" + context.getDataSet()); parameters.put("LB", parentFile.getSample().getName() + "_" + context.getDataSet() + (libString.equals("addToOldLib") ? "" : "_lib2")); @@ -90,7 +93,14 @@ public BamFile pairAndSort() { List parentFiles = new LinkedList(); parentFiles.addAll(filesInGroup); - Job job = new Job(context, context.createJobName(parentFiles.get(0), COConstants.TOOL_SAMPESORT, true), COConstants.TOOL_SAMPESORT, null, parameters, parentFiles, Arrays.asList((BaseFile)bamFile, indexFile, flagstatsFile)); + Job job = new Job( + context, + context.createJobName(parentFiles.get(0), COConstants.TOOL_SAMPESORT, true), + COConstants.TOOL_SAMPESORT, + (List) null, + parameters, + parentFiles, + Arrays.asList((BaseFile)bamFile, indexFile, flagstatsFile)); BEJobResult jobResult = job.run(); flagstatsFile.setCreatingJobsResult(jobResult); diff --git a/src/de/dkfz/b080/co/files/LaneFile.java b/src/de/dkfz/b080/co/files/LaneFile.java index e415669..aed6b85 100644 --- a/src/de/dkfz/b080/co/files/LaneFile.java +++ b/src/de/dkfz/b080/co/files/LaneFile.java @@ -1,5 +1,8 @@ package de.dkfz.b080.co.files; +import de.dkfz.b080.co.common.IndexID; +import de.dkfz.b080.co.common.LaneID; +import de.dkfz.b080.co.common.RunID; import de.dkfz.roddy.config.Configuration; import de.dkfz.roddy.core.ExecutionContext; import de.dkfz.roddy.execution.io.ExecutionResult; @@ -34,7 +37,7 @@ public class LaneFile extends COBaseFile implements ITestdataSource { private FastqcFile fastqcFile; private AlignedSequenceFile alignedSequenceFile; - public LaneFile(ConstructionHelperForBaseFiles helper) { + public LaneFile(BaseFile.ConstructionHelperForBaseFiles helper) { super(helper); } @@ -137,15 +140,15 @@ public AlignedSequenceFile getAlignedSequenceFile() { return alignedSequenceFile; } - public String getLaneId() { - return ((COFileStageSettings) fileStageSettings).getLaneId(); + public LaneID getLaneId() { + return ((COFileStageSettings) fileStageSettings).getLaneID(); } - public String getIndex() { + public IndexID getIndex() { return ((COFileStageSettings) fileStageSettings).getIndex(); } - public String getRunID() { + public RunID getRunID() { return ((COFileStageSettings) fileStageSettings).getRunID(); } @@ -215,7 +218,7 @@ public boolean createTestData() { String cmd = String.format(getDecompressionString() + " %s | head -n %s | " + recompressionString + " > %s", filePath, testLineCnt, targetFilePath); logger.log(Level.INFO, cmd); ExecutionResult er = es.execute(cmd); - if (!er.successful) { + if (!er.isSuccessful()) { logger.severe("Could not create testdata for file " + targetFilePath); } diff --git a/src/de/dkfz/b080/co/files/LaneFileGroup.java b/src/de/dkfz/b080/co/files/LaneFileGroup.java index d6d19a2..73d8188 100644 --- a/src/de/dkfz/b080/co/files/LaneFileGroup.java +++ b/src/de/dkfz/b080/co/files/LaneFileGroup.java @@ -1,5 +1,7 @@ package de.dkfz.b080.co.files; +import de.dkfz.b080.co.common.LaneID; +import de.dkfz.b080.co.common.RunID; import de.dkfz.roddy.config.Configuration; import de.dkfz.roddy.core.ExecutionContext; import de.dkfz.roddy.execution.io.ExecutionResult; @@ -33,7 +35,11 @@ public class LaneFileGroup extends FileGroup { private FastqcGroup allFastqcFiles; private AlignedSequenceFileGroup allAlignedFiles; - public LaneFileGroup(ExecutionContext executionContext, String id, String run, Sample sample, List files) { + public LaneFileGroup(ExecutionContext executionContext, + String id, + String run, + Sample sample, + List files) { super(files); this.id = id; this.run = run; @@ -68,7 +74,9 @@ public Sample getSample() { //This method could be a candidate for public FastqcGroup calcFastqcForAll() { - final boolean useSingleEndProcessing = getExecutionContext().getConfiguration().getConfigurationValues().getBoolean(COConstants.FLAG_USE_SINGLE_END_PROCESSING, false); + final boolean useSingleEndProcessing = + getExecutionContext().getConfiguration().getConfigurationValues(). + getBoolean(COConstants.FLAG_USE_SINGLE_END_PROCESSING, false); LinkedList files = new LinkedList(); for (LaneFile f : filesInGroup) { @@ -85,7 +93,9 @@ public FastqcGroup calcFastqcForAll() { } public AlignedSequenceFileGroup alignAll() { - final boolean useSingleEndProcessing = getExecutionContext().getConfiguration().getConfigurationValues().getBoolean(COConstants.FLAG_USE_SINGLE_END_PROCESSING, false); + final boolean useSingleEndProcessing = + getExecutionContext().getConfiguration().getConfigurationValues(). + getBoolean(COConstants.FLAG_USE_SINGLE_END_PROCESSING, false); LinkedList files = new LinkedList(); int i = 0; for (LaneFile f : filesInGroup) { @@ -108,37 +118,44 @@ public AlignedSequenceFileGroup alignAll() { public BamFile alignAndPairSlim() { ExecutionContext context = getExecutionContext(); Configuration configuration = context.getConfiguration(); - boolean useAcceleratedHardware = configuration.getConfigurationValues().getBoolean(COConstants.FLAG_USE_ACCELERATED_HARDWARE); + boolean useAcceleratedHardware = + configuration.getConfigurationValues(). + getBoolean(COConstants.FLAG_USE_ACCELERATED_HARDWARE); // Bad hack: Decrease the file stage level by one! LaneFile laneFile0 = filesInGroup.get(0).getFSDecreasedCopy(); LaneFile laneFile1 = filesInGroup.get(1).getFSDecreasedCopy(); - String libString = configuration.getConfigurationValues().getString(COConstants.PRM_CVAL_LIBRARY); + String libString = + configuration.getConfigurationValues(). + getString(COConstants.PRM_CVAL_LIBRARY); String sampleName = laneFile0.getSample().getName(); String pid = context.getDataSet().getId(); - String run = laneFile0.getRunID(); - String lane = laneFile0.getLaneId(); + RunID run = laneFile0.getRunID(); + LaneID lane = laneFile0.getLaneId(); String lb = sampleName + "_" + pid + (libString.equals("addToOldLib") ? "" : "_lib2"); String laneId0 = "RAW_SEQ_FILE_1_INDEX=" + ((COFileStageSettings) laneFile0.getFileStage()).getNumericIndex(); String laneId1 = "RAW_SEQ_FILE_2_INDEX=" + ((COFileStageSettings) laneFile1.getFileStage()).getNumericIndex(); - final String TOOL = useAcceleratedHardware ? COConstants.TOOL_ACCELERATED_ALIGNANDPAIR_SLIM : COConstants.TOOL_ALIGNANDPAIR_SLIM; + final String TOOL = + useAcceleratedHardware ? + COConstants.TOOL_ACCELERATED_ALIGNANDPAIR_SLIM : + COConstants.TOOL_ALIGNANDPAIR_SLIM; BamFile bamFile = GenericMethod.callGenericTool(TOOL, laneFile0, laneFile1, "SAMPLE=" + sampleName, "sample=" + sampleName, - "RUN=" + run, "run=" + run, - "LANE=" + lane, "lane=" + lane, + "RUN=" + run.toString(), "run=" + run.toString(), + "LANE=" + lane.toString(), "lane=" + lane.toString(), "LB=" + lb, laneId0, laneId1); return bamFile; } public BamFile alignAndPair() { - ExecutionContext run = getExecutionContext(); + ExecutionContext runContext = getExecutionContext(); LaneFile laneFile0 = filesInGroup.get(0); LaneFile laneFile1 = filesInGroup.get(1); - Configuration configuration = run.getConfiguration(); + Configuration configuration = runContext.getConfiguration(); BamFile bamFile = (BamFile) BaseFile.constructManual(BamFile.class, this); FlagstatsFile flagstatsFile = (FlagstatsFile)BaseFile.constructManual(FlagstatsFile.class, bamFile); BamIndexFile bamIndexFile = (BamIndexFile) BaseFile.constructManual(BamIndexFile.class, bamFile); @@ -152,15 +169,15 @@ public BamFile alignAndPair() { final String TOOL = useAcceleratedHardware ? COConstants.TOOL_ACCELERATED_ALIGNANDPAIR : COConstants.TOOL_ALIGNANDPAIR; String sampleName = laneFile0.getSample().getName(); - String pid = run.getDataSet().getId(); + String pid = runContext.getDataSet().getId(); - Map parameters = run.getDefaultJobParameters(TOOL); + Map parameters = runContext.getDefaultJobParameters(TOOL); parameters.put(COConstants.PRM_FILENAME_SORTED_BAM, bamFile.getAbsolutePath()); parameters.put(COConstants.PRM_FILENAME_FLAGSTAT, flagstatsFile.getAbsolutePath()); parameters.put(COConstants.PRM_FILENAME_BAM_INDEX, bamIndexFile.getAbsolutePath()); parameters.put(COConstants.PRM_RAW_SEQ_1, laneFile0.getAbsolutePath()); parameters.put(COConstants.PRM_RAW_SEQ_2, laneFile1.getAbsolutePath()); - parameters.put(COConstants.PRM_ID, laneFile0.getRunID() + "_" + laneFile0.getLaneId()); + parameters.put(COConstants.PRM_ID, laneFile0.getRunID().toString() + "_" + laneFile0.getLaneId().toString()); parameters.put(COConstants.PRM_SM, "sample_" + sampleName + "_" + pid); parameters.put(COConstants.PRM_LB, sampleName + "_" + pid + (libString.equals("addToOldLib") ? "" : "_lib2")); @@ -171,7 +188,14 @@ public BamFile alignAndPair() { List parentFiles = new LinkedList<>(); parentFiles.addAll(filesInGroup); - Job job = new Job(run, run.createJobName(parentFiles.get(0), TOOL, true), TOOL, null, parameters, parentFiles, Arrays.asList((BaseFile) bamFile, flagstatsFile)); + Job job = new Job( + runContext, + runContext.createJobName(parentFiles.get(0), TOOL,true), + TOOL, + (List) null, + parameters, + parentFiles, + Arrays.asList((BaseFile) bamFile, flagstatsFile)); BEJobResult jobResult = job.run(); flagstatsFile.setCreatingJobsResult(jobResult); if (indexCreated) bamIndexFile.setCreatingJobsResult(jobResult); @@ -222,7 +246,7 @@ public void createTestDataForLaneFiles() { fullCommandList.append(" wait"); logger.log(Level.INFO, fullCommandList.toString()); ExecutionResult er = es.execute(fullCommandList.toString()); - if (!er.successful) { + if (!er.isSuccessful()) { logger.severe("Could not create testdata for one or more files."); } } diff --git a/src/de/dkfz/b080/co/methods/Common.groovy b/src/de/dkfz/b080/co/methods/Common.groovy index be48b17..bfe636a 100644 --- a/src/de/dkfz/b080/co/methods/Common.groovy +++ b/src/de/dkfz/b080/co/methods/Common.groovy @@ -83,7 +83,7 @@ class Common { if (bamFileFileStage.stage.isMoreDetailedOrEqualTo(COFileStage.RUN)) { runId = bamFileFileStage.runID; if (bamFileFileStage.stage.isMoreDetailedOrEqualTo(COFileStage.LANE)) - lane = bamFileFileStage.laneId; + lane = bamFileFileStage.laneID; } def temp = run.getDefaultJobParameters(QCSUMMARY); diff --git a/src/de/dkfz/b080/co/methods/Samtools.groovy b/src/de/dkfz/b080/co/methods/Samtools.groovy index 3ddb7a3..12bd286 100644 --- a/src/de/dkfz/b080/co/methods/Samtools.groovy +++ b/src/de/dkfz/b080/co/methods/Samtools.groovy @@ -9,12 +9,13 @@ import de.dkfz.roddy.execution.jobs.ScriptCallingMethod import de.dkfz.roddy.execution.jobs.StaticScriptProviderClass import de.dkfz.roddy.knowledge.files.BaseFile import de.dkfz.roddy.tools.LoggerWrapper +import groovy.transform.CompileStatic /** * * @author michael */ -@groovy.transform.CompileStatic +@CompileStatic @StaticScriptProviderClass class Samtools { diff --git a/src/de/dkfz/b080/co/qcworkflow/BisulfiteCoreWorkflow.groovy b/src/de/dkfz/b080/co/qcworkflow/BisulfiteCoreWorkflow.groovy index 8a57aff..487ffc7 100644 --- a/src/de/dkfz/b080/co/qcworkflow/BisulfiteCoreWorkflow.groovy +++ b/src/de/dkfz/b080/co/qcworkflow/BisulfiteCoreWorkflow.groovy @@ -1,87 +1,68 @@ package de.dkfz.b080.co.qcworkflow import de.dkfz.b080.co.common.AlignmentAndQCConfig -import de.dkfz.b080.co.common.BasicCOProjectsRuntimeService import de.dkfz.b080.co.common.COProjectsRuntimeService import de.dkfz.b080.co.files.* -import de.dkfz.roddy.config.Configuration -import de.dkfz.roddy.config.RecursiveOverridableMapContainerForConfigurationValues -import de.dkfz.roddy.core.DataSet import de.dkfz.roddy.core.ExecutionContext import de.dkfz.roddy.core.ExecutionContextError import de.dkfz.roddy.execution.io.ExecutionService import de.dkfz.roddy.execution.io.fs.FileSystemAccessProvider - -import static de.dkfz.b080.co.files.COConstants.FLAG_EXTRACT_SAMPLES_FROM_OUTPUT_FILES +import groovy.transform.CompileStatic /** * @author michael */ -@groovy.transform.CompileStatic -public class BisulfiteCoreWorkflow extends QCPipeline { +@CompileStatic +class BisulfiteCoreWorkflow extends QCPipeline { @Override - public boolean execute(ExecutionContext context) { - Configuration cfg = context.getConfiguration(); - RecursiveOverridableMapContainerForConfigurationValues cfgValues = cfg.getConfigurationValues(); - cfgValues.put(FLAG_EXTRACT_SAMPLES_FROM_OUTPUT_FILES, "false", "boolean"); //Disable sample extraction from output for alignment workflows. - //cfgValues.put(COConstants.FLAG_USE_ACCELERATED_HARDWARE, "false", "boolean"); //Disable accelerated hardware usage for testing - - // Run flags - final boolean runFastQCOnly = cfgValues.getBoolean(COConstants.FLAG_RUN_FASTQC_ONLY, false) - final boolean runFastQC = cfgValues.getBoolean(COConstants.FLAG_RUN_FASTQC, true) - final boolean runAlignmentOnly = cfgValues.getBoolean(COConstants.FLAG_RUN_ALIGNMENT_ONLY, false); - final boolean runCoveragePlots = cfgValues.getBoolean(COConstants.FLAG_RUN_COVERAGE_PLOTS, true); - final boolean runCollectBamFileMetrics = cfgValues.getBoolean(COConstants.FLAG_RUN_COLLECT_BAMFILE_METRICS, false); + boolean execute(ExecutionContext context) { + AlignmentAndQCConfig cfg = new AlignmentAndQCConfig(context) + cfg.setUseOnlyExistingTargetBam(true) - COProjectsRuntimeService runtimeService = (COProjectsRuntimeService) context.getRuntimeService(); - - List samples = runtimeService.getSamplesForContext(context); + COProjectsRuntimeService runtimeService = (COProjectsRuntimeService) context.runtimeService + List samples = runtimeService.getSamplesForContext(context) - BamFileGroup mergedBamFiles = new BamFileGroup(); + BamFileGroup mergedBamFiles = new BamFileGroup() Map coverageTextFilesBySample = [:] for (Sample sample in samples) { - - List availableLibrariesForSample = sample.getLibraries(); - BamFileGroup mergedBamsPerLibrary = new BamFileGroup(); + BamFileGroup mergedBamsPerLibrary = new BamFileGroup() // Create per library merged bams - for (String library in availableLibrariesForSample) { + for (String library in sample.libraries) { BamFileGroup sortedBamFiles = [] - List rawSequenceGroups = runtimeService.loadLaneFilesForSampleAndLibrary(context, sample, library) + List rawSequenceGroups = runtimeService. + loadLaneFilesForSampleAndLibrary(context, sample, library) if (rawSequenceGroups == null || rawSequenceGroups.size() > 0) { for (LaneFileGroup rawSequenceGroup : rawSequenceGroups) { - if (runFastQC && !runAlignmentOnly) - rawSequenceGroup.calcFastqcForAll(); - if (runFastQCOnly) - continue; - + if (cfg.runFastqc && !cfg.runAlignmentOnly) + rawSequenceGroup.calcFastqcForAll() + if (cfg.runFastqcOnly) + continue - BamFile bamFile = rawSequenceGroup.alignAndPairSlim(); - //rawSequenceGroup.alignAndPairSlim() + BamFile bamFile = rawSequenceGroup.alignAndPairSlim() - bamFile.setAsTemporaryFile(); // Bam files created with sai files are only temporary. - sortedBamFiles.addFile(bamFile); + bamFile.setAsTemporaryFile() // Bam files created with sai files are only temporary. + sortedBamFiles.addFile(bamFile) } } - if (!sortedBamFiles.getFilesInGroup()) continue; + if (!sortedBamFiles.filesInGroup) continue - if (runAlignmentOnly) continue; + if (cfg.runAlignmentOnly) continue - BamFile mergedLibraryBam; - if (availableLibrariesForSample.size() == 1) { - mergedLibraryBam = sortedBamFiles.mergeAndRemoveDuplicatesSlim(sample); - if (runCollectBamFileMetrics) mergedLibraryBam.collectMetrics(); + BamFile mergedLibraryBam + if (sample.libraries.size() == 1) { + mergedLibraryBam = sortedBamFiles.mergeAndRemoveDuplicatesSlim(sample) + if (cfg.runCollectBamFileMetrics) mergedLibraryBam.collectMetrics() - Sample.SampleType sampleType = sample.getType(); - if (!coverageTextFilesBySample.containsKey(sampleType)) - coverageTextFilesBySample.put(sampleType, new CoverageTextFileGroup()); - coverageTextFilesBySample.get(sampleType).addFile(mergedLibraryBam.calcReadBinsCoverage()); + if (!coverageTextFilesBySample.containsKey(sample.type)) + coverageTextFilesBySample.put(sample.type, new CoverageTextFileGroup()) + coverageTextFilesBySample.get(sample.type).addFile(mergedLibraryBam.calcReadBinsCoverage()) mergedBamFiles.addFile(mergedLibraryBam); // Unfortunately, due to the way Roddy works, the following call needs to be encapsulated into @@ -101,38 +82,39 @@ public class BisulfiteCoreWorkflow extends QCPipeline { } // Merge library bams into per sample bams - if(availableLibrariesForSample.size() > 1) { - BamFile mergedBam = mergedBamsPerLibrary.mergeSlim(sample); + if(sample.libraries.size() > 1) { + BamFile mergedBam = mergedBamsPerLibrary.mergeSlim(sample) // Unfortunately, due to the way Roddy works, the following call needs to be encapsulated into // a method, in order to put library and merged methylation results into different directories. // This allows for selection via onMethod="BisulfiteCoreWorkflow.mergedMethylationCallingMeta". mergedBam.mergedMethylationCallingMeta() - if (runCollectBamFileMetrics) mergedBam.collectMetrics(); + if (cfg.runCollectBamFileMetrics) mergedBam.collectMetrics() - Sample.SampleType sampleType = sample.getType(); - if (!coverageTextFilesBySample.containsKey(sampleType)) - coverageTextFilesBySample.put(sampleType, new CoverageTextFileGroup()); - coverageTextFilesBySample.get(sampleType).addFile(mergedBam.calcReadBinsCoverage()); + if (!coverageTextFilesBySample.containsKey(sample.type)) + coverageTextFilesBySample.put(sample.type, new CoverageTextFileGroup()) + coverageTextFilesBySample.get(sample.type).addFile(mergedBam.calcReadBinsCoverage()) - mergedBamFiles.addFile(mergedBam); + mergedBamFiles.addFile(mergedBam) } } - if (!mergedBamFiles.getFilesInGroup()) { - context.addErrorEntry(ExecutionContextError.EXECUTION_NOINPUTDATA.expand("There were no merged bam files available.")); + if (!mergedBamFiles.filesInGroup) { + context.addErrorEntry(ExecutionContextError.EXECUTION_NOINPUTDATA. + expand("There were no merged bam files available.")) return false; } - if (runCoveragePlots && coverageTextFilesBySample.keySet().size() >= 2) { - coverageTextFilesBySample.get(Sample.SampleType.CONTROL).plotAgainst(coverageTextFilesBySample.get(Sample.SampleType.TUMOR)); - } else if (runCoveragePlots && coverageTextFilesBySample.keySet().size() == 1) { + if (cfg.runCoveragePlots && coverageTextFilesBySample.keySet().size() >= 2) { + coverageTextFilesBySample.get(Sample.SampleType.CONTROL). + plotAgainst(coverageTextFilesBySample.get(Sample.SampleType.TUMOR)); + } else if (cfg.runCoveragePlots && coverageTextFilesBySample.keySet().size() == 1) { //TODO: Think if this conflicts with plotAgainst on rerun! Maybe missing files are not recognized. - ((CoverageTextFileGroup) coverageTextFilesBySample.values().toArray()[0]).plot(); + ((CoverageTextFileGroup) coverageTextFilesBySample.values().toArray()[0]).plot() } - return true; + return true } @@ -144,18 +126,18 @@ public class BisulfiteCoreWorkflow extends QCPipeline { List samples = runtimeService.getSamplesForContext(context) for (Sample sample : samples) { LinkedHashMap laneFileGroups = [:] - for (String lib : sample.getLibraries()) { + for (String lib : sample.libraries) { for (LaneFileGroup group : runtimeService.loadLaneFilesForSampleAndLibrary(context, sample, lib)) { - String key = group.getRun() + " " + group.getId() + String key = group.run + " " + group.id if (laneFileGroups.containsKey(key)) { context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID. - expand("Duplicate lane identifiers for ${group.getRun()}_${group.getId()} among libraries " + - "(pid=${context.getDataSet()}, sample=${sample.getName()})! Check run and FASTQ names.")) + expand("Duplicate lane identifiers for ${group.run}_${group.id} among libraries " + + "(pid=${context.dataSet}, sample=${sample.name})! Check run and FASTQ names.")) returnValue = false } else { laneFileGroups[key] = group } - cnt += group.getFilesInGroup().size() + cnt += group.filesInGroup.size() } } } @@ -169,14 +151,16 @@ public class BisulfiteCoreWorkflow extends QCPipeline { protected boolean checkCytosinePositionIndex(ExecutionContext context) { AlignmentAndQCConfig aqcfg = new AlignmentAndQCConfig(context) - FileSystemAccessProvider accessProvider = FileSystemAccessProvider.getInstance() - File cposidx = aqcfg.getCytosinePositionIndex() + FileSystemAccessProvider accessProvider = FileSystemAccessProvider.instance + File cposidx = aqcfg.cytosinePositionIndex if (cposidx.toString().equals("")) { - context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID.expand("${AlignmentAndQCConfig.CVALUE_CYTOSINE_POSITIONS_INDEX} is not defined!")) + context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID. + expand("${AlignmentAndQCConfig.CVALUE_CYTOSINE_POSITIONS_INDEX} is not defined!")) return false } else if (!accessProvider.fileExists(cposidx) || !accessProvider.isReadable(cposidx)) { - context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID.expand("Cytosine position index '${cposidx}' is not accessible!")) + context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID. + expand("Cytosine position index '${cposidx}' is not accessible!")) return false } else { return true @@ -186,10 +170,11 @@ public class BisulfiteCoreWorkflow extends QCPipeline { protected boolean checkClipIndex(ExecutionContext context) { AlignmentAndQCConfig aqcfg = new AlignmentAndQCConfig(context) - FileSystemAccessProvider accessProvider = FileSystemAccessProvider.getInstance() - if (!aqcfg.getClipIndex().toString().startsWith('$' + ExecutionService.RODDY_CVALUE_DIRECTORY_EXECUTION) - && !accessProvider.isReadable(aqcfg.getClipIndex())) { - context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID.expand("Clip index '${aqcfg.getClipIndex()}' is not accessible!")); + FileSystemAccessProvider accessProvider = FileSystemAccessProvider.instance + if (!aqcfg.clipIndex.toString().startsWith('$' + ExecutionService.RODDY_CVALUE_DIRECTORY_EXECUTION) + && !accessProvider.isReadable(aqcfg.clipIndex)) { + context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID. + expand("Clip index '${aqcfg.clipIndex}' is not accessible!")) return false } else { return true @@ -197,7 +182,7 @@ public class BisulfiteCoreWorkflow extends QCPipeline { } @Override - public boolean checkExecutability(ExecutionContext context) { + boolean checkExecutability(ExecutionContext context) { boolean result = super.checkExecutability(context) result &= checkCytosinePositionIndex(context) result &= checkClipIndex(context) diff --git a/src/de/dkfz/b080/co/qcworkflow/PostMergeQCAnalysisWorkflow.java b/src/de/dkfz/b080/co/qcworkflow/PostMergeQCAnalysisWorkflow.java index ca5708e..167fe32 100644 --- a/src/de/dkfz/b080/co/qcworkflow/PostMergeQCAnalysisWorkflow.java +++ b/src/de/dkfz/b080/co/qcworkflow/PostMergeQCAnalysisWorkflow.java @@ -1,10 +1,11 @@ package de.dkfz.b080.co.qcworkflow; -import de.dkfz.b080.co.common.BasicCOProjectsRuntimeService; +import de.dkfz.b080.co.common.AlignmentAndQCConfig; import de.dkfz.b080.co.common.COProjectsRuntimeService; -import de.dkfz.b080.co.files.*; -import de.dkfz.roddy.config.Configuration; -import de.dkfz.roddy.config.RecursiveOverridableMapContainerForConfigurationValues; +import de.dkfz.b080.co.files.BamFile; +import de.dkfz.b080.co.files.BamFileGroup; +import de.dkfz.b080.co.files.CoverageTextFileGroup; +import de.dkfz.b080.co.files.Sample; import de.dkfz.roddy.core.ExecutionContext; import de.dkfz.roddy.core.ExecutionContextError; import de.dkfz.roddy.core.Workflow; @@ -13,9 +14,6 @@ import java.util.List; import java.util.Map; -import static de.dkfz.b080.co.files.COConstants.FLAG_EXTRACT_SAMPLES_FROM_OUTPUT_FILES; -import static de.dkfz.b080.co.files.COConstants.FLAG_USE_EXISTING_MERGED_BAMS; - /** * A short workflow which only does post merge alignment quality control */ @@ -25,16 +23,12 @@ public PostMergeQCAnalysisWorkflow() {} @Override public boolean execute(ExecutionContext context) { - Configuration cfg = context.getConfiguration(); - RecursiveOverridableMapContainerForConfigurationValues cfgValues = cfg.getConfigurationValues(); - cfgValues.put(FLAG_EXTRACT_SAMPLES_FROM_OUTPUT_FILES, "true", "boolean"); //Enable sample extraction from output for this workflow. - cfgValues.put(FLAG_USE_EXISTING_MERGED_BAMS, "true", "boolean"); //Enable usage of existing merged bams for this workflow. - final boolean runCoveragePlots = cfgValues.getBoolean(COConstants.FLAG_RUN_COVERAGE_PLOTS, true); - final boolean runExomeAnalysis = cfgValues.getBoolean(COConstants.FLAG_RUN_EXOME_ANALYSIS); + AlignmentAndQCConfig cfg = new AlignmentAndQCConfig(context); + cfg.setExtractSamplesFromOutputFiles(true); // Enable sample extraction from output for this workflow. + cfg.setUseOnlyExistingTargetBam(true); // Enable usage of existing merged bams for this workflow. COProjectsRuntimeService runtimeService = (COProjectsRuntimeService) context.getRuntimeService(); - - List samples = runtimeService.getSamplesForContext(context); + List samples = runtimeService.metadataAccessor.getSamples(context); if (samples.size() == 0) return false; @@ -42,10 +36,11 @@ public boolean execute(ExecutionContext context) { Map coverageTextFilesBySample = new LinkedHashMap<>(); for (Sample sample : samples) { - BamFile mergedBam = new BamFile(runtimeService.getMergedBamFileForDataSetAndSample(context, sample)); + BamFile mergedBam = new BamFile(runtimeService.metadataAccessor. + getMergedBamFileFromFilesystem(context, null, sample)); mergedBam.performPostMergeQCAnalysis(); - if (runExomeAnalysis) { + if (cfg.getRunExomeAnalysis()) { mergedBam.extractTargetsCalculateCoverage(); } @@ -62,7 +57,7 @@ public boolean execute(ExecutionContext context) { return false; } - if (runCoveragePlots && coverageTextFilesBySample.keySet().size() >= 2) { + if (cfg.getRunCoveragePlots() && coverageTextFilesBySample.keySet().size() >= 2) { coverageTextFilesBySample.get(Sample.SampleType.CONTROL).plotAgainst(coverageTextFilesBySample.get(Sample.SampleType.TUMOR)); } else if (coverageTextFilesBySample.keySet().size() == 1) { ((CoverageTextFileGroup) coverageTextFilesBySample.values().toArray()[0]).plot(); diff --git a/src/de/dkfz/b080/co/qcworkflow/QCPipeline.groovy b/src/de/dkfz/b080/co/qcworkflow/QCPipeline.groovy index 3ace2ed..2496958 100644 --- a/src/de/dkfz/b080/co/qcworkflow/QCPipeline.groovy +++ b/src/de/dkfz/b080/co/qcworkflow/QCPipeline.groovy @@ -34,7 +34,7 @@ public class QCPipeline extends Workflow { final boolean runFastQCOnly = cfgValues.getBoolean(COConstants.FLAG_RUN_FASTQC_ONLY, false); final boolean runAlignmentOnly = cfgValues.getBoolean(COConstants.FLAG_RUN_ALIGNMENT_ONLY, false); final boolean runCoveragePlots = cfgValues.getBoolean(COConstants.FLAG_RUN_COVERAGE_PLOTS, true); - final boolean runSlimWorkflow = cfgValues.getBoolean(COConstants.FLAG_RUN_SLIM_WORKFLOW, false); + final boolean runSlimWorkflow = true final boolean runExomeAnalysis = cfgValues.getBoolean(COConstants.FLAG_RUN_EXOME_ANALYSIS); final boolean runCollectBamFileMetrics = cfgValues.getBoolean(COConstants.FLAG_RUN_COLLECT_BAMFILE_METRICS, false); @@ -94,22 +94,10 @@ public class QCPipeline extends Workflow { } private BamFileGroup createSortedBams(ExecutionContext context, COProjectsRuntimeService runtimeService, Sample sample) { - Configuration cfg = context.getConfiguration(); - - RecursiveOverridableMapContainerForConfigurationValues cfgValues = cfg.getConfigurationValues(); - // Run flags - final boolean runFastQCOnly = cfgValues.getBoolean(COConstants.FLAG_RUN_FASTQC_ONLY, false); - final boolean runFastQC = cfgValues.getBoolean(COConstants.FLAG_RUN_FASTQC, true); - final boolean runAlignmentOnly = cfgValues.getBoolean(COConstants.FLAG_RUN_ALIGNMENT_ONLY, false); - - final boolean useOnlyExistingPairedBams = cfgValues.getBoolean(COConstants.FLAG_USE_EXISTING_PAIRED_BAMS, false); - // Usage flags - final boolean useCombinedAlignAndSampe = cfgValues.getBoolean(COConstants.FLAG_USE_COMBINED_ALIGN_AND_SAMPE, false); - final boolean runSlimWorkflow = cfgValues.getBoolean(COConstants.FLAG_RUN_SLIM_WORKFLOW, false); - + AlignmentAndQCConfig cfg = new AlignmentAndQCConfig(context) BamFileGroup sortedBamFiles = new BamFileGroup(); - if (useOnlyExistingPairedBams) { + if (cfg.useOnlyExistingPairedBams) { //Start from the paired bams instead of the lane files. sortedBamFiles = runtimeService.getPairedBamFilesForDataSet(context, sample); } else { @@ -119,23 +107,25 @@ public class QCPipeline extends Workflow { if (rawSequenceGroups == null || rawSequenceGroups.size() == 0) return sortedBamFiles; for (LaneFileGroup rawSequenceGroup : rawSequenceGroups) { - if (runFastQC && !runAlignmentOnly) + if (cfg.runFastqc && !cfg.runAlignmentOnly) rawSequenceGroup.calcFastqcForAll(); - if (runFastQCOnly) + if (cfg.runFastqcOnly) continue; BamFile bamFile = null; + final boolean runSlimWorkflow = true - if (useCombinedAlignAndSampe) { //I.e. bwa mem - if (runSlimWorkflow) { + // TODO Remove all this dead code. + if (cfg.useCombinedAlignAndSampe) { //I.e. bwa mem + if (cfg.runSlimWorkflow) { bamFile = rawSequenceGroup.alignAndPairSlim(); } else { bamFile = rawSequenceGroup.alignAndPair(); } } else { //I.e. bwa align rawSequenceGroup.alignAll(); - if (runSlimWorkflow) { + if (cfg.runSlimWorkflow) { bamFile = rawSequenceGroup.getAllAlignedFiles().pairAndSortSlim(); } else { bamFile = rawSequenceGroup.getAllAlignedFiles().pairAndSort(); @@ -157,7 +147,7 @@ public class QCPipeline extends Workflow { //To avoid problems with qcsummary the step is done manually. sortedBamFiles.runDefaultOperations(); - for (BamFile sortedBam : sortedBamFiles.getFilesInGroup()) + for (BamFile sortedBam : sortedBamFiles.filesInGroup) sortedBam.createQCSummaryFile(); BamFile mergedBam = sortedBamFiles.mergeAndRemoveDuplicates(); @@ -172,23 +162,28 @@ public class QCPipeline extends Workflow { private boolean valueIsEmpty(ExecutionContext context, Object value, String variableName) { if (value == null || value.toString() == "") { - context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID.expand("Expected value to be set: ${variableName}")) + context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID. + expand("Expected value to be set: ${variableName}")) return true } return false } private boolean fileIsAccessible(ExecutionContext context, File file, String variableName) { - if (valueIsEmpty(context, file, variableName) || !FileSystemAccessProvider.getInstance().checkFile(file, false, context)) { - context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID.expand("File '${file}' not accessible: ${variableName}")) + if (valueIsEmpty(context, file, variableName) || + !FileSystemAccessProvider.instance.checkFile(file, false, context)) { + context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID. + expand("File '${file}' not accessible: ${variableName}")) return false } return true } private boolean directoryIsAccessible(ExecutionContext context, File directory, String variableName) { - if (valueIsEmpty(context, directory, variableName) || !FileSystemAccessProvider.getInstance().checkDirectory(directory, context, false)) { - context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID.expand("Directory '${directory}' not accessible: ${variableName}")) + if (valueIsEmpty(context, directory, variableName) || + !FileSystemAccessProvider.instance.checkDirectory(directory, context, false)) { + context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID. + expand("Directory '${directory}' not accessible: ${variableName}")) return false } return true @@ -199,26 +194,31 @@ public class QCPipeline extends Workflow { boolean returnValue returnValue = - !valueIsEmpty(context, config.getIndexPrefix(), AlignmentAndQCConfig.CVALUE_INDEX_PREFIX) && - directoryIsAccessible(context, new File(config.getIndexPrefix()).getParentFile(), AlignmentAndQCConfig.CVALUE_INDEX_PREFIX) - returnValue = - fileIsAccessible(context, config.getChromosomeSizesFile(), AlignmentAndQCConfig.CVALUE_CHROMOSOME_SIZES_FILE) - if (config.getRunExomeAnalysis()) { - returnValue = - fileIsAccessible(context, config.getTargetRegionsFile(), AlignmentAndQCConfig.CVALUE_TARGET_REGIONS_FILE) && - !valueIsEmpty(context, config.getTargetSize(), AlignmentAndQCConfig.CVALUE_TARGET_SIZE) + !valueIsEmpty(context, config.indexPrefix, + AlignmentAndQCConfig.CVALUE_INDEX_PREFIX) && + directoryIsAccessible(context, new File(config.indexPrefix).parentFile, + AlignmentAndQCConfig.CVALUE_INDEX_PREFIX) + returnValue &= + fileIsAccessible(context, config.chromosomeSizesFile, + AlignmentAndQCConfig.CVALUE_CHROMOSOME_SIZES_FILE) + if (config.runExomeAnalysis) { + returnValue &= + fileIsAccessible(context, config.targetRegionsFile, + AlignmentAndQCConfig.CVALUE_TARGET_REGIONS_FILE) && + !valueIsEmpty(context, config.targetSize, + AlignmentAndQCConfig.CVALUE_TARGET_SIZE) } return returnValue } private boolean checkSamples(ExecutionContext context) { - BasicCOProjectsRuntimeService runtimeService = (BasicCOProjectsRuntimeService) context.getRuntimeService() + BasicCOProjectsRuntimeService runtimeService = (BasicCOProjectsRuntimeService) context.runtimeService List samples = runtimeService.getSamplesForContext(context) if (samples.size() == 0) { - context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID.expand("No samples found for PID ${context.getDataSet()}!")) + context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID.expand("No samples found for PID ${context.dataSet}!")) return false } else { - logger.postAlwaysInfo("Found " + samples.size() + " samples for dataset " + context.getDataSet().getId()); + logger.postAlwaysInfo("Found " + samples.size() + " samples for dataset " + context.dataSet.id); return true } } @@ -228,7 +228,8 @@ public class QCPipeline extends Workflow { AlignmentAndQCConfig cfg = new AlignmentAndQCConfig(context) - BasicCOProjectsRuntimeService runtimeService = (BasicCOProjectsRuntimeService) context.getRuntimeService() + BasicCOProjectsRuntimeService runtimeService = + (BasicCOProjectsRuntimeService) context.runtimeService List samples = runtimeService.getSamplesForContext(context) if (cfg.useOnlyExistingTargetBam && cfg.fastqFileListIsSet) { @@ -248,13 +249,13 @@ public class QCPipeline extends Workflow { for (Sample sample : samples) { List laneFileGroups = ((COProjectsRuntimeService) runtimeService).loadLaneFilesForSample(context, sample) for (LaneFileGroup lfg : laneFileGroups) { - cnt += lfg.getFilesInGroup().size() + cnt += lfg.filesInGroup.size() } - logger.postAlwaysInfo("Processed sample " + sample.getName() + " and found " + laneFileGroups.size() + " groups of lane files.") + logger.postAlwaysInfo("Processed sample " + sample.name + " and found " + laneFileGroups.size() + " groups of lane files.") } if (cnt <= 0) { context.addErrorEntry(ExecutionContextError.EXECUTION_NOINPUTDATA. - expand("No lane files found for PID ${context.getDataSet()}!")) + expand("No lane files found for PID ${context.dataSet}!")) returnValue = false } } @@ -284,7 +285,7 @@ public class QCPipeline extends Workflow { } def accessProvider = FileSystemAccessProvider.getInstance() - def bamFile = new File(aqcfg.getSingleBamParameter()) + def bamFile = new File(aqcfg.singleBamParameter) if (!accessProvider.fileExists(bamFile) || !accessProvider.isReadable(bamFile)) { context.addErrorEntry(ExecutionContextError.EXECUTION_SETUP_INVALID. expand("A 'bam' parameter was set, but the BAM file is not readable: '${bamFile}'")) @@ -296,7 +297,7 @@ public class QCPipeline extends Workflow { public boolean checkFingerprintingSitesFile(ExecutionContext context) { def aqcfg = new AlignmentAndQCConfig(context) - def accessProvider = FileSystemAccessProvider.getInstance() + def accessProvider = FileSystemAccessProvider.instance boolean result = true if (aqcfg.runFingerprinting) { if (!accessProvider.fileExists(aqcfg.fingerprintingSitesFile) @@ -323,7 +324,8 @@ public class QCPipeline extends Workflow { public boolean createTestdata(ExecutionContext context) { boolean allOk = true; - COProjectsRuntimeService runtimeService = (COProjectsRuntimeService) context.getRuntimeService(); + COProjectsRuntimeService runtimeService = + (COProjectsRuntimeService) context.runtimeService List samples = runtimeService.getSamplesForContext(context); for (Sample sample : samples) { @@ -332,7 +334,7 @@ public class QCPipeline extends Workflow { List rawSequenceGroups = runtimeService.getLanesForSample(context, sample); for (LaneFileGroup lfg : rawSequenceGroups) { - for (LaneFile lf : lfg.getFilesInGroup()) { + for (LaneFile lf : lfg.filesInGroup) { allLaneFiles.addFile(lf); } }