Skip to content

Commit 4f505b4

Browse files
bbimberbbimber
authored
Merge pull request #176 from LabKey/fb_merge_22.7_to_develop
Merge discvr-22.7 to develop
2 parents 63a8e69 + 56adacf commit 4f505b4

31 files changed

+3637
-1020
lines changed

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SamSorter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ else if (sortOrder != SAMFileHeader.SortOrder.coordinate)
9999
input.delete();
100100

101101
//note: if there is a pre-existing index, we need to delete this since it is out of date
102-
File idx = new File(input.getPath() + ".bai");
102+
File idx = SequencePipelineService.get().getExpectedIndex(output);
103103
if (idx.exists())
104104
{
105105
getLogger().debug("deleting old BAM index");

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SamtoolsIndexer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ public SamtoolsIndexer(Logger log)
2222

2323
public File execute(File inputBam) throws PipelineJobException
2424
{
25-
getLogger().info("Sorting SAM/BAM: " + inputBam.getPath());
25+
getLogger().info("Indexing SAM/BAM: " + inputBam.getPath());
2626

27-
File idx = new File(inputBam.getPath() + ".bai");
27+
File idx = SequencePipelineService.get().getExpectedIndex(inputBam);
2828
if (idx.exists())
2929
{
3030
getLogger().debug("deleting existing index: " + idx.getPath());

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ static public void setInstance(SequencePipelineService instance)
112112

113113
abstract public long getLineCount(File f) throws PipelineJobException;
114114

115+
abstract public File getExpectedIndex(File bamOrCram);
116+
115117
abstract public File ensureBamIndex(File f, Logger log, boolean forceDeleteExisting) throws PipelineJobException;
116118

117119
abstract public SAMFileHeader.SortOrder getBamSortOrder(File bam) throws IOException;

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SortSamWrapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public File execute(File inputFile, @Nullable File outputFile, SAMFileHeader.Sor
6161
FileUtils.moveFile(outputBam, inputFile);
6262

6363
//note: if there is a pre-existing index, we need to delete this since it is out of date
64-
File expectedIndex = new File(inputFile.getPath() + ".bai");
64+
File expectedIndex = SequencePipelineService.get().getExpectedIndex(inputFile);
6565
if (expectedIndex.exists())
6666
{
6767
getLogger().info("deleting out of date index: " + expectedIndex.getPath());

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ public String getVersionString() throws PipelineJobException
9797
return StringUtils.trimToNull(executeWithOutput(args));
9898
}
9999

100-
protected List<String> getBaseArgs()
100+
public List<String> getBaseArgs()
101101
{
102102
List<String> args = new ArrayList<>();
103103
args.add(SequencePipelineService.get().getJava8FilePath());

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/ScatterGatherUtils.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package org.labkey.sequenceanalysis;
2+
3+
import org.apache.commons.io.FileUtils;
4+
import org.labkey.api.pipeline.PipelineJobException;
5+
import org.labkey.api.sequenceanalysis.SequenceOutputFile;
6+
import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
7+
import org.labkey.sequenceanalysis.pipeline.VariantProcessingJob;
8+
import org.labkey.sequenceanalysis.run.util.GenotypeGVCFsWrapper;
9+
10+
import java.io.File;
11+
import java.io.IOException;
12+
import java.util.ArrayList;
13+
import java.util.Arrays;
14+
import java.util.List;
15+
16+
public class ScatterGatherUtils
17+
{
18+
public static void doCopyGvcfLocally(List<SequenceOutputFile> inputFiles, SequenceOutputHandler.JobContext ctx) throws PipelineJobException
19+
{
20+
List<File> inputVCFs = new ArrayList<>();
21+
inputFiles.forEach(f -> inputVCFs.add(f.getFile()));
22+
23+
ctx.getLogger().info("making local copies of gVCFs/GenomicsDB");
24+
GenotypeGVCFsWrapper.copyVcfsLocally(ctx, inputVCFs, new ArrayList<>(), false);
25+
}
26+
27+
public static File getLocalCopyDir(SequenceOutputHandler.JobContext ctx, boolean createIfDoesntExist)
28+
{
29+
if (ctx.getJob() instanceof VariantProcessingJob)
30+
{
31+
return ((VariantProcessingJob)ctx.getJob()).getLocationForCachedInputs(ctx.getWorkDir(), createIfDoesntExist);
32+
}
33+
34+
return ctx.getOutputDir();
35+
}
36+
37+
public static void possiblyCacheSupportFiles(SequenceOutputHandler.JobContext ctx) throws PipelineJobException
38+
{
39+
for (String param : Arrays.asList("exclude_intervals", "forceSitesFile"))
40+
{
41+
if (ctx.getParams().get("variantCalling.GenotypeGVCFs." + param) != null)
42+
{
43+
File inputFile = ctx.getSequenceSupport().getCachedData(ctx.getParams().getInt("variantCalling.GenotypeGVCFs." + param));
44+
if (!inputFile.exists())
45+
{
46+
throw new PipelineJobException("Unable to find file: " + inputFile.getPath());
47+
}
48+
49+
ctx.getLogger().debug("Making local copy of file: " + inputFile.getName());
50+
File localCopy = new File(ScatterGatherUtils.getLocalCopyDir(ctx, true), inputFile.getName());
51+
File doneFile = new File(localCopy.getPath() + ".copyDone");
52+
if (!doneFile.exists())
53+
{
54+
try
55+
{
56+
FileUtils.copyFile(inputFile, localCopy);
57+
FileUtils.touch(doneFile);
58+
}
59+
catch (IOException e)
60+
{
61+
throw new PipelineJobException(e);
62+
}
63+
}
64+
}
65+
}
66+
}
67+
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,7 @@ public BasePipelineStepAction(String htmlFile)
389389
_htmlFile = htmlFile;
390390
}
391391

392+
@Override
392393
public ModelAndView getView(Object form, BindException errors)
393394
{
394395
LinkedHashSet<ClientDependency> cds = new LinkedHashSet<>();

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java‎

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -125,24 +125,7 @@
125125
import org.labkey.sequenceanalysis.run.util.FastqcRunner;
126126
import org.labkey.sequenceanalysis.run.util.GenomicsDBAppendHandler;
127127
import org.labkey.sequenceanalysis.run.util.GenomicsDBImportHandler;
128-
import org.labkey.sequenceanalysis.run.variant.DepthOfCoverageHandler;
129-
import org.labkey.sequenceanalysis.run.variant.GenotypeConcordanceStep;
130-
import org.labkey.sequenceanalysis.run.variant.GenotypeFiltrationStep;
131-
import org.labkey.sequenceanalysis.run.variant.MendelianViolationReportStep;
132-
import org.labkey.sequenceanalysis.run.variant.MergeVcfsAndGenotypesHandler;
133-
import org.labkey.sequenceanalysis.run.variant.MultiAllelicPositionsHandler;
134-
import org.labkey.sequenceanalysis.run.variant.PlinkPcaStep;
135-
import org.labkey.sequenceanalysis.run.variant.SNPEffStep;
136-
import org.labkey.sequenceanalysis.run.variant.SampleRenameStep;
137-
import org.labkey.sequenceanalysis.run.variant.SelectSNVsStep;
138-
import org.labkey.sequenceanalysis.run.variant.SelectSamplesStep;
139-
import org.labkey.sequenceanalysis.run.variant.SelectVariantsStep;
140-
import org.labkey.sequenceanalysis.run.variant.VariantAnnotatorStep;
141-
import org.labkey.sequenceanalysis.run.variant.VariantEvalBySampleStep;
142-
import org.labkey.sequenceanalysis.run.variant.VariantEvalStep;
143-
import org.labkey.sequenceanalysis.run.variant.VariantFiltrationStep;
144-
import org.labkey.sequenceanalysis.run.variant.VariantQCStep;
145-
import org.labkey.sequenceanalysis.run.variant.VariantsToTableStep;
128+
import org.labkey.sequenceanalysis.run.variant.*;
146129
import org.labkey.sequenceanalysis.util.Barcoder;
147130
import org.labkey.sequenceanalysis.util.ChainFileValidator;
148131
import org.labkey.sequenceanalysis.util.ScatterGatherUtils;
@@ -313,13 +296,15 @@ public static void registerPipelineSteps()
313296
SequencePipelineService.get().registerPipelineStep(new VariantFiltrationStep.Provider());
314297
SequencePipelineService.get().registerPipelineStep(new GenotypeConcordanceStep.Provider());
315298
SequencePipelineService.get().registerPipelineStep(new SampleRenameStep.Provider());
299+
SequencePipelineService.get().registerPipelineStep(new SplitVcfBySamplesStep.Provider());
316300

317301
SequencePipelineService.get().registerPipelineStep(new VariantEvalStep.Provider());
318302
SequencePipelineService.get().registerPipelineStep(new VariantEvalBySampleStep.Provider());
319303
SequencePipelineService.get().registerPipelineStep(new VariantsToTableStep.Provider());
320304
SequencePipelineService.get().registerPipelineStep(new VariantQCStep.Provider());
321305
SequencePipelineService.get().registerPipelineStep(new PlinkPcaStep.Provider());
322306
SequencePipelineService.get().registerPipelineStep(new MendelianViolationReportStep.Provider());
307+
SequencePipelineService.get().registerPipelineStep(new SummarizeGenotypeQualityStep.Provider());
323308

324309
//handlers
325310
SequenceAnalysisService.get().registerFileHandler(new LiftoverHandler());

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.labkey.api.sequenceanalysis.pipeline.PipelineStepCtx;
1818
import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider;
1919
import org.labkey.api.sequenceanalysis.pipeline.PreprocessingStep;
20+
import org.labkey.api.sequenceanalysis.pipeline.SamtoolsIndexer;
2021
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
2122
import org.labkey.api.sequenceanalysis.pipeline.TaskFileManager;
2223
import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper;
@@ -437,10 +438,16 @@ public long getLineCount(File f) throws PipelineJobException
437438
return SequenceUtil.getLineCount(f);
438439
}
439440

441+
@Override
442+
public File getExpectedIndex(File bamOrCram)
443+
{
444+
return SequenceUtil.getExpectedIndex(bamOrCram);
445+
}
446+
440447
@Override
441448
public File ensureBamIndex(File inputBam, Logger log, boolean forceDeleteExisting) throws PipelineJobException
442449
{
443-
File expectedIndex = new File(inputBam.getPath() + ".bai");
450+
File expectedIndex = SequenceUtil.getExpectedIndex(inputBam);
444451
if (expectedIndex.exists() && (expectedIndex.lastModified() < inputBam.lastModified() || forceDeleteExisting))
445452
{
446453
log.info("deleting out of date index: " + expectedIndex.getPath());
@@ -449,10 +456,8 @@ public File ensureBamIndex(File inputBam, Logger log, boolean forceDeleteExistin
449456

450457
if (!expectedIndex.exists())
451458
{
452-
log.debug("\tcreating temp index for BAM: " + inputBam.getName());
453-
BuildBamIndexWrapper buildBamIndexWrapper = new BuildBamIndexWrapper(log);
454-
buildBamIndexWrapper.setStringency(ValidationStringency.SILENT);
455-
buildBamIndexWrapper.executeCommand(inputBam);
459+
log.debug("\tcreating index for BAM: " + inputBam.getName());
460+
new SamtoolsIndexer(log).execute(inputBam);
456461

457462
return expectedIndex;
458463
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/GenotypeGVCFHandler.java‎

Lines changed: 4 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.labkey.api.util.FileType;
3535
import org.labkey.api.view.ActionURL;
3636
import org.labkey.api.writer.PrintWriters;
37+
import org.labkey.sequenceanalysis.ScatterGatherUtils;
3738
import org.labkey.sequenceanalysis.SequenceAnalysisModule;
3839
import org.labkey.sequenceanalysis.pipeline.JobContextImpl;
3940
import org.labkey.sequenceanalysis.pipeline.ProcessVariantsHandler;
@@ -783,63 +784,13 @@ public boolean isRequired(PipelineJob job)
783784
@Override
784785
public void doWork(List<SequenceOutputFile> inputFiles, JobContext ctx) throws PipelineJobException
785786
{
786-
doCopyGvcfLocally(inputFiles, ctx);
787-
possiblyCacheSupportFiles(ctx);
788-
}
789-
790-
public static void possiblyCacheSupportFiles(JobContext ctx) throws PipelineJobException
791-
{
792-
for (String param : Arrays.asList("exclude_intervals", "forceSitesFile"))
793-
{
794-
if (ctx.getParams().get("variantCalling.GenotypeGVCFs." + param) != null)
795-
{
796-
File inputFile = ctx.getSequenceSupport().getCachedData(ctx.getParams().getInt("variantCalling.GenotypeGVCFs." + param));
797-
if (!inputFile.exists())
798-
{
799-
throw new PipelineJobException("Unable to find file: " + inputFile.getPath());
800-
}
801-
802-
ctx.getLogger().debug("Making local copy of file: " + inputFile.getName());
803-
File localCopy = new File(getLocalCopyDir(ctx, true), inputFile.getName());
804-
File doneFile = new File(localCopy.getPath() + ".copyDone");
805-
if (!doneFile.exists())
806-
{
807-
try
808-
{
809-
FileUtils.copyFile(inputFile, localCopy);
810-
FileUtils.touch(doneFile);
811-
}
812-
catch (IOException e)
813-
{
814-
throw new PipelineJobException(e);
815-
}
816-
}
817-
}
818-
}
819-
}
820-
821-
public static void doCopyGvcfLocally(List<SequenceOutputFile> inputFiles, JobContext ctx) throws PipelineJobException
822-
{
823-
List<File> inputVCFs = new ArrayList<>();
824-
inputFiles.forEach(f -> inputVCFs.add(f.getFile()));
825-
826-
ctx.getLogger().info("making local copies of gVCFs/GenomicsDB");
827-
GenotypeGVCFsWrapper.copyVcfsLocally(ctx, inputVCFs, new ArrayList<>(), false);
828-
}
829-
830-
public static File getLocalCopyDir(JobContext ctx, boolean createIfDoesntExist)
831-
{
832-
if (ctx.getJob() instanceof VariantProcessingJob)
833-
{
834-
return ((VariantProcessingJob)ctx.getJob()).getLocationForCachedInputs(ctx.getWorkDir(), createIfDoesntExist);
835-
}
836-
837-
return ctx.getOutputDir();
787+
ScatterGatherUtils.doCopyGvcfLocally(inputFiles, ctx);
788+
ScatterGatherUtils.possiblyCacheSupportFiles(ctx);
838789
}
839790

840791
protected File getPossiblyLocalFile(JobContext ctx, File sourceFile)
841792
{
842-
File cacheDir = getLocalCopyDir(ctx, false);
793+
File cacheDir = ScatterGatherUtils.getLocalCopyDir(ctx, false);
843794
if (!cacheDir.exists())
844795
{
845796
return sourceFile;

0 commit comments

Comments
 (0)