Skip to content

Commit b2a83a5

Browse files
bbimberbbimber
authored
Merge pull request #226 from BimberLab/23.3_fb_java8
Backport switch from java8->newer java to fix tests
2 parents da136fe + c65f6e8 commit b2a83a5

7 files changed

Lines changed: 12 additions & 238 deletions

File tree

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -48,21 +48,6 @@ public void setMaxRamOverride(Integer maxRamOverride)
4848
_maxRamOverride = maxRamOverride;
4949
}
5050

51-
public void addJava8HomeToEnvironment()
52-
{
53-
//since GATK requires java8, set JAVA_HOME to match this:
54-
File java8 = new File(SequencePipelineService.get().getJava8FilePath()).getParentFile();
55-
if (java8.getParentFile() == null)
56-
{
57-
getLogger().debug("unexpected path to java8, cannot determine JAVA_HOME: " + java8.getPath());
58-
return;
59-
}
60-
61-
String javaDir = java8.getParentFile().getPath();
62-
getLogger().debug("setting JAVA_HOME to java8 location: " + javaDir);
63-
addToEnvironment("JAVA_HOME", javaDir);
64-
}
65-
6651
public boolean jarExists()
6752
{
6853
return getJAR(false) != null;
@@ -80,7 +65,7 @@ protected void ensureDictionary(File referenceFasta) throws PipelineJobException
8065
public String getVersionString() throws PipelineJobException
8166
{
8267
List<String> args = new ArrayList<>();
83-
args.add(SequencePipelineService.get().getJava8FilePath());
68+
args.add(SequencePipelineService.get().getJavaFilepath());
8469
args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
8570
args.add("-jar");
8671
args.add(getJAR().getPath());
@@ -102,7 +87,7 @@ protected String getPackageName()
10287
public List<String> getBaseArgs(@Nullable String toolName)
10388
{
10489
List<String> args = new ArrayList<>();
105-
args.add(SequencePipelineService.get().getJava8FilePath());
90+
args.add(SequencePipelineService.get().getJavaFilepath());
10691
args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
10792
args.add("-jar");
10893
args.add(getJAR().getPath());

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatkWrapper.java‎

Lines changed: 0 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -54,38 +54,6 @@ public void setMaxRamOverride(Integer maxRamOverride)
5454
_maxRamOverride = maxRamOverride;
5555
}
5656

57-
protected void addJavaHomeToEnvironment()
58-
{
59-
//since GATK requires java8, set JAVA_HOME to match this:
60-
File java8 = new File(SequencePipelineService.get().getJava8FilePath()).getParentFile();
61-
if (java8.getParentFile() == null)
62-
{
63-
getLogger().debug("unexpected path to java8, cannot determine JAVA_HOME: " + java8.getPath());
64-
return;
65-
}
66-
67-
String javaDir = java8.getParentFile().getPath();
68-
getLogger().debug("setting JAVA_HOME to java8 location: " + javaDir);
69-
addToEnvironment("JAVA_HOME", javaDir);
70-
}
71-
72-
protected File getQueueJAR()
73-
{
74-
String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("GATKPATH");
75-
if (path != null)
76-
{
77-
return new File(path);
78-
}
79-
80-
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM);
81-
if (path == null)
82-
{
83-
path = PipelineJobService.get().getAppProperties().getToolsDirectory();
84-
}
85-
86-
return path == null ? new File("Queue.jar") : new File(path, "Queue.jar");
87-
}
88-
8957
public boolean jarExists()
9058
{
9159
return getJAR() == null || !getJAR().exists();
@@ -100,69 +68,6 @@ protected void ensureDictionary(File referenceFasta) throws PipelineJobException
10068
new CreateSequenceDictionaryWrapper(getLogger()).execute(referenceFasta, false);
10169
}
10270

103-
public String getVersionString() throws PipelineJobException
104-
{
105-
List<String> args = new ArrayList<>();
106-
args.add(SequencePipelineService.get().getJava8FilePath());
107-
args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
108-
args.add("-jar");
109-
args.add(getJAR().getPath());
110-
args.add("--version");
111-
112-
return StringUtils.trimToNull(executeWithOutput(args));
113-
}
114-
115-
public Integer getMinRamPerQueueJob()
116-
{
117-
return _minRamPerQueueJob;
118-
}
119-
120-
public void setMinRamPerQueueJob(Integer minRamPerQueueJob)
121-
{
122-
_minRamPerQueueJob = minRamPerQueueJob;
123-
}
124-
125-
protected Integer getScatterForQueueJob()
126-
{
127-
// NOTE: Queue will create n number of jobs, dividing memory evenly between them. Because it is possible
128-
// to submit a job w/ lower available RAM and comparably high CPUs, this could result in queue not having enough memory per job.
129-
// therefore do a quick check and potentially scale down scatter
130-
Integer maxThreads = SequencePipelineService.get().getMaxThreads(getLogger());
131-
if (maxThreads != null)
132-
{
133-
if (_minRamPerQueueJob != null && _minRamPerQueueJob > 0)
134-
{
135-
String maxRamSetting = StringUtils.trimToNull(System.getenv("SEQUENCEANALYSIS_MAX_RAM"));
136-
if (maxRamSetting != null)
137-
{
138-
try
139-
{
140-
Integer maxRamAllowed = ConvertHelper.convert(maxRamSetting, Integer.class);
141-
if (maxRamAllowed != null)
142-
{
143-
int adjusted = Math.max(maxRamAllowed / _minRamPerQueueJob, 1);
144-
if (adjusted < maxThreads)
145-
{
146-
getLogger().debug("lowering max threads to match available RAM. setting to: " + adjusted);
147-
maxThreads = adjusted;
148-
}
149-
}
150-
}
151-
catch (ConvergenceException e)
152-
{
153-
getLogger().warn("non-numeric value for SEQUENCEANALYSIS_MAX_RAM: [" + maxRamSetting + "]");
154-
}
155-
}
156-
}
157-
}
158-
else
159-
{
160-
maxThreads = 1;
161-
}
162-
163-
return maxThreads;
164-
}
165-
16671
protected List<String> getBaseArgs()
16772
{
16873
List<String> args = new ArrayList<>();

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/PicardWrapper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public String getVersion() throws PipelineJobException
3636
}
3737

3838
List<String> params = new LinkedList<>();
39-
params.add(SequencePipelineService.get().getJava8FilePath());
39+
params.add(SequencePipelineService.get().getJavaFilepath());
4040
params.add("-jar");
4141
params.add(getJar().getPath());
4242
params.add(getToolName());
@@ -95,7 +95,7 @@ protected List<String> getBaseArgs(boolean basicArgsOnly) throws PipelineJobExce
9595
}
9696

9797
List<String> params = new LinkedList<>();
98-
params.add(SequencePipelineService.get().getJava8FilePath());
98+
params.add(SequencePipelineService.get().getJavaFilepath());
9999
params.addAll(SequencePipelineService.get().getJavaOpts());
100100
params.add("-jar");
101101
params.add(getJar().getPath());

‎SequenceAnalysis/pipeline_code/sequence_tools_install.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -276,10 +276,10 @@ then
276276
rm -Rf gatk-4*
277277
rm -Rf $LKTOOLS_DIR/GenomeAnalysisTK4.jar
278278

279-
wget $WGET_OPTS https://github.com/broadinstitute/gatk/releases/download/4.3.0.0/gatk-4.3.0.0.zip
280-
unzip gatk-4.3.0.0.zip
279+
wget $WGET_OPTS https://github.com/broadinstitute/gatk/releases/download/4.4.0.0/gatk-4.4.0.0.zip
280+
unzip gatk-4.4.0.0.zip
281281

282-
cp ./gatk-4.3.0.0/gatk-package-4.3.0.0-local.jar $LKTOOLS_DIR/GenomeAnalysisTK4.jar
282+
cp ./gatk-4.4.0.0/gatk-package-4.4.0.0-local.jar $LKTOOLS_DIR/GenomeAnalysisTK4.jar
283283
else
284284
echo "Already installed"
285285
fi
@@ -695,7 +695,7 @@ then
695695
rm -Rf $LKTOOLS_DIR/htsjdk-*
696696
rm -Rf $LKTOOLS_DIR/libIntelDeflater.so
697697

698-
wget $WGET_OPTS https://github.com/broadinstitute/picard/releases/download/2.27.4/picard.jar
698+
wget $WGET_OPTS https://github.com/broadinstitute/picard/releases/download/3.0.0/picard.jar
699699

700700
cp -R ./picard.jar $LKTOOLS_DIR/
701701
else

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/bampostprocessing/IndelRealignerStep.java‎

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
import java.io.File;
1818
import java.util.Arrays;
19+
import java.util.Collections;
1920

2021
/**
2122
* User: bimber
@@ -24,7 +25,7 @@
2425
*/
2526
public class IndelRealignerStep extends AbstractCommandPipelineStep<IndelRealignerWrapper> implements BamProcessingStep
2627
{
27-
public IndelRealignerStep(PipelineStepProvider provider, PipelineContext ctx)
28+
public IndelRealignerStep(PipelineStepProvider<?> provider, PipelineContext ctx)
2829
{
2930
super(provider, ctx, new IndelRealignerWrapper(ctx.getLogger()));
3031
}
@@ -33,17 +34,7 @@ public static class Provider extends AbstractPipelineStepProvider<IndelRealigner
3334
{
3435
public Provider()
3536
{
36-
super("IndelRealigner", "Indel Realigner", "GATK", "The step runs GATK's IndelRealigner tool. This tools performs local realignment to minmize the number of mismatching bases across all the reads.", Arrays.asList(
37-
ToolParameterDescriptor.create("useQueue", "Use Queue?", "If checked, this tool will attempt to run using GATK queue, allowing parallelization using scatter/gather.", "checkbox", new JSONObject()
38-
{{
39-
put("checked", false);
40-
}}, false),
41-
//TODO: consider supporting:
42-
//--maxReadsForRealignment
43-
//--maxReadsForConsensuses
44-
45-
ToolParameterDescriptor.create("minRamPerQueueJob", "Min RAM Per Queue Job", "This only applies if queue is checked. If provided, the scatter count (number of jobs) for queue will be adjusted to ensure at least this amount of RAM, in GB, is available for each job", "ldk-integerfield", null, null)
46-
), null, "http://www.broadinstitute.org/gatk/gatkdocs/org_broadinstitute_sting_gatk_walkers_indels_IndelRealigner.html");
37+
super("IndelRealigner", "Indel Realigner", "GATK", "The step runs GATK's IndelRealigner tool. This tools performs local realignment to minmize the number of mismatching bases across all the reads.", Collections.emptyList(), null, "http://www.broadinstitute.org/gatk/gatkdocs/org_broadinstitute_sting_gatk_walkers_indels_IndelRealigner.html");
4738
}
4839

4940
@Override
@@ -65,22 +56,7 @@ public Output processBam(Readset rs, File inputBam, ReferenceGenome referenceGen
6556
getPipelineCtx().getLogger().debug("dict exists: " + preExistingDictionary + ", " + dictionary.getPath());
6657

6758
File outputBam = new File(outputDirectory, FileUtil.getBaseName(inputBam) + ".realigned.bam");
68-
File created;
69-
if (getProvider().getParameterByName("useQueue").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false))
70-
{
71-
Integer minRamPerQueueJob = getProvider().getParameterByName("minRamPerQueueJob").extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Integer.class);
72-
if (minRamPerQueueJob != null)
73-
{
74-
getWrapper().setMinRamPerQueueJob(minRamPerQueueJob);
75-
}
76-
77-
created = getWrapper().executeWithQueue(inputBam, outputBam, referenceGenome.getWorkingFastaFile(), null);
78-
}
79-
else
80-
{
81-
created = getWrapper().execute(inputBam, outputBam, referenceGenome.getWorkingFastaFile(), null);
82-
}
83-
59+
File created = getWrapper().execute(inputBam, outputBam, referenceGenome.getWorkingFastaFile(), null);
8460
if (created != null)
8561
{
8662
output.setBAM(created);

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/IndelRealignerWrapper.java‎

Lines changed: 0 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -89,92 +89,6 @@ public File execute(File inputBam, @Nullable File outputBam, File referenceFasta
8989
return processOutput(tempFiles, inputBam, outputBam, realignedBam);
9090
}
9191

92-
public File executeWithQueue(File inputBam, File outputBam, File referenceFasta, @Nullable File knownIndelsVcf) throws PipelineJobException
93-
{
94-
getLogger().info("Running GATK IndelRealigner using Queue for: " + inputBam.getName());
95-
addJavaHomeToEnvironment();
96-
97-
List<File> tempFiles = new ArrayList<>();
98-
File workingBam = performSharedWork(inputBam, outputBam, referenceFasta, tempFiles);
99-
if (!workingBam.equals(inputBam))
100-
{
101-
tempFiles.add(workingBam);
102-
}
103-
104-
File intervalsFile = buildTargetIntervals(referenceFasta, workingBam, knownIndelsVcf, getExpectedIntervalsFile(inputBam));
105-
if (intervalsFile == null)
106-
{
107-
getLogger().info("no intervals to realign, skipping");
108-
return processOutput(tempFiles, inputBam, outputBam, null);
109-
}
110-
111-
try
112-
{
113-
Module module = ModuleLoader.getInstance().getModule(SequenceAnalysisModule.class);
114-
FileResource r = (FileResource)module.getModuleResolver().lookup(Path.parse("external/qscript/IndelRealignerRunner.scala"));
115-
File scalaScript = r.getFile();
116-
117-
if (scalaScript == null)
118-
throw new FileNotFoundException("Not found: " + scalaScript);
119-
120-
if (!scalaScript.exists())
121-
throw new FileNotFoundException("Not found: " + scalaScript.getPath());
122-
123-
List<String> args = new ArrayList<>();
124-
args.add(SequencePipelineService.get().getJava8FilePath());
125-
//for now, ignore java opts since queue's scatter/gather causes issues
126-
//args.addAll(SequencePipelineService.get().getJavaOpts());
127-
args.add("-classpath");
128-
args.add(getJAR().getPath());
129-
args.addAll(SequencePipelineService.get().getJavaOpts());
130-
args.add("-jar");
131-
args.add(getQueueJAR().getPath());
132-
args.add("-S");
133-
args.add(scalaScript.getPath());
134-
args.add("-jobRunner");
135-
args.add("ParallelShell");
136-
args.add("-run");
137-
138-
args.add("-R");
139-
args.add(referenceFasta.getPath());
140-
args.add("-I");
141-
args.add(workingBam.getPath());
142-
args.add("-targetIntervals");
143-
args.add(intervalsFile.getPath());
144-
145-
args.add("-runDir");
146-
args.add(outputBam.getParentFile().getPath());
147-
148-
String tmpDir = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("JAVA_TMP_DIR");
149-
if (StringUtils.trimToNull(tmpDir) != null)
150-
{
151-
args.add("-tempDir");
152-
args.add(tmpDir);
153-
}
154-
155-
args.add("-o");
156-
157-
File realignedBam = outputBam == null ? new File(getOutputDir(inputBam), FileUtil.getBaseName(inputBam) + ".realigned.bam") : outputBam;
158-
args.add(realignedBam.getPath());
159-
160-
args.add("-startFromScratch");
161-
args.add("-scatterCount");
162-
args.add(getScatterForQueueJob().toString());
163-
164-
execute(args);
165-
if (!realignedBam.exists())
166-
{
167-
throw new PipelineJobException("Expected output not found: " + realignedBam.getPath());
168-
}
169-
170-
return processOutput(tempFiles, inputBam, outputBam, realignedBam);
171-
}
172-
catch (IOException e)
173-
{
174-
throw new PipelineJobException(e);
175-
}
176-
}
177-
17892
private File processOutput(List<File> tempFiles, File inputBam, File outputBam, File realignedBam) throws PipelineJobException
17993
{
18094
if (!tempFiles.isEmpty())

‎jbrowse/src/org/labkey/jbrowse/model/JsonFile.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -972,8 +972,6 @@ private void prepareLuceneIndex(Logger log) throws PipelineJobException
972972
log.debug("Generating VCF full text index for file: " + getExpData().getFile().getName());
973973

974974
DISCVRSeqRunner runner = new DISCVRSeqRunner(log);
975-
runner.addJava8HomeToEnvironment();
976-
977975
if (!runner.jarExists())
978976
{
979977
log.error("Unable to find DISCVRSeq.jar, skiping lucene index creation");
@@ -994,10 +992,6 @@ private void prepareLuceneIndex(Logger log) throws PipelineJobException
994992
args.add(field);
995993
}
996994

997-
// Always include this:
998-
args.add("-AN");
999-
args.add("SampleList");
1000-
1001995
runner.execute(args);
1002996
}
1003997

0 commit comments

Comments
 (0)