Skip to content

Commit 62421f9

Browse files
bbimberbbimber
authored
Merge pull request #203 from LabKey/fb_merge_22.11_to_develop
Merge discvr-22.11 to develop
2 parents 879fa77 + 7bc2a1c commit 62421f9

File tree

32 files changed

+778
-152
lines changed

32 files changed

+778
-152
lines changed

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.jetbrains.annotations.Nullable;
2323
import org.labkey.api.pipeline.PipelineJobException;
2424
import org.labkey.api.pipeline.PipelineJobService;
25+
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
2526
import org.labkey.api.util.StringUtilsLabKey;
2627

2728
import java.io.BufferedReader;
@@ -287,4 +288,60 @@ protected boolean isThrowNonZeroExits()
287288
{
288289
return _throwNonZeroExits;
289290
}
291+
292+
protected static File resolveFileInPath(String exe, @Nullable String packageName, boolean throwIfNotFound)
293+
{
294+
File fn;
295+
String path;
296+
297+
if (packageName != null)
298+
{
299+
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(packageName);
300+
if (path != null)
301+
{
302+
fn = new File(path, exe);
303+
if (fn.exists())
304+
{
305+
return fn;
306+
}
307+
}
308+
}
309+
310+
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM);
311+
if (path != null)
312+
{
313+
fn = new File(path, exe);
314+
if (fn.exists())
315+
{
316+
return fn;
317+
}
318+
}
319+
320+
path = PipelineJobService.get().getAppProperties().getToolsDirectory();
321+
if (path != null)
322+
{
323+
fn = new File(path, exe);
324+
if (fn.exists())
325+
{
326+
return fn;
327+
}
328+
}
329+
330+
String[] paths = System.getenv("PATH").split(File.pathSeparator);
331+
for (String pathDir : paths)
332+
{
333+
fn = new File(pathDir, exe);
334+
if (fn.exists())
335+
{
336+
return fn;
337+
}
338+
}
339+
340+
if (throwIfNotFound)
341+
{
342+
throw new IllegalArgumentException("Unable to find file: "+ exe);
343+
}
344+
345+
return null;
346+
}
290347
}

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import org.apache.logging.log4j.Logger;
66
import org.jetbrains.annotations.Nullable;
77
import org.labkey.api.pipeline.PipelineJobException;
8-
import org.labkey.api.pipeline.PipelineJobService;
98
import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
109
import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
1110
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
@@ -34,21 +33,14 @@ protected String getJarName()
3433
return "GenomeAnalysisTK4.jar";
3534
}
3635

37-
protected File getJAR()
36+
public File getJAR()
3837
{
39-
String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(getPackageName());
40-
if (path != null)
41-
{
42-
return new File(path);
43-
}
44-
45-
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM);
46-
if (path == null)
47-
{
48-
path = PipelineJobService.get().getAppProperties().getToolsDirectory();
49-
}
38+
return getJAR(true);
39+
}
5040

51-
return path == null ? new File(getJarName()) : new File(path, getJarName());
41+
public File getJAR(boolean throwIfNotFound)
42+
{
43+
return resolveFileInPath(getJarName(), getPackageName(), throwIfNotFound);
5244
}
5345

5446
public void setMaxRamOverride(Integer maxRamOverride)
@@ -73,7 +65,7 @@ public void addJava8HomeToEnvironment()
7365

7466
public boolean jarExists()
7567
{
76-
return getJAR() == null || !getJAR().exists();
68+
return getJAR(false) != null;
7769
}
7870

7971
protected void ensureDictionary(File referenceFasta) throws PipelineJobException

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/PicardWrapper.java‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ public PicardWrapper(@Nullable Logger logger)
3030

3131
public String getVersion() throws PipelineJobException
3232
{
33+
if (!jarExists())
34+
{
35+
throw new PipelineJobException("Unable to find picard.jar");
36+
}
37+
3338
List<String> params = new LinkedList<>();
3439
params.add(SequencePipelineService.get().getJava8FilePath());
3540
params.add("-jar");
@@ -49,26 +54,20 @@ public String getVersion() throws PipelineJobException
4954
return ret;
5055
}
5156

52-
public static File getPicardJar()
57+
public static @Nullable File getPicardJar(boolean throwIfNotFound)
5358
{
54-
String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("PICARDPATH");
55-
if (path != null)
56-
{
57-
return new File(path);
58-
}
59-
60-
path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM);
61-
if (path == null)
62-
{
63-
path = PipelineJobService.get().getAppProperties().getToolsDirectory();
64-
}
59+
return resolveFileInPath("picard.jar", "PICARDPATH", throwIfNotFound);
60+
}
6561

66-
return path == null ? null : new File(path, "picard.jar");
62+
public boolean jarExists()
63+
{
64+
File jar = getPicardJar(false);
65+
return jar != null && jar.exists();
6766
}
6867

6968
protected File getJar()
7069
{
71-
return getPicardJar();
70+
return getPicardJar(false);
7271
}
7372

7473
public ValidationStringency getStringency()
@@ -83,13 +82,18 @@ public void setStringency(ValidationStringency stringency)
8382

8483
abstract protected String getToolName();
8584

86-
protected List<String> getBaseArgs()
85+
protected List<String> getBaseArgs() throws PipelineJobException
8786
{
8887
return getBaseArgs(false);
8988
}
9089

91-
protected List<String> getBaseArgs(boolean basicArgsOnly)
90+
protected List<String> getBaseArgs(boolean basicArgsOnly) throws PipelineJobException
9291
{
92+
if (!jarExists())
93+
{
94+
throw new PipelineJobException("Unable to find picard.jar");
95+
}
96+
9397
List<String> params = new LinkedList<>();
9498
params.add(SequencePipelineService.get().getJava8FilePath());
9599
params.addAll(SequencePipelineService.get().getJavaOpts());

‎SequenceAnalysis/pipeline_code/sequence_tools_install.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -851,9 +851,15 @@ then
851851
rm -Rf $LKTOOLS_DIR/_preamble.py
852852
rm -Rf $LKTOOLS_DIR/cutadapt_pip
853853

854-
PIP_VERSION=`pip -V | cut -d '(' -f 2 | sed 's/python //' | cut -c 1 2>1`
854+
if [ -x "$(command -v pip3)" ];then
855+
PIP_EXE=`command -v pip3`
856+
else
857+
PIP_EXE=pip
858+
fi
855859

860+
PIP_VERSION=`$PIP_EXE -V | cut -d '(' -f 2 | sed 's/python //' | cut -c 1 2>1`
856861
if [[ $PIP_VERSION == '2' ]];then
862+
echo 'Using python 2 compatible cutadapt'
857863
wget https://pypi.python.org/packages/source/c/cutadapt/cutadapt-1.8.1.tar.gz
858864
gunzip cutadapt-1.8.1.tar.gz
859865
tar -xf cutadapt-1.8.1.tar
@@ -865,7 +871,7 @@ then
865871
cp -R ./cutadapt-1.8.1/cutadapt ${LKTOOLS_DIR}/cutadapt
866872
else
867873
CUTADAPT_BRANCH=v4.0
868-
pip install --target ${LKTOOLS_DIR}/cutadapt_pip git+https://github.com/marcelm/cutadapt.git@${CUTADAPT_BRANCH}
874+
$PIP_EXE install --target ${LKTOOLS_DIR}/cutadapt_pip git+https://github.com/marcelm/cutadapt.git@${CUTADAPT_BRANCH}
869875
ln -s ${LKTOOLS_DIR}/cutadapt_pip/bin/cutadapt ${LKTOOLS_DIR}/cutadapt
870876
fi
871877
else

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ private void processContainer(Container c, Logger log) throws IOException, Pipel
309309
if (!gz.exists())
310310
{
311311
ReferenceGenomeImpl genome = new ReferenceGenomeImpl(fasta, fastaData, libraryId, null);
312-
log.error("GZipped genome missing for: " + genome.getName());
312+
log.error("GZipped genome missing for: " + genome.getGenomeId());
313313

314314
if (SystemUtils.IS_OS_WINDOWS)
315315
{

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package org.labkey.sequenceanalysis;
22

33
import htsjdk.samtools.SAMFileHeader;
4-
import htsjdk.samtools.ValidationStringency;
54
import org.apache.commons.io.FilenameUtils;
65
import org.apache.commons.lang3.StringUtils;
76
import org.apache.logging.log4j.LogManager;
@@ -32,9 +31,9 @@
3231
import org.labkey.sequenceanalysis.pipeline.TaskFileManagerImpl;
3332
import org.labkey.sequenceanalysis.run.preprocessing.PreprocessingOutputImpl;
3433
import org.labkey.sequenceanalysis.run.preprocessing.TrimmomaticWrapper;
35-
import org.labkey.sequenceanalysis.run.util.BuildBamIndexWrapper;
3634
import org.labkey.sequenceanalysis.run.util.SortVcfWrapper;
3735
import org.labkey.sequenceanalysis.util.FastqUtils;
36+
import org.labkey.sequenceanalysis.util.ReferenceLibraryHelperImpl;
3837
import org.labkey.sequenceanalysis.util.SequenceUtil;
3938

4039
import java.io.File;
@@ -231,7 +230,26 @@ public <StepType extends PipelineStep> List<PipelineStepCtx<StepType>> getSteps(
231230
@Override
232231
public void ensureSequenceDictionaryExists(File referenceFasta, Logger log, boolean forceRecreate) throws PipelineJobException
233232
{
234-
new CreateSequenceDictionaryWrapper(log).execute(referenceFasta, false);
233+
CreateSequenceDictionaryWrapper wrapper = new CreateSequenceDictionaryWrapper(log);
234+
if (wrapper.jarExists())
235+
{
236+
new CreateSequenceDictionaryWrapper(log).execute(referenceFasta, false);
237+
return;
238+
}
239+
240+
log.debug("picard.jar not found, creating directly");
241+
ReferenceLibraryHelperImpl helper = new ReferenceLibraryHelperImpl(referenceFasta, log);
242+
if (forceRecreate)
243+
{
244+
File dict = helper.getSequenceDictionaryFile(false);
245+
if (dict.exists())
246+
{
247+
log.debug("Deleting pre-existing dictionary: " + dict.getPath());
248+
dict.delete();
249+
}
250+
}
251+
252+
helper.getSequenceDictionaryFile(true);
235253
}
236254

237255
@Override

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,7 @@ public RecordedActionSet run() throws PipelineJobException
353353
CreateSequenceDictionaryWrapper wrapper = new CreateSequenceDictionaryWrapper(getJob().getLogger());
354354
wrapper.execute(fasta, true);
355355
}
356-
catch (PipelineJobException e)
356+
catch (PipelineJobException | IllegalArgumentException e)
357357
{
358358
getJob().getLogger().warn("Unable to create sequence dictionary");
359359
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -230,15 +230,13 @@ private int addTrackForLibrary(File file, String trackName, String trackDescript
230230
}
231231

232232
SAMSequenceDictionary dict = null;
233-
if (genome.getSequenceDictionary().exists())
233+
if (!genome.getSequenceDictionary().exists())
234234
{
235-
dict = SAMSequenceDictionaryExtractor.extractDictionary(genome.getSequenceDictionary().toPath());
236-
}
237-
else
238-
{
239-
getJob().getLogger().warn("genome dictionary not found, will not be able to import some kinds of tracks");
235+
SequencePipelineService.get().ensureSequenceDictionaryExists(genome.getWorkingFastaFile(), getJob().getLogger(), false);
240236
}
241237

238+
dict = SAMSequenceDictionaryExtractor.extractDictionary(genome.getSequenceDictionary().toPath());
239+
242240
if (SequenceUtil.FILETYPE.bed.getFileType().isType(file))
243241
{
244242
validateAndTransformTsv(file, outputFile, nameTranslationMap, knownChrs, 0, 1, 2, -1, dict, action);
@@ -502,7 +500,7 @@ private void validateAndTransformTsv(File file, File out, Map<String, Pair<Strin
502500
getJob().getLogger().info("writing log of translation to : " + outLog.getPath());
503501
}
504502

505-
try (CSVReader reader = new CSVReader(IOUtil.openFileForBufferedReading(file), '\t', CSVWriter.NO_QUOTE_CHARACTER); CSVWriter writer = new CSVWriter(PrintWriters.getPrintWriter(out), '\t', CSVWriter.NO_QUOTE_CHARACTER, CSVWriter.NO_ESCAPE_CHARACTER);CSVWriter logWriter = new CSVWriter(PrintWriters.getPrintWriter(outLog), '\t', CSVWriter.NO_QUOTE_CHARACTER))
503+
try (CSVReader reader = new CSVReader(IOUtil.openFileForBufferedReading(file), '\t', CSVWriter.NO_QUOTE_CHARACTER); CSVWriter writer = new CSVWriter(IOUtil.openFileForBufferedUtf8Writing(out), '\t', CSVWriter.NO_QUOTE_CHARACTER, CSVWriter.NO_ESCAPE_CHARACTER);CSVWriter logWriter = new CSVWriter(PrintWriters.getPrintWriter(outLog), '\t', CSVWriter.NO_QUOTE_CHARACTER))
506504
{
507505
if (getPipelineJob().doChrTranslation())
508506
{

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceOutputHandlerRemoteTask.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ public static void possiblyCacheGenomes(SequenceJob job, List<SequenceOutputFile
112112
}
113113

114114
@NotNull
115+
@Override
115116
public RecordedActionSet run() throws PipelineJobException
116117
{
117118
SequenceTaskHelper.logModuleVersions(getJob().getLogger());
@@ -122,6 +123,7 @@ public RecordedActionSet run() throws PipelineJobException
122123
possiblyCacheGenomes(getPipelineJob(), getPipelineJob().getFiles());
123124

124125
getJob().setStatus(PipelineJob.TaskStatus.running, "Running: " + handler.getName());
126+
getJob().getLogger().info("Output file IDs: " + getPipelineJob().getFiles().stream().map(SequenceOutputFile::getRowid).map(String::valueOf).collect(Collectors.joining(",")));
125127
handler.getProcessor().processFilesRemote(getPipelineJob().getFiles(), ctx);
126128

127129
//Note: on job resume the TaskFileManager could be replaced with one from the resumer

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/BuildBamIndexWrapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public File executeCommand(File file) throws PipelineJobException
4646
return output;
4747
}
4848

49-
private List<String> getParams(File file)
49+
private List<String> getParams(File file) throws PipelineJobException
5050
{
5151
List<String> params = getBaseArgs();
5252

0 commit comments

Comments
 (0)