Skip to content

Commit d8a793c

Browse files
bbimberbbimber
authored
Merge pull request #333 from LabKey/fb_merge_24.11_to_develop
Merge discvr-24.11 to develop
2 parents 791f2b3 + b712d04 commit d8a793c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1236
-156
lines changed

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public static boolean copyIndexIfExists(PipelineContext ctx, AlignmentOutputImpl
4040

4141
public static boolean copyIndexIfExists(PipelineContext ctx, AlignmentOutputImpl output, String localName, String webserverName, ReferenceGenome genome, boolean forceCopyLocal) throws PipelineJobException
4242
{
43-
ctx.getLogger().debug("copying index to shared dir if exists: " + localName);
43+
ctx.getLogger().debug("checking if index exists: " + localName + ". copy local: " + forceCopyLocal);
4444
if (ctx.getWorkDir() == null)
4545
{
4646
throw new PipelineJobException("PipelineContext.getWorkDir() is null");

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PedigreeToolParameterDescriptor.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,25 @@ public class PedigreeToolParameterDescriptor extends ToolParameterDescriptor
66
{
77
public static String NAME = "pedigreeSource";
88

9+
private final boolean _isRequired;
10+
911
public PedigreeToolParameterDescriptor()
12+
{
13+
this(true);
14+
}
15+
16+
public PedigreeToolParameterDescriptor(final boolean isRequired)
1017
{
1118
super(null, NAME, "Pedigree Source", "This is the table used for pedigree data", "laboratory-pedigreeselectorfield", "laboratory.subjects", new JSONObject(){{
12-
put("allowBlank", false);
19+
put("allowBlank", !isRequired);
1320
}});
21+
22+
_isRequired = isRequired;
23+
}
24+
25+
public boolean isRequired()
26+
{
27+
return _isRequired;
1428
}
1529

1630
public static String getClientDependencyPath()

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenome.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public interface ReferenceGenome extends Serializable
7979

8080
/**
8181
* @param name The name used by the aligner to identify its cached directory
82-
* @return The folder expected containing the cached index, which is not guarenteed to exist. See AlignerIndexUtil for related methods.
82+
* @return The folder expected containing the cached index, which is not guaranteed to exist. See AlignerIndexUtil for related methods.
8383
*/
8484
File getAlignerIndexDir(String name);
8585

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,8 @@ static public void setInstance(SequencePipelineService instance)
100100
*/
101101
abstract public String getDockerCommand();
102102

103+
abstract public boolean useLocalDockerContainerStorage();
104+
103105
abstract public Collection<String> getDockerVolumes(Container c);
104106

105107
/**

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ToolParameterDescriptor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public class ToolParameterDescriptor
4141
private String _label;
4242
private String _description;
4343
private final String _fieldXtype;
44-
private final JSONObject _additionalExtConfig;
44+
protected JSONObject _additionalExtConfig;
4545
private final Object _defaultValue;
4646

4747
public ToolParameterDescriptor(CommandLineParam ca, String name, String label, String description, String fieldXtype, @Nullable Object defaultValue, @Nullable JSONObject additionalExtConfig)

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/VariantProcessingStep.java‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,21 +72,46 @@ interface Output extends PipelineStepOutput
7272
File getVCF();
7373
}
7474

75-
interface RequiresPedigree
75+
interface RequiresPedigree extends SupportsPedigree
76+
{
77+
@Override
78+
default boolean isRequired()
79+
{
80+
return true;
81+
}
82+
}
83+
84+
interface SupportsPedigree
7685
{
7786
default String getDemographicsProviderName(PipelineStepProvider<?> provider, PipelineJob job, int stepIdx)
7887
{
7988
return provider.getParameterByName(PedigreeToolParameterDescriptor.NAME).extractValue(job, provider, stepIdx, String.class);
8089
}
8190

82-
default DemographicsProvider getDemographicsProvider(PipelineStepProvider<?> provider, PipelineJob job, int stepIdx)
91+
default @Nullable DemographicsProvider getDemographicsProvider(PipelineStepProvider<?> provider, PipelineJob job, int stepIdx)
8392
{
8493
if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer)
8594
{
8695
throw new IllegalStateException("getDemographicsProvider() can only be run from the webserver");
8796
}
8897

89-
return LaboratoryService.get().getDemographicsProviderByName(job.getContainer(), job.getUser(), getDemographicsProviderName(provider, job, stepIdx));
98+
String dpn = getDemographicsProviderName(provider, job, stepIdx);
99+
if (dpn == null)
100+
{
101+
if (isRequired())
102+
{
103+
throw new IllegalArgumentException("The DemographicsProvider name cannot be null");
104+
}
105+
106+
return null;
107+
}
108+
109+
return LaboratoryService.get().getDemographicsProviderByName(job.getContainer(), job.getUser(), dpn);
110+
}
111+
112+
default boolean isRequired()
113+
{
114+
return false;
90115
}
91116
}
92117

‎SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/DockerWrapper.java‎

Lines changed: 86 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
package org.labkey.api.sequenceanalysis.run;
22

3+
import org.apache.commons.io.FileUtils;
34
import org.apache.commons.lang3.StringUtils;
45
import org.apache.logging.log4j.Logger;
56
import org.jetbrains.annotations.Nullable;
67
import org.labkey.api.pipeline.PipelineJobException;
78
import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
89
import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker;
910
import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
11+
import org.labkey.api.util.FileUtil;
1012
import org.labkey.api.writer.PrintWriters;
1113

1214
import java.io.File;
@@ -28,7 +30,8 @@ public class DockerWrapper extends AbstractCommandWrapper
2830
private final PipelineContext _ctx;
2931
private File _tmpDir = null;
3032
private String _entryPoint = null;
31-
private boolean _runPrune = true;
33+
private boolean _runPrune = false;
34+
private boolean _useLocalContainerStorage;
3235
private String _alternateUserHome = null;
3336
private final Map<String, String> _dockerEnvironment = new HashMap<>();
3437

@@ -37,6 +40,8 @@ public DockerWrapper(String containerName, Logger log, PipelineContext ctx)
3740
super(log);
3841
_containerName = containerName;
3942
_ctx = ctx;
43+
44+
_useLocalContainerStorage = SequencePipelineService.get().useLocalDockerContainerStorage();
4045
}
4146

4247
public void setAlternateUserHome(String alternateUserHome)
@@ -59,6 +64,11 @@ public void setRunPrune(boolean runPrune)
5964
_runPrune = runPrune;
6065
}
6166

67+
public void setUseLocalContainerStorage(boolean useLocalContainerStorage)
68+
{
69+
_useLocalContainerStorage = useLocalContainerStorage;
70+
}
71+
6272
public void executeWithDocker(List<String> containerArgs, File workDir, PipelineOutputTracker tracker) throws PipelineJobException
6373
{
6474
executeWithDocker(containerArgs, workDir, tracker, null);
@@ -79,14 +89,34 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
7989
writer.println("set -e");
8090

8191
writer.println("DOCKER='" + SequencePipelineService.get().getDockerCommand() + "'");
82-
writer.println("$DOCKER pull " + _containerName);
92+
93+
writer.println("IMAGE_EXISTS=`$DOCKER images -q \"" + getEffectiveContainerName() + "\" | wc -l`");
94+
writer.println("LOCAL=not_present");
95+
writer.println("if [[ $IMAGE_EXISTS > 0 ]];then");
96+
writer.println("\tLOCAL=`docker inspect --format='{{.Digest}}' " + getEffectiveContainerName() + "`");
97+
writer.println("fi");
98+
writer.println("LATEST=`regctl image digest --list " + getEffectiveContainerName() + "`");
99+
writer.println("if [ $LOCAL != $LATEST ];then");
100+
writer.println("\t$DOCKER pull " + getLocalStorageArgs() + getEffectiveContainerName());
101+
writer.println("else");
102+
writer.println("\techo 'Image up to date'");
103+
writer.println("fi");
104+
83105
if (_runPrune)
84106
{
85-
writer.println("$DOCKER image prune -f");
107+
writer.println("$DOCKER image prune " + getLocalStorageArgs() + "-f");
86108
}
87109

88110
writer.println("$DOCKER run --rm=true \\");
89111
writer.println("\t--group-add keep-groups \\");
112+
writer.println("\t--transient-store \\");
113+
114+
if (_useLocalContainerStorage)
115+
{
116+
getLogger().debug("Using local container storage: " + getLocalContainerDir().getPath());
117+
prepareLocalStorage();
118+
writer.println("\t" + getLocalStorageArgs() + "\\");
119+
}
90120

91121
// NOTE: getDockerVolumes() should be refactored to remove the -v and this logic should be updated accordingly:
92122
File homeDir = new File(System.getProperty("user.home"));
@@ -149,7 +179,7 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
149179
{
150180
writer.println("\t-e " + key + "='" + _dockerEnvironment.get(key) + "' \\");
151181
}
152-
writer.println("\t" + _containerName + " \\");
182+
writer.println("\t" + getEffectiveContainerName() + " \\");
153183
writer.println("\t" + dockerBashScript.getPath());
154184
writer.println("DOCKER_EXIT_CODE=$?");
155185
writer.println("echo 'Docker run exit code: '$DOCKER_EXIT_CODE");
@@ -170,6 +200,23 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
170200
localBashScript.setExecutable(true);
171201
dockerBashScript.setExecutable(true);
172202
execute(Arrays.asList("/bin/bash", localBashScript.getPath()));
203+
204+
if (_useLocalContainerStorage)
205+
{
206+
try
207+
{
208+
FileUtils.deleteDirectory(getLocalContainerDir());
209+
}
210+
catch (IOException e)
211+
{
212+
throw new PipelineJobException(e);
213+
}
214+
}
215+
}
216+
217+
private String getEffectiveContainerName()
218+
{
219+
return _containerName;
173220
}
174221

175222
public void addToDockerEnvironment(String key, String value)
@@ -203,4 +250,39 @@ private Collection<File> inspectInputFiles(Collection<File> inputFiles)
203250

204251
return Collections.emptySet();
205252
}
253+
254+
private File getLocalContainerDir()
255+
{
256+
return new File(SequencePipelineService.get().getJavaTempDir(), "containers");
257+
}
258+
259+
private File prepareLocalStorage() throws PipelineJobException
260+
{
261+
try
262+
{
263+
if (getLocalContainerDir().exists())
264+
{
265+
getLogger().debug("Deleting existing container dir: " + getLocalContainerDir());
266+
FileUtils.deleteDirectory(getLocalContainerDir());
267+
}
268+
269+
FileUtil.createDirectory(getLocalContainerDir().toPath());
270+
271+
return getLocalContainerDir();
272+
}
273+
catch (IOException e)
274+
{
275+
throw new PipelineJobException(e);
276+
}
277+
}
278+
279+
private String getLocalStorageArgs()
280+
{
281+
if (!_useLocalContainerStorage)
282+
{
283+
return "";
284+
}
285+
286+
return "--root=" + getLocalContainerDir().getPath() + " ";
287+
}
206288
}

‎SequenceAnalysis/pipeline_code/extra_tools_install.sh‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,4 +228,80 @@ then
228228
python3 -m pip install --user multiqc
229229
else
230230
echo "Already installed"
231-
fi
231+
fi
232+
233+
234+
if [[ ! -e ${LKTOOLS_DIR}/gt || ! -z $FORCE_REINSTALL ]];
235+
then
236+
echo "Cleaning up previous installs"
237+
rm -Rf gt*
238+
rm -Rf $LKTOOLS_DIR/gt*
239+
240+
wget https://github.com/genometools/genometools/releases/download/v1.6.5/gt-1.6.5-Linux_x86_64-64bit-complete.tar.gz
241+
tar -xf gt-1.6.5-Linux_x86_64-64bit-complete.tar.gz
242+
243+
install ./gt-1.6.5-Linux_x86_64-64bit-complete/bin/gt $LKTOOLS_DIR/
244+
mv ./gt-1.6.5-Linux_x86_64-64bit-complete/gtdata $LKTOOLS_DIR/
245+
else
246+
echo "Already installed"
247+
fi
248+
249+
if [[ ! -e ${LKTOOLS_DIR}/king || ! -z $FORCE_REINSTALL ]];
250+
then
251+
echo "Cleaning up previous installs"
252+
rm -Rf king*
253+
rm -Rf Linux-king*
254+
rm -Rf $LKTOOLS_DIR/king*
255+
256+
wget https://www.kingrelatedness.com/Linux-king.tar.gz
257+
tar -xf Linux-king.tar.gz
258+
259+
install king $LKTOOLS_DIR/
260+
else
261+
echo "Already installed"
262+
fi
263+
264+
if [[ ! -e ${LKTOOLS_DIR}/regctl || ! -z $FORCE_REINSTALL ]];
265+
then
266+
echo "Cleaning up previous installs"
267+
rm -Rf regctl*
268+
rm -Rf $LKTOOLS_DIR/regctl*
269+
270+
curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 > regctl
271+
chmod 755 regctl
272+
273+
install regctl $LKTOOLS_DIR/
274+
else
275+
echo "Already installed"
276+
fi
277+
278+
if [[ ! -e ${LKTOOLS_DIR}/svtyper || ! -z $FORCE_REINSTALL ]];
279+
then
280+
echo "Cleaning up previous installs"
281+
rm -Rf $LKTOOLS_DIR/svtyper*
282+
283+
# NOTE: this fork is used to ensure python3 compatibility
284+
#python3 -m pip install --user git+https://github.com/hall-lab/svtyper.git
285+
python3 -m pip install --user git+https://github.com/bbimber/svtyper.git
286+
287+
SVTYPER=`which svtyper`
288+
ln -s $SVTYPER ${LKTOOLS_DIR}/svtyper
289+
290+
SVTYPER=`which svtyper-sso`
291+
ln -s $SVTYPER ${LKTOOLS_DIR}/svtyper-sso
292+
else
293+
echo "Already installed"
294+
fi
295+
296+
if [[ ! -e ${LKTOOLS_DIR}/graphtyper || ! -z $FORCE_REINSTALL ]];
297+
then
298+
echo "Cleaning up previous installs"
299+
rm -Rf $LKTOOLS_DIR/graphtyper*
300+
301+
wget https://github.com/DecodeGenetics/graphtyper/releases/download/v2.7.7/graphtyper
302+
chmod a+x graphtyper
303+
304+
mv ./graphtyper $LKTOOLS_DIR/
305+
else
306+
echo "Already installed"
307+
fi

‎SequenceAnalysis/pipeline_code/sequence_tools_install.sh‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -985,15 +985,15 @@ then
985985
rm -Rf $LKTOOLS_DIR/blast_formatter
986986
rm -Rf $LKTOOLS_DIR/makeblastdb
987987

988-
wget $WGET_OPTS ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.2.31/ncbi-blast-2.2.31+-x64-linux.tar.gz
989-
gunzip ncbi-blast-2.2.31+-x64-linux.tar.gz
990-
tar -xf ncbi-blast-2.2.31+-x64-linux.tar
991-
gzip ncbi-blast-2.2.31+-x64-linux.tar
992-
993-
install ./ncbi-blast-2.2.31+/bin/blastn $LKTOOLS_DIR/blastn
994-
install ./ncbi-blast-2.2.31+/bin/blast_formatter $LKTOOLS_DIR/blast_formatter
995-
install ./ncbi-blast-2.2.31+/bin/makeblastdb $LKTOOLS_DIR/makeblastdb
996-
install ./ncbi-blast-2.2.31+/bin/makembindex $LKTOOLS_DIR/makembindex
988+
wget $WGET_OPTS ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.16.0/ncbi-blast-2.16.0+-x64-linux.tar.gz
989+
gunzip ncbi-blast-2.16.0+-x64-linux.tar.gz
990+
tar -xf ncbi-blast-2.16.0+-x64-linux.tar
991+
gzip ncbi-blast-2.16.0+-x64-linux.tar
992+
993+
install ./ncbi-blast-2.16.0+/bin/blastn $LKTOOLS_DIR/blastn
994+
install ./ncbi-blast-2.16.0+/bin/blast_formatter $LKTOOLS_DIR/blast_formatter
995+
install ./ncbi-blast-2.16.0+/bin/makeblastdb $LKTOOLS_DIR/makeblastdb
996+
install ./ncbi-blast-2.16.0+/bin/makembindex $LKTOOLS_DIR/makembindex
997997
else
998998
echo "Already installed"
999999
fi

0 commit comments

Comments
 (0)