Merge discvr-24.11 to develop

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java

@@ -40,7 +40,7 @@ public static boolean copyIndexIfExists(PipelineContext ctx, AlignmentOutputImpl
 
     public static boolean copyIndexIfExists(PipelineContext ctx, AlignmentOutputImpl output, String localName, String webserverName, ReferenceGenome genome, boolean forceCopyLocal) throws PipelineJobException
     {
-        ctx.getLogger().debug("copying index to shared dir if exists: " + localName);
+        ctx.getLogger().debug("checking if index exists: " + localName + ". copy local: " + forceCopyLocal);
         if (ctx.getWorkDir() == null)
         {
             throw new PipelineJobException("PipelineContext.getWorkDir() is null");

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PedigreeToolParameterDescriptor.java

@@ -6,11 +6,25 @@ public class PedigreeToolParameterDescriptor extends ToolParameterDescriptor
 {
     public static String NAME = "pedigreeSource";
 
+    private final boolean _isRequired;
+
     public PedigreeToolParameterDescriptor()
+    {
+        this(true);
+    }
+
+    public PedigreeToolParameterDescriptor(final boolean isRequired)
     {
         super(null, NAME, "Pedigree Source", "This is the table used for pedigree data", "laboratory-pedigreeselectorfield", "laboratory.subjects", new JSONObject(){{
-            put("allowBlank", false);
+            put("allowBlank", !isRequired);
         }});
+
+        _isRequired = isRequired;
+    }
+
+    public boolean isRequired()
+    {
+        return _isRequired;
     }
 
     public static String getClientDependencyPath()

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ReferenceGenome.java

@@ -79,7 +79,7 @@ public interface ReferenceGenome extends Serializable
 
     /**
      * @param name The name used by the aligner to identify its cached directory
-     * @return The folder expected containing the cached index, which is not guarenteed to exist.  See AlignerIndexUtil for related methods.
+     * @return The folder expected containing the cached index, which is not guaranteed to exist.  See AlignerIndexUtil for related methods.
      */
     File getAlignerIndexDir(String name);
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java

@@ -100,6 +100,8 @@ static public void setInstance(SequencePipelineService instance)
      */
     abstract public String getDockerCommand();
 
+    abstract public boolean useLocalDockerContainerStorage();
+
     abstract public Collection<String> getDockerVolumes(Container c);
 
     /**

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/ToolParameterDescriptor.java

@@ -41,7 +41,7 @@ public class ToolParameterDescriptor
     private String _label;
     private String _description;
     private final String _fieldXtype;
-    private final JSONObject _additionalExtConfig;
+    protected JSONObject _additionalExtConfig;
     private final Object _defaultValue;
 
     public ToolParameterDescriptor(CommandLineParam ca, String name, String label, String description, String fieldXtype, @Nullable Object defaultValue, @Nullable JSONObject additionalExtConfig)

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/VariantProcessingStep.java

@@ -72,21 +72,46 @@ interface Output extends PipelineStepOutput
         File getVCF();
     }
 
-    interface RequiresPedigree
+    interface RequiresPedigree extends SupportsPedigree
+    {
+        @Override
+        default boolean isRequired()
+        {
+            return true;
+        }
+    }
+
+    interface SupportsPedigree
     {
         default String getDemographicsProviderName(PipelineStepProvider<?> provider, PipelineJob job, int stepIdx)
         {
             return provider.getParameterByName(PedigreeToolParameterDescriptor.NAME).extractValue(job, provider, stepIdx, String.class);
         }
 
-        default DemographicsProvider getDemographicsProvider(PipelineStepProvider<?> provider, PipelineJob job, int stepIdx)
+        default @Nullable DemographicsProvider getDemographicsProvider(PipelineStepProvider<?> provider, PipelineJob job, int stepIdx)
         {
             if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer)
             {
                 throw new IllegalStateException("getDemographicsProvider() can only be run from the webserver");
             }
 
-            return LaboratoryService.get().getDemographicsProviderByName(job.getContainer(), job.getUser(), getDemographicsProviderName(provider, job, stepIdx));
+            String dpn = getDemographicsProviderName(provider, job, stepIdx);
+            if (dpn == null)
+            {
+                if (isRequired())
+                {
+                    throw new IllegalArgumentException("The DemographicsProvider name cannot be null");
+                }
+
+                return null;
+            }
+
+            return LaboratoryService.get().getDemographicsProviderByName(job.getContainer(), job.getUser(), dpn);
+        }
+
+        default boolean isRequired()
+        {
+            return false;
         }
     }
 

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/DockerWrapper.java

@@ -1,12 +1,14 @@
 package org.labkey.api.sequenceanalysis.run;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
 import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
+import org.labkey.api.util.FileUtil;
 import org.labkey.api.writer.PrintWriters;
 
 import java.io.File;
@@ -28,7 +30,8 @@ public class DockerWrapper extends AbstractCommandWrapper
     private final PipelineContext _ctx;
     private File _tmpDir = null;
     private String _entryPoint = null;
-    private boolean _runPrune = true;
+    private boolean _runPrune = false;
+    private boolean _useLocalContainerStorage;
     private String _alternateUserHome = null;
     private final Map<String, String> _dockerEnvironment = new HashMap<>();
 
@@ -37,6 +40,8 @@ public DockerWrapper(String containerName, Logger log, PipelineContext ctx)
         super(log);
         _containerName = containerName;
         _ctx = ctx;
+
+        _useLocalContainerStorage = SequencePipelineService.get().useLocalDockerContainerStorage();
     }
 
     public void setAlternateUserHome(String alternateUserHome)
@@ -59,6 +64,11 @@ public void setRunPrune(boolean runPrune)
         _runPrune = runPrune;
     }
 
+    public void setUseLocalContainerStorage(boolean useLocalContainerStorage)
+    {
+        _useLocalContainerStorage = useLocalContainerStorage;
+    }
+
     public void executeWithDocker(List<String> containerArgs, File workDir, PipelineOutputTracker tracker) throws PipelineJobException
     {
         executeWithDocker(containerArgs, workDir, tracker, null);
@@ -79,14 +89,34 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
             writer.println("set -e");
 
             writer.println("DOCKER='" + SequencePipelineService.get().getDockerCommand() + "'");
-            writer.println("$DOCKER pull " + _containerName);
+
+            writer.println("IMAGE_EXISTS=`$DOCKER images -q \"" + getEffectiveContainerName() + "\" | wc -l`");
+            writer.println("LOCAL=not_present");
+            writer.println("if [[ $IMAGE_EXISTS > 0 ]];then");
+            writer.println("\tLOCAL=`docker inspect --format='{{.Digest}}' " + getEffectiveContainerName() + "`");
+            writer.println("fi");
+            writer.println("LATEST=`regctl image digest --list " + getEffectiveContainerName() + "`");
+            writer.println("if [ $LOCAL != $LATEST ];then");
+            writer.println("\t$DOCKER pull " + getLocalStorageArgs() + getEffectiveContainerName());
+            writer.println("else");
+            writer.println("\techo 'Image up to date'");
+            writer.println("fi");
+
             if (_runPrune)
             {
-                writer.println("$DOCKER image prune -f");
+                writer.println("$DOCKER image prune " + getLocalStorageArgs() + "-f");
             }
 
             writer.println("$DOCKER run --rm=true \\");
             writer.println("\t--group-add keep-groups \\");
+            writer.println("\t--transient-store \\");
+
+            if (_useLocalContainerStorage)
+            {
+                getLogger().debug("Using local container storage: " + getLocalContainerDir().getPath());
+                prepareLocalStorage();
+                writer.println("\t" + getLocalStorageArgs() + "\\");
+            }
 
             // NOTE: getDockerVolumes() should be refactored to remove the -v and this logic should be updated accordingly:
             File homeDir = new File(System.getProperty("user.home"));
@@ -149,7 +179,7 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
             {
                 writer.println("\t-e " + key + "='" + _dockerEnvironment.get(key) + "' \\");
             }
-            writer.println("\t" + _containerName + " \\");
+            writer.println("\t" + getEffectiveContainerName() + " \\");
             writer.println("\t" + dockerBashScript.getPath());
             writer.println("DOCKER_EXIT_CODE=$?");
             writer.println("echo 'Docker run exit code: '$DOCKER_EXIT_CODE");
@@ -170,6 +200,23 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
         localBashScript.setExecutable(true);
         dockerBashScript.setExecutable(true);
         execute(Arrays.asList("/bin/bash", localBashScript.getPath()));
+
+        if (_useLocalContainerStorage)
+        {
+            try
+            {
+                FileUtils.deleteDirectory(getLocalContainerDir());
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException(e);
+            }
+        }
+    }
+
+    private String getEffectiveContainerName()
+    {
+        return _containerName;
     }
 
     public void addToDockerEnvironment(String key, String value)
@@ -203,4 +250,39 @@ private Collection<File> inspectInputFiles(Collection<File> inputFiles)
 
         return Collections.emptySet();
     }
+
+    private File getLocalContainerDir()
+    {
+        return new File(SequencePipelineService.get().getJavaTempDir(), "containers");
+    }
+
+    private File prepareLocalStorage() throws PipelineJobException
+    {
+        try
+        {
+            if (getLocalContainerDir().exists())
+            {
+                getLogger().debug("Deleting existing container dir: " + getLocalContainerDir());
+                FileUtils.deleteDirectory(getLocalContainerDir());
+            }
+
+            FileUtil.createDirectory(getLocalContainerDir().toPath());
+
+            return getLocalContainerDir();
+        }
+        catch (IOException e)
+        {
+            throw new PipelineJobException(e);
+        }
+    }
+
+    private String getLocalStorageArgs()
+    {
+        if (!_useLocalContainerStorage)
+        {
+            return "";
+        }
+
+        return "--root=" + getLocalContainerDir().getPath() + " ";
+    }
 }

SequenceAnalysis/pipeline_code/extra_tools_install.sh

@@ -228,4 +228,80 @@ then
     python3 -m pip install --user multiqc
 else
     echo "Already installed"
-fi
+fi
+
+
+if [[ ! -e ${LKTOOLS_DIR}/gt || ! -z $FORCE_REINSTALL ]];
+then
+    echo "Cleaning up previous installs"
+    rm -Rf gt*
+    rm -Rf $LKTOOLS_DIR/gt*
+
+    wget https://github.com/genometools/genometools/releases/download/v1.6.5/gt-1.6.5-Linux_x86_64-64bit-complete.tar.gz
+    tar -xf gt-1.6.5-Linux_x86_64-64bit-complete.tar.gz
+
+    install ./gt-1.6.5-Linux_x86_64-64bit-complete/bin/gt $LKTOOLS_DIR/
+    mv ./gt-1.6.5-Linux_x86_64-64bit-complete/gtdata $LKTOOLS_DIR/
+else
+    echo "Already installed"
+fi
+
+if [[ ! -e ${LKTOOLS_DIR}/king || ! -z $FORCE_REINSTALL ]];
+then
+    echo "Cleaning up previous installs"
+    rm -Rf king*
+    rm -Rf Linux-king*
+    rm -Rf $LKTOOLS_DIR/king*
+
+    wget https://www.kingrelatedness.com/Linux-king.tar.gz
+    tar -xf Linux-king.tar.gz
+
+    install king $LKTOOLS_DIR/
+else
+    echo "Already installed"
+fi
+
+if [[ ! -e ${LKTOOLS_DIR}/regctl || ! -z $FORCE_REINSTALL ]];
+then
+    echo "Cleaning up previous installs"
+    rm -Rf regctl*
+    rm -Rf $LKTOOLS_DIR/regctl*
+
+    curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 > regctl
+    chmod 755 regctl
+
+    install regctl $LKTOOLS_DIR/
+else
+    echo "Already installed"
+fi
+
+if [[ ! -e ${LKTOOLS_DIR}/svtyper || ! -z $FORCE_REINSTALL ]];
+then
+    echo "Cleaning up previous installs"
+    rm -Rf $LKTOOLS_DIR/svtyper*
+
+    # NOTE: this fork is used to ensure python3 compatibility
+    #python3 -m pip install --user git+https://github.com/hall-lab/svtyper.git
+    python3 -m pip install --user git+https://github.com/bbimber/svtyper.git
+
+    SVTYPER=`which svtyper`
+    ln -s $SVTYPER ${LKTOOLS_DIR}/svtyper
+
+    SVTYPER=`which svtyper-sso`
+    ln -s $SVTYPER ${LKTOOLS_DIR}/svtyper-sso
+else
+    echo "Already installed"
+fi
+
+if [[ ! -e ${LKTOOLS_DIR}/graphtyper || ! -z $FORCE_REINSTALL ]];
+then
+    echo "Cleaning up previous installs"
+    rm -Rf $LKTOOLS_DIR/graphtyper*
+
+    wget https://github.com/DecodeGenetics/graphtyper/releases/download/v2.7.7/graphtyper
+    chmod a+x graphtyper
+
+    mv ./graphtyper $LKTOOLS_DIR/
+else
+    echo "Already installed"
+fi

SequenceAnalysis/pipeline_code/sequence_tools_install.sh

@@ -985,15 +985,15 @@ then
     rm -Rf $LKTOOLS_DIR/blast_formatter
     rm -Rf $LKTOOLS_DIR/makeblastdb
 
-    wget $WGET_OPTS ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.2.31/ncbi-blast-2.2.31+-x64-linux.tar.gz
-    gunzip ncbi-blast-2.2.31+-x64-linux.tar.gz
-    tar -xf ncbi-blast-2.2.31+-x64-linux.tar
-    gzip ncbi-blast-2.2.31+-x64-linux.tar
-
-    install ./ncbi-blast-2.2.31+/bin/blastn $LKTOOLS_DIR/blastn
-    install ./ncbi-blast-2.2.31+/bin/blast_formatter $LKTOOLS_DIR/blast_formatter
-    install ./ncbi-blast-2.2.31+/bin/makeblastdb $LKTOOLS_DIR/makeblastdb
-    install ./ncbi-blast-2.2.31+/bin/makembindex $LKTOOLS_DIR/makembindex
+    wget $WGET_OPTS ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.16.0/ncbi-blast-2.16.0+-x64-linux.tar.gz
+    gunzip ncbi-blast-2.16.0+-x64-linux.tar.gz
+    tar -xf ncbi-blast-2.16.0+-x64-linux.tar
+    gzip ncbi-blast-2.16.0+-x64-linux.tar
+
+    install ./ncbi-blast-2.16.0+/bin/blastn $LKTOOLS_DIR/blastn
+    install ./ncbi-blast-2.16.0+/bin/blast_formatter $LKTOOLS_DIR/blast_formatter
+    install ./ncbi-blast-2.16.0+/bin/makeblastdb $LKTOOLS_DIR/makeblastdb
+    install ./ncbi-blast-2.16.0+/bin/makembindex $LKTOOLS_DIR/makembindex
 else
     echo "Already installed"
 fi