Switch docker to conditionally mount volumes based on provider

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/JobResourceSettings.java

@@ -2,6 +2,7 @@
 
 import org.labkey.api.data.Container;
 
+import java.util.Collection;
 import java.util.List;
 
 /**
@@ -12,4 +13,6 @@ public interface JobResourceSettings
     boolean isAvailable(Container c);
 
     List<ToolParameterDescriptor> getParams();
+
+    Collection<String> getDockerVolumes(Container c);
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/PipelineContext.java

@@ -16,11 +16,11 @@
 package org.labkey.api.sequenceanalysis.pipeline;
 
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.WorkDirectory;
 
 import java.io.File;
+import java.util.Collection;
 
 /**
  * User: bimber
@@ -51,4 +51,6 @@ public interface PipelineContext
      * This is the directory where the source files were located.  In the situation where this is a split job, forceParent=true will return the parent job's sourceDirectory.  This can be important if files are written here prior to split.
      */
     File getSourceDirectory(boolean forceParent);
+
+    Collection<String> getDockerVolumes();
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SequencePipelineService.java

@@ -19,13 +19,15 @@
 import htsjdk.samtools.SAMFileHeader;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.data.Container;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 import org.labkey.api.sequenceanalysis.run.CommandWrapper;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -98,6 +100,8 @@ static public void setInstance(SequencePipelineService instance)
      */
     abstract public String getDockerCommand();
 
+    abstract public Collection<String> getDockerVolumes(Container c);
+
     abstract public List<File> getSequenceJobInputFiles(PipelineJob job);
 
     /**

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/DockerWrapper.java

@@ -4,6 +4,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
 import org.labkey.api.sequenceanalysis.pipeline.PipelineOutputTracker;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.writer.PrintWriters;
@@ -17,12 +18,14 @@
 public class DockerWrapper extends AbstractCommandWrapper
 {
     private final String _containerName;
+    private final PipelineContext _ctx;
     private File _tmpDir = null;
 
-    public DockerWrapper(String containerName, Logger log)
+    public DockerWrapper(String containerName, Logger log, PipelineContext ctx)
     {
         super(log);
         _containerName = containerName;
+        _ctx = ctx;
     }
 
     public void setTmpDir(File tmpDir)
@@ -49,6 +52,7 @@ public void executeWithDocker(List<String> containerArgs, File workDir, Pipeline
             writer.println("sudo $DOCKER run --rm=true \\");
             writer.println("\t-v \"${WD}:/work\" \\");
             writer.println("\t-v \"${HOME}:/homeDir\" \\");
+            _ctx.getDockerVolumes().forEach(writer::println);
             if (_tmpDir != null)
             {
                 writer.println("\t-v \"" + _tmpDir.getPath() + ":/tmp\" \\");

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java

@@ -67,10 +67,31 @@
 import org.labkey.sequenceanalysis.button.DownloadSraButton;
 import org.labkey.sequenceanalysis.button.ReprocessLibraryButton;
 import org.labkey.sequenceanalysis.button.RunMultiQCButton;
-import org.labkey.sequenceanalysis.pipeline.*;
+import org.labkey.sequenceanalysis.pipeline.AlignmentAnalysisJob;
+import org.labkey.sequenceanalysis.pipeline.AlignmentImportJob;
+import org.labkey.sequenceanalysis.pipeline.CacheGenomePipelineJob;
+import org.labkey.sequenceanalysis.pipeline.CacheGenomeTrigger;
+import org.labkey.sequenceanalysis.pipeline.ConvertToCramHandler;
+import org.labkey.sequenceanalysis.pipeline.IlluminaImportJob;
+import org.labkey.sequenceanalysis.pipeline.ImportFastaSequencesPipelineJob;
+import org.labkey.sequenceanalysis.pipeline.ImportGenomeTrackPipelineJob;
+import org.labkey.sequenceanalysis.pipeline.OrphanFilePipelineProvider;
+import org.labkey.sequenceanalysis.pipeline.ProcessVariantsHandler;
+import org.labkey.sequenceanalysis.pipeline.ReadsetImportJob;
+import org.labkey.sequenceanalysis.pipeline.ReblockGvcfHandler;
+import org.labkey.sequenceanalysis.pipeline.ReferenceLibraryPipelineProvider;
+import org.labkey.sequenceanalysis.pipeline.SequenceAlignmentJob;
+import org.labkey.sequenceanalysis.pipeline.SequenceAlignmentTask;
+import org.labkey.sequenceanalysis.pipeline.SequenceJob;
+import org.labkey.sequenceanalysis.pipeline.SequenceJobSupportImpl;
+import org.labkey.sequenceanalysis.pipeline.SequenceOutputHandlerPipelineProvider;
+import org.labkey.sequenceanalysis.pipeline.SequencePipelineProvider;
+import org.labkey.sequenceanalysis.pipeline.SequenceReadsetHandlerPipelineProvider;
+import org.labkey.sequenceanalysis.pipeline.VariantProcessingJob;
 import org.labkey.sequenceanalysis.query.SequenceAnalysisUserSchema;
 import org.labkey.sequenceanalysis.query.SequenceTriggerHelper;
 import org.labkey.sequenceanalysis.run.RestoreSraDataHandler;
+import org.labkey.sequenceanalysis.run.alignment.BWAMem2Wrapper;
 import org.labkey.sequenceanalysis.run.alignment.BWAMemWrapper;
 import org.labkey.sequenceanalysis.run.alignment.BWASWWrapper;
 import org.labkey.sequenceanalysis.run.alignment.BWAWrapper;
@@ -82,7 +103,27 @@
 import org.labkey.sequenceanalysis.run.alignment.Pbmm2Wrapper;
 import org.labkey.sequenceanalysis.run.alignment.StarWrapper;
 import org.labkey.sequenceanalysis.run.alignment.VulcanWrapper;
-import org.labkey.sequenceanalysis.run.analysis.*;
+import org.labkey.sequenceanalysis.run.analysis.BamIterator;
+import org.labkey.sequenceanalysis.run.analysis.BcftoolsFillTagsStep;
+import org.labkey.sequenceanalysis.run.analysis.BcftoolsFixploidyStep;
+import org.labkey.sequenceanalysis.run.analysis.DeepVariantAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.ExportOverlappingReadsAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.GenrichStep;
+import org.labkey.sequenceanalysis.run.analysis.HaplotypeCallerAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.ImmunoGenotypingAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.LofreqAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.MergeLoFreqVcfHandler;
+import org.labkey.sequenceanalysis.run.analysis.NextCladeHandler;
+import org.labkey.sequenceanalysis.run.analysis.PARalyzerAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.PangolinHandler;
+import org.labkey.sequenceanalysis.run.analysis.PbsvAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.PbsvJointCallingHandler;
+import org.labkey.sequenceanalysis.run.analysis.PindelAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.SequenceBasedTypingAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.SnpCountAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.SubreadAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.UnmappedReadExportHandler;
+import org.labkey.sequenceanalysis.run.analysis.ViralAnalysis;
 import org.labkey.sequenceanalysis.run.assembly.TrinityRunner;
 import org.labkey.sequenceanalysis.run.bampostprocessing.AddOrReplaceReadGroupsStep;
 import org.labkey.sequenceanalysis.run.bampostprocessing.BaseQualityScoreRecalibrator;
@@ -116,7 +157,27 @@
 import org.labkey.sequenceanalysis.run.util.GenomicsDBAppendHandler;
 import org.labkey.sequenceanalysis.run.util.GenomicsDBImportHandler;
 import org.labkey.sequenceanalysis.run.util.SVAnnotateStep;
-import org.labkey.sequenceanalysis.run.variant.*;
+import org.labkey.sequenceanalysis.run.variant.DepthOfCoverageHandler;
+import org.labkey.sequenceanalysis.run.variant.GenotypeConcordanceStep;
+import org.labkey.sequenceanalysis.run.variant.GenotypeFiltrationStep;
+import org.labkey.sequenceanalysis.run.variant.KingInferenceStep;
+import org.labkey.sequenceanalysis.run.variant.MendelianViolationReportStep;
+import org.labkey.sequenceanalysis.run.variant.MergeVcfsAndGenotypesHandler;
+import org.labkey.sequenceanalysis.run.variant.MultiAllelicPositionsHandler;
+import org.labkey.sequenceanalysis.run.variant.PlinkPcaStep;
+import org.labkey.sequenceanalysis.run.variant.SNPEffStep;
+import org.labkey.sequenceanalysis.run.variant.SampleRenameStep;
+import org.labkey.sequenceanalysis.run.variant.SelectSNVsStep;
+import org.labkey.sequenceanalysis.run.variant.SelectSamplesStep;
+import org.labkey.sequenceanalysis.run.variant.SelectVariantsStep;
+import org.labkey.sequenceanalysis.run.variant.SplitVcfBySamplesStep;
+import org.labkey.sequenceanalysis.run.variant.SummarizeGenotypeQualityStep;
+import org.labkey.sequenceanalysis.run.variant.VariantAnnotatorStep;
+import org.labkey.sequenceanalysis.run.variant.VariantEvalBySampleStep;
+import org.labkey.sequenceanalysis.run.variant.VariantEvalStep;
+import org.labkey.sequenceanalysis.run.variant.VariantFiltrationStep;
+import org.labkey.sequenceanalysis.run.variant.VariantQCStep;
+import org.labkey.sequenceanalysis.run.variant.VariantsToTableStep;
 import org.labkey.sequenceanalysis.util.Barcoder;
 import org.labkey.sequenceanalysis.util.ChainFileValidator;
 import org.labkey.sequenceanalysis.util.ScatterGatherUtils;
@@ -237,6 +298,7 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new BowtieWrapper.Provider());
         SequencePipelineService.get().registerPipelineStep(new Bowtie2Wrapper.Provider());
         SequencePipelineService.get().registerPipelineStep(new BWAMemWrapper.Provider());
+        SequencePipelineService.get().registerPipelineStep(new BWAMem2Wrapper.Provider());
         SequencePipelineService.get().registerPipelineStep(new BWAWrapper.Provider());
         SequencePipelineService.get().registerPipelineStep(new BWASWWrapper.Provider());
         SequencePipelineService.get().registerPipelineStep(new MosaikWrapper.Provider());

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequencePipelineServiceImpl.java

@@ -8,6 +8,7 @@
 import org.jetbrains.annotations.Nullable;
 import org.junit.Assert;
 import org.junit.Test;
+import org.labkey.api.data.Container;
 import org.labkey.api.pipeline.PipelineJob;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.PipelineJobService;
@@ -43,6 +44,7 @@
 import java.lang.reflect.ParameterizedType;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -457,6 +459,29 @@ public String getDockerCommand()
         return "docker";
     }
 
+    @Override
+    public Collection<String> getDockerVolumes(Container c)
+    {
+        if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer)
+        {
+            throw new IllegalArgumentException("SequencePipelineService.getDockerVolumes() should only be called from the webserver!");
+        }
+
+        Set<String> volumeLines = new HashSet<>();
+        for (JobResourceSettings settings : SequencePipelineServiceImpl.get().getResourceSettings())
+        {
+            if (settings.isAvailable(c))
+            {
+                for (String volume : settings.getDockerVolumes(c))
+                {
+                    volumeLines.add("-v '" + volume + "':'" + volume + "'");
+                }
+            }
+        }
+
+        return volumeLines;
+    }
+
     @Override
     public List<File> getSequenceJobInputFiles(PipelineJob job)
     {

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/GLNexusHandler.java

@@ -191,7 +191,7 @@ else if (genomeIds.isEmpty())
                 {
                     ctx.getLogger().debug("Running GLNexus for contig: " + r.getSequenceName());
                     ctx.getJob().setStatus(PipelineJob.TaskStatus.running, "Processing: " + r.getSequenceName());
-                    new GLNexusWrapper(ctx.getLogger()).execute(inputVcfs, contigVcf, ctx.getFileManager(), binVersion, configType, r);
+                    new GLNexusWrapper(ctx.getLogger()).execute(inputVcfs, contigVcf, ctx.getFileManager(), binVersion, configType, r, ctx);
                     vcfs.add(contigVcf);
                     try
                     {
@@ -261,7 +261,7 @@ private File ensureLocalCopy(File input, File workingDirectory, PipelineOutputTr
             }
         }
 
-        public void execute(List<File> inputGvcfs, File outputVcf, PipelineOutputTracker tracker, String binVersion, String configType, SAMSequenceRecord rec) throws PipelineJobException
+        public void execute(List<File> inputGvcfs, File outputVcf, PipelineOutputTracker tracker, String binVersion, String configType, SAMSequenceRecord rec, JobContext ctx) throws PipelineJobException
         {
             File workDir = outputVcf.getParentFile();
             tracker.addIntermediateFile(outputVcf);
@@ -291,6 +291,7 @@ public void execute(List<File> inputGvcfs, File outputVcf, PipelineOutputTracker
                 writer.println("sudo $DOCKER run --rm=true \\");
                 writer.println("\t-v \"${WD}:/work\" \\");
                 writer.println("\t-v \"${HOME}:/homeDir\" \\");
+                ctx.getDockerVolumes().forEach(writer::println);
                 writer.println("\t -w /work \\");
                 if (!StringUtils.isEmpty(System.getenv("TMPDIR")))
                 {

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/JobContextImpl.java

@@ -14,6 +14,7 @@
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.LinkedHashSet;
 
 /**
@@ -128,4 +129,9 @@ public LinkedHashSet<RecordedAction> getActions()
     {
         return _actions;
     }
+
+    public Collection<String> getDockerVolumes()
+    {
+        return _job.getDockerVolumes();
+    }
 }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceJob.java

@@ -36,6 +36,7 @@
 import org.labkey.api.sequenceanalysis.SequenceOutputFile;
 import org.labkey.api.sequenceanalysis.pipeline.HasJobParams;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputTracker;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.settings.AppProps;
 import org.labkey.api.util.FileType;
 import org.labkey.api.util.FileUtil;
@@ -73,6 +74,7 @@ public class SequenceJob extends PipelineJob implements FileAnalysisJobSupport,
     private List<File> _inputFiles;
     private List<SequenceOutputFile> _outputsToCreate = new ArrayList<>();
     private PipeRoot _folderFileRoot;
+    private Collection<String> _dockerVolumes;
 
     transient private JSONObject _params;
 
@@ -104,6 +106,7 @@ protected SequenceJob(SequenceJob parentJob, String jobName, String subdirectory
         _folderPrefix = parentJob._folderPrefix;
         _inputFiles = parentJob._inputFiles;
         _folderFileRoot = parentJob._folderFileRoot;
+        _dockerVolumes = parentJob._dockerVolumes;
 
         _params = parentJob.getParameterJson();
 
@@ -133,6 +136,7 @@ public SequenceJob(String providerName, Container c, User u, @Nullable String jo
         writeParameters(params);
 
         _folderFileRoot = c.isWorkbook() ? PipelineService.get().findPipelineRoot(c.getParent()) : pipeRoot;
+        _dockerVolumes = SequencePipelineService.get().getDockerVolumes(c);
 
         setLogFile(_getLogFile());
         writeSupportToDisk();
@@ -182,6 +186,16 @@ public void setFolderFileRoot(PipeRoot folderFileRoot)
         _folderFileRoot = folderFileRoot;
     }
 
+    public Collection<String> getDockerVolumes()
+    {
+        return Collections.unmodifiableCollection(_dockerVolumes);
+    }
+
+    public void setDockerVolumes(Collection<String> dockerVolumes)
+    {
+        _dockerVolumes = dockerVolumes;
+    }
+
     public void setDescription(String description)
     {
         _description = description;

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceTaskHelper.java

@@ -50,6 +50,7 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -344,4 +345,9 @@ public void cacheExpDatasForParams() throws PipelineJobException
             }
         }
     }
+
+    public Collection<String> getDockerVolumes()
+    {
+        return _job.getDockerVolumes();
+    }
 }