Merge discvr-25.3 to develop

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AbstractAlignmentStepProvider.java

@@ -34,6 +34,7 @@ abstract public class AbstractAlignmentStepProvider<StepType extends AlignmentSt
     public static String SUPPORT_MERGED_UNALIGNED = "supportsMergeUnaligned";
     public static String COLLECT_WGS_METRICS = "collectWgsMetrics";
     public static String CONVERT_TO_CRAM = "convertToCram";
+    public static String CRAM_ARCHIVAL_MODE = "doCramArchivalMode";
     public static String COLLECT_WGS_METRICS_NON_ZERO = "collectWgsMetricsNonZero";
     public static String DISCARD_BAM = "discardBam";
 
@@ -116,6 +117,10 @@ private static List<ToolParameterDescriptor> getParamList(List<ToolParameterDesc
             put("checked", false);
         }}, false));
 
+        parameters.add(ToolParameterDescriptor.create("doCramArchivalMode", "CRAM Archival Mode", "If selected, the CRAM will undergo additional compression to save space. This is lossy and may not be compatible with all downstream tools. See samtools view --output-fmt-option archive", "checkbox", new JSONObject(){{
+            put("checked", false);
+        }}, false));
+
         parameters.add(ToolParameterDescriptor.create(ALIGNMENT_MODE_PARAM, "Alignment Mode", "If your readset has more than one pair of FASTQs, there pipeline can either align each pair sequentially (and then merge these BAMs), or merge the pairs of FASTQs first and then perform alignment once.  The default is to align each pair of FASTQs separately; however, some pipelines like STAR require the latter.", "ldk-simplecombo", new JSONObject(){{
             put("storeValues", ALIGNMENT_MODE.ALIGN_THEN_MERGE.name() + ";" + ALIGNMENT_MODE.MERGE_THEN_ALIGN.name());
             put("value", alignmentMode.name());

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignmentStep.java

@@ -80,11 +80,19 @@ default boolean supportsMetrics()
     boolean supportsGzipFastqs();
 
     @Override
-    AlignmentStepProvider getProvider();
+    AlignmentStepProvider<?> getProvider();
 
     default String getAlignmentDescription()
     {
-        return "Aligner: " + getProvider().getName();
+        ToolParameterDescriptor cramArchivalParam = getProvider().getParameterByName(AbstractAlignmentStepProvider.CRAM_ARCHIVAL_MODE);
+        boolean doArchival = cramArchivalParam != null && cramArchivalParam.extractValue(getPipelineCtx().getJob(), getProvider(), getStepIdx(), Boolean.class, false);
+        String ret = "Aligner: " + getProvider().getName();
+        if (doArchival)
+        {
+            ret = ret + "\nCRAM Archival Mode";
+        }
+
+        return ret;
     }
 
     interface AlignmentOutput extends PipelineStepOutput

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/SamtoolsCramConverter.java

@@ -18,25 +18,44 @@ public SamtoolsCramConverter(Logger log)
         super(log);
     }
 
-    public File convert(File inputBam, File outputCram, File gzippedFasta, boolean doIndex, @Nullable Integer threads) throws PipelineJobException
+    public File convert(File inputBam, File outputCram, File gzippedFasta, boolean doIndex, @Nullable Integer threads, boolean archivalMode) throws PipelineJobException
     {
         getLogger().info("Converting SAM/BAM to CRAM: " + inputBam.getPath());
+        if (inputBam.equals(outputCram))
+        {
+            throw new PipelineJobException("Input/output files are the same");
+        }
 
         List<String> params = new ArrayList<>();
         params.add(getSamtoolsPath().getPath());
         params.add("view");
 
-        params.add("-C");
+        params.add("--output-fmt");
+        params.add("cram,version=3.0" + (archivalMode ? ",lossy_names=1" : ""));
 
         params.add("-o");
         params.add(outputCram.getPath());
 
+        // CRAM does, however, have an optional archive settings mode (samtools view  ...)
+        // which is a lossy compression, doing things like removing read names, removing additional accessory fields, and additional compression of quality scores.
+        // In all cases, the base sequence of the reads is preserved: https://www.htslib.org/doc/samtools.html
+        if (archivalMode)
+        {
+            params.add("--output-fmt-option");
+            params.add("archive");
+        }
+
         params.add("-T");
         params.add(gzippedFasta.getPath());
 
+        if (doIndex)
+        {
+            params.add("--write-index");
+        }
+
         if (threads != null)
         {
-            params.add("--threads");
+            params.add("-@");
             params.add(String.valueOf(threads));
         }
 
@@ -49,11 +68,6 @@ public File convert(File inputBam, File outputCram, File gzippedFasta, boolean d
             throw new PipelineJobException("Missing output: " + outputCram.getPath());
         }
 
-        if (doIndex)
-        {
-            doIndex(outputCram, threads);
-        }
-
         return outputCram;
     }
 

SequenceAnalysis/pipeline_code/extra_tools_install.sh

@@ -305,3 +305,17 @@ then
 else
     echo "Already installed"
 fi
+
+if [[ ! -e ${LKTOOLS_DIR}/bbmap || ! -z $FORCE_REINSTALL ]];
+then
+    echo "Cleaning up previous installs"
+    rm -Rf $LKTOOLS_DIR/bbmap
+
+    wget https://sourceforge.net/projects/bbmap/files/BBMap_39.25.tar.gz
+    tar -xf BBMap_39.25.tar.gz
+
+    mv bbmap $LKTOOLS_DIR/
+    ln -s $LKTOOLS_DIR/bbmap/bbmap.sh $LKTOOLS_DIR/bbmap.sh
+else
+    echo "Already installed"
+fi

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java

@@ -5183,4 +5183,62 @@ public void setDoNotRequireSra(boolean doNotRequireSra)
             _doNotRequireSra = doNotRequireSra;
         }
     }
+
+    @RequiresSiteAdmin
+    public static class CreateExpDataForFileAction extends ConfirmAction<CreateExpDataForFileForm>
+    {
+        @Override
+        public void validateCommand(CreateExpDataForFileForm form, Errors errors)
+        {
+
+        }
+
+        @Override
+        public URLHelper getSuccessURL(CreateExpDataForFileForm form)
+        {
+            return getContainer().getStartURL(getUser());
+        }
+
+        @Override
+        public ModelAndView getConfirmView(CreateExpDataForFileForm form, BindException errors) throws Exception
+        {
+            return new HtmlView(HtmlString.unsafe("This will create a new ExpData with a DataFileUrl pointing to the provided URI. This should be a full URI, such as file:///my/path/myFile.txt." +
+                    "<br><br>" +
+                    "<label>DataFileUrl </label><input name=\"dataFileUrl\" value = \"" + HtmlString.of(form.getDataFileUrl()) + "\"><br>"));
+        }
+
+        @Override
+        public boolean handlePost(CreateExpDataForFileForm form, BindException errors) throws Exception
+        {
+            URI newUri = URI.create(form.getDataFileUrl());
+            File f = new File(newUri);
+            if (!f.exists())
+            {
+                throw new PipelineJobException("Missing file: " + form.getDataFileUrl());
+            }
+
+            DataType dataType = new DataType("File");
+
+            ExpData d = ExperimentService.get().createData(getContainer(), dataType, f.getName());
+            d.setDataFileURI(newUri);
+            d.save(getUser());
+
+            return true;
+        }
+    }
+
+    public static class CreateExpDataForFileForm
+    {
+        private String _dataFileUrl;
+
+        public String getDataFileUrl()
+        {
+            return _dataFileUrl;
+        }
+
+        public void setDataFileUrl(String dataFileUrl)
+        {
+            _dataFileUrl = dataFileUrl;
+        }
+    }
 }

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java

@@ -91,6 +91,7 @@
 import org.labkey.sequenceanalysis.query.SequenceAnalysisUserSchema;
 import org.labkey.sequenceanalysis.query.SequenceTriggerHelper;
 import org.labkey.sequenceanalysis.run.RestoreSraDataHandler;
+import org.labkey.sequenceanalysis.run.alignment.BBMapWrapper;
 import org.labkey.sequenceanalysis.run.alignment.BWAMem2Wrapper;
 import org.labkey.sequenceanalysis.run.alignment.BWAMemWrapper;
 import org.labkey.sequenceanalysis.run.alignment.BWASWWrapper;
@@ -157,8 +158,6 @@
 import org.labkey.sequenceanalysis.run.reference.VirusReferenceLibraryStep;
 import org.labkey.sequenceanalysis.run.util.CombineGVCFsHandler;
 import org.labkey.sequenceanalysis.run.util.FastqcRunner;
-import org.labkey.sequenceanalysis.run.util.GenomicsDBAppendHandler;
-import org.labkey.sequenceanalysis.run.util.GenomicsDBImportHandler;
 import org.labkey.sequenceanalysis.run.util.SVAnnotateStep;
 import org.labkey.sequenceanalysis.run.variant.DepthOfCoverageHandler;
 import org.labkey.sequenceanalysis.run.variant.GenotypeConcordanceStep;
@@ -309,6 +308,7 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new StarWrapper.Provider());
         SequencePipelineService.get().registerPipelineStep(new Pbmm2Wrapper.Provider());
         SequencePipelineService.get().registerPipelineStep(new VulcanWrapper.Provider());
+        SequencePipelineService.get().registerPipelineStep(new BBMapWrapper.Provider());
 
         //de novo assembly
         SequencePipelineService.get().registerPipelineStep(new TrinityRunner.Provider());
@@ -395,8 +395,6 @@ public static void registerPipelineSteps()
         SequenceAnalysisService.get().registerFileHandler(new RecalculateSequenceMetricsHandler());
         SequenceAnalysisService.get().registerFileHandler(new ListVcfSamplesHandler());
         SequenceAnalysisService.get().registerFileHandler(new MultiQCBamHandler());
-        SequenceAnalysisService.get().registerFileHandler(new GenomicsDBImportHandler());
-        SequenceAnalysisService.get().registerFileHandler(new GenomicsDBAppendHandler());
         SequenceAnalysisService.get().registerFileHandler(new MergeLoFreqVcfHandler());
         SequenceAnalysisService.get().registerFileHandler(new PangolinHandler());
         SequenceAnalysisService.get().registerFileHandler(new NextCladeHandler());

SequenceAnalysis/src/org/labkey/sequenceanalysis/api/picard/CigarPositionIterable.java

@@ -81,7 +81,7 @@ private void initializeCigar()
             int i = 0;
             for (char el : _explodedCigar)
             {
-                CigarOperator op = CigarOperator.valueOf(Character.toString(el));
+                CigarOperator op = CigarOperator.characterToEnum(el);
                 if (op.consumesReadBases())
                 {
                     _readPositions[i] = readPos;
@@ -155,7 +155,7 @@ public PositionInfo(SAMRecord record, int pos, char[] ops, Integer[] readPos, In
         {
             _record = record;
             _pos = pos;
-            _op = CigarOperator.valueOf(Character.toString(ops[pos]));
+            _op = CigarOperator.characterToEnum(ops[pos]);
             _readPos = readPos[pos];
             _refPos = refPos[pos];