Merge pull request #210 from LabKey/fb_merge_22.11_to_develop

Merge 22.11 to develop

Author: bbimber

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/SequenceAnalysisService.java

@@ -105,4 +105,13 @@ static public void setInstance(SequenceAnalysisService instance)
     abstract public void mergeFastqFiles(File output, List<File> inputs, Logger log) throws PipelineJobException;
 
     abstract public void registerAccessoryFileProvider(Function<File, List<File>> fn);
+
+    abstract public void registerReadsetListener(ReadsetListener listener);
+
+    public static interface ReadsetListener
+    {
+        public void onReadsetCreate(User u, Readset rs, @Nullable Readset replacedReadset, @Nullable PipelineJob job);
+
+        public boolean isAvailable(Container c, User u);
+    }
 }

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java

@@ -78,7 +78,7 @@ private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable
                 File lockFile = new File(webserverIndexDir.getPath() + ".copyLock");
                 if (lockFile.exists())
                 {
-                    ctx.getLogger().error("Another job is actively saving this cached index.  Will not re-created, but if this job tries to use it before copy is complete this might cause issues.");
+                    ctx.getLogger().error("Another job is actively saving this cached index.  This job will skip that step; however, if this job tries to start alignment and use the index before copy is complete this might cause issues.");
                 }
 
                 hasCachedIndex = true;

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/VariantProcessingStep.java

@@ -38,6 +38,11 @@ default void init(PipelineJob job, SequenceAnalysisJobSupport support, List<Sequ
 
     }
 
+    default void complete(PipelineJob job, List<SequenceOutputFile> inputs, List<SequenceOutputFile> outputsCreated, SequenceAnalysisJobSupport support) throws PipelineJobException
+    {
+
+    }
+
     enum ScatterGatherMethod
     {
         none(),

SequenceAnalysis/resources/web/SequenceAnalysis/window/ReprocessLibraryWindow.js

@@ -27,6 +27,11 @@ Ext4.define('SequenceAnalysis.window.ReprocessLibraryWindow', {
                 html: 'This will cause the server to reprocess the selected reference genomes, which should be used if the data or attributes have changed.',
                 border: false,
                 style: 'padding-bottom: 10px;'
+            },{
+                xtype: 'checkbox',
+                itemId: 'alignerIndexesOnly',
+                fieldLabel: 'Prepare Aligner Indexes Only',
+                checked: false
             }],
             buttons: [{
                 text: 'Submit',
@@ -46,10 +51,13 @@ Ext4.define('SequenceAnalysis.window.ReprocessLibraryWindow', {
     onSubmit: function(){
         Ext4.Msg.wait('Saving...');
 
+        var skipFastaRecreate = !!this.down('#alignerIndexesOnly').getValue();
+
         LABKEY.Ajax.request({
             url: LABKEY.ActionURL.buildURL('sequenceanalysis', 'recreateReferenceLibrary'),
             jsonData: {
-                libraryIds: this.libraryIds
+                libraryIds: this.libraryIds,
+                skipFastaRecreate: skipFastaRecreate
             },
             scope: this,
             success: function(){

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java

@@ -3190,7 +3190,7 @@ public ApiResponse execute(RecreateReferenceLibraryForm form, BindException erro
                         throw new PipelineValidationException("Insufficient permissions to update reference genome: " + libraryId);
                     }
 
-                    PipelineService.get().queueJob(ReferenceLibraryPipelineJob.recreate(c, getUser(), root, libraryId, form.isSkipCacheIndexes(), false));
+                    PipelineService.get().queueJob(ReferenceLibraryPipelineJob.recreate(c, getUser(), root, libraryId, form.isSkipCacheIndexes(), false, form.isSkipFastaRecreate()));
                 }
 
                 return new ApiSimpleResponse("success", true);
@@ -3207,6 +3207,7 @@ public static class RecreateReferenceLibraryForm
     {
         private int[] _libraryIds;
         private boolean _skipCacheIndexes = false;
+        private boolean _skipFastaRecreate = false;
 
         public int[] getLibraryIds()
         {
@@ -3227,6 +3228,16 @@ public void setSkipCacheIndexes(boolean skipCacheIndexes)
         {
             _skipCacheIndexes = skipCacheIndexes;
         }
+
+        public boolean isSkipFastaRecreate()
+        {
+            return _skipFastaRecreate;
+        }
+
+        public void setSkipFastaRecreate(boolean skipFastaRecreate)
+        {
+            _skipFastaRecreate = skipFastaRecreate;
+        }
     }
 
     @RequiresPermission(ReadPermission.class)

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java

@@ -533,6 +533,7 @@ public static List<String> getAssociatedFiles(File f, boolean includeGz)
         if (_bamFileType.isType(f))
         {
             ret.add(f.getName() + ".bai");
+            ret.add(f.getName() + ".pbi");
         }
         else if (_cramFileType.isType(f))
         {

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisManager.java

@@ -718,7 +718,7 @@ public ReferenceLibraryPipelineJob createReferenceLibrary(Container c, User u, S
         try
         {
             PipeRoot root = PipelineService.get().getPipelineRootSetting(c);
-            ReferenceLibraryPipelineJob job = new ReferenceLibraryPipelineJob(c, u, root, name, assemblyId, description, libraryMembers, null, skipCacheIndexes, skipTriggers, unplacedContigPrefixes, extraTriggers);
+            ReferenceLibraryPipelineJob job = new ReferenceLibraryPipelineJob(c, u, root, name, assemblyId, description, libraryMembers, null, skipCacheIndexes, skipTriggers, unplacedContigPrefixes, extraTriggers, false);
             PipelineService.get().queueJob(job);
 
             return job;

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisServiceImpl.java

@@ -34,6 +34,7 @@
 import org.labkey.api.sequenceanalysis.ReferenceLibraryHelper;
 import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
 import org.labkey.api.sequenceanalysis.SequenceDataProvider;
+import org.labkey.api.sequenceanalysis.model.Readset;
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
 import org.labkey.api.util.FileType;
@@ -80,6 +81,7 @@ public class SequenceAnalysisServiceImpl extends SequenceAnalysisService
     private final Set<SequenceOutputHandler<SequenceOutputHandler.SequenceOutputProcessor>> _fileHandlers = new HashSet<>();
     private final Set<SequenceOutputHandler<SequenceOutputHandler.SequenceReadsetProcessor>> _readsetHandlers = new HashSet<>();
     private final Map<String, SequenceDataProvider> _dataProviders = new HashMap<>();
+    private final List<ReadsetListener> _readsetListeners = new ArrayList<>();
 
     private SequenceAnalysisServiceImpl()
     {
@@ -539,4 +541,20 @@ public List<Function<File, List<File>>> getAccessoryFileProviders()
     {
         return Collections.unmodifiableList(_accessoryFileProviders);
     }
+
+    @Override
+    public void registerReadsetListener(ReadsetListener listener)
+    {
+        _readsetListeners.add(listener);
+    }
+
+    public void onReadsetCreate(User u, Readset rs, @Nullable Readset replacedReadset, @Nullable PipelineJob job)
+    {
+        _readsetListeners.forEach(l -> {
+            Container c = ContainerManager.getForId(rs.getContainer());
+            if (l.isAvailable(c, u)) {
+                l.onReadsetCreate(u, rs, replacedReadset, job);
+            }
+        });
+    }
 }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java

@@ -57,9 +57,9 @@
 import org.labkey.sequenceanalysis.SequenceAnalysisServiceImpl;
 import org.labkey.sequenceanalysis.model.ReferenceLibraryMember;
 import org.labkey.sequenceanalysis.run.util.FastaIndexer;
-import picard.sam.CreateSequenceDictionary;
 
 import java.io.File;
+import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -113,6 +113,33 @@ public PipelineJob.Task<?> createTask(PipelineJob job)
             return new CreateReferenceLibraryTask(this, job);
         }
 
+        @Override
+        public boolean isParticipant(PipelineJob job) throws IOException
+        {
+            if (!(job instanceof ReferenceLibraryPipelineJob rpj))
+            {
+                throw new IllegalArgumentException("Pipeline job is not a ReferenceLibraryPipelineJob");
+            }
+
+            if (rpj.isSkipFastaRecreate())
+            {
+                job.getLogger().debug("Will skip re-creation of FASTA");
+                try
+                {
+                    ReferenceGenomeImpl genome = SequenceAnalysisServiceImpl.get().getReferenceGenome(rpj.getLibraryId(), job.getUser());
+                    rpj.setReferenceGenome(genome);
+                }
+                catch (PipelineJobException e)
+                {
+                    job.error(e.getMessage(), e);
+                }
+
+                return false;
+            }
+
+            return super.isParticipant(job);
+        }
+
         @Override
         public boolean isJobComplete(PipelineJob job)
         {

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/PrepareAlignerIndexesTask.java

@@ -47,23 +47,22 @@ public Factory()
         @Override
         public boolean isParticipant(PipelineJob job)
         {
-
             if (!SequenceTaskHelper.isAlignmentUsed(job))
             {
                 return false;
             }
 
             try
             {
-                SequenceTaskHelper taskHelper = new SequenceTaskHelper((SequenceAlignmentJob)job, job.getLogFile().getParentFile());
-                AlignmentStep alignmentStep = taskHelper.getSingleStep(AlignmentStep.class).create(taskHelper);
-
                 if (!(job instanceof SequenceAlignmentJob))
                 {
                     job.getLogger().error("Job is not a SequenceAlignmentJob");
                     return true;
                 }
 
+                SequenceTaskHelper taskHelper = new SequenceTaskHelper((SequenceAlignmentJob)job, job.getLogFile().getParentFile());
+                AlignmentStep alignmentStep = taskHelper.getSingleStep(AlignmentStep.class).create(taskHelper);
+
                 ReferenceGenome referenceGenome = ((SequenceAlignmentJob)job).getTargetGenome();
                 if (referenceGenome == null)
                 {