Skip to content

Commit f422a4d

Browse files
bbimberbbimber
authored
Merge pull request #190 from LabKey/fb_merge_22.11_to_develop
Merge discvr-22.11 to develop
2 parents 2127613 + 80e7da8 commit f422a4d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+204
-97
lines changed

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/BaseSequencePanel.js‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,10 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
269269
Ext4.defer(field.focus, 200, field);
270270
}
271271
}
272+
},{
273+
xtype: 'checkbox',
274+
itemId: 'useReadsetContainer',
275+
fieldLabel: 'Submit Jobs to Same Folder/Workbook as Readset'
272276
}]
273277
}],
274278
buttons: [{
@@ -294,6 +298,12 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
294298
json = Ext4.decode(json);
295299
}
296300

301+
var useReadsetContainer = win.down('#useReadsetContainer').getValue();
302+
if (!useReadsetContainer) {
303+
delete json.useOutputFileContainer;
304+
delete json.submitJobToReadsetContainer;
305+
}
306+
297307
win.sequencePanel.applySavedValues(json);
298308
win.close();
299309
}
@@ -354,5 +364,15 @@ Ext4.define('SequenceAnalysis.panel.BaseSequencePanel', {
354364
Ext4.Array.forEach(sections, function(s){
355365
s.applySavedValues(values);
356366
}, this);
367+
368+
// For top-level properties:
369+
Ext4.Array.forEach(['submissionType', 'useOutputFileContainer', 'submitJobToReadsetContainer'], function(val) {
370+
if (values[val]) {
371+
var field = this.down('[name="' + val + '"]');
372+
if (field) {
373+
field.setValue(values[val]);
374+
}
375+
}
376+
}, this);
357377
}
358378
});

‎SequenceAnalysis/resources/web/SequenceAnalysis/panel/SequenceAnalysisPanel.js‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -318,9 +318,8 @@ Ext4.define('SequenceAnalysis.panel.SequenceAnalysisPanel', {
318318
checked: false,
319319
xtype: 'checkbox'
320320
}, this.getSaveTemplateCfg(),{
321-
fieldLabel: 'Submit Jobs To Same Container As Readset?',
321+
fieldLabel: 'Submit Jobs To Same Folder/Workbook As Readset?',
322322
helpPopup: 'If checked, each alignment job will be submitted to the container/workbook of the parent readset, rather than this container',
323-
hidden: !LABKEY.Security.currentUser.isAdmin,
324323
name: 'submitJobToReadsetContainer',
325324
inputValue: true,
326325
uncheckedValue: false,

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ProcessVariantsHandler.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,8 +677,15 @@ private void processFile(File input, Integer libraryId, Integer readsetId, JobCo
677677
so1.setCreated(new Date());
678678
so1.setModified(new Date());
679679
so1.setReadset(readsetId);
680-
so1.setDescription("Total samples: " + sampleCount);
680+
String description = "Total samples: " + sampleCount;
681681

682+
String extraDescription = StringUtils.trimToNull(ctx.getParams().optString("jobDescription"));
683+
if (extraDescription != null)
684+
{
685+
description = description + '\n' + extraDescription;
686+
}
687+
688+
so1.setDescription(description);
682689
_resumer.getFileManager().addSequenceOutput(so1);
683690
}
684691
}

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReadsetInitTask.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.labkey.api.pipeline.RecordedAction;
2828
import org.labkey.api.pipeline.RecordedActionSet;
2929
import org.labkey.api.pipeline.WorkDirectoryTask;
30+
import org.labkey.api.sequenceanalysis.SequenceAnalysisService;
3031
import org.labkey.api.sequenceanalysis.pipeline.TaskFileManager;
3132
import org.labkey.api.util.Compress;
3233
import org.labkey.api.util.FileType;
@@ -45,8 +46,10 @@
4546
import java.util.Arrays;
4647
import java.util.Collection;
4748
import java.util.Collections;
49+
import java.util.HashMap;
4850
import java.util.HashSet;
4951
import java.util.List;
52+
import java.util.Map;
5053
import java.util.Set;
5154

5255
/**
@@ -163,6 +166,8 @@ public RecordedActionSet run() throws PipelineJobException
163166
List<FileGroup> fileGroups = getHelper().getSettings().getFileGroups(getPipelineJob());
164167
List<SequenceReadsetImpl> readsets = getHelper().getSettings().getReadsets(getPipelineJob());
165168

169+
checkForDuplicateFileNames(readsets, fileGroups);
170+
166171
if (!SequenceNormalizationTask.shouldRunRemote(getJob()))
167172
{
168173
getJob().getLogger().info("No files required external normalization, processing inputs locally");
@@ -486,5 +491,61 @@ private static void moveInputToAnalysisDir(File input, SequenceJob job, Collecti
486491
throw new PipelineJobException(e);
487492
}
488493
}
494+
495+
private void checkForDuplicateFileNames(List<SequenceReadsetImpl> readsets, List<FileGroup> fileGroups) throws PipelineJobException
496+
{
497+
// check for duplicate filename between incoming and existing
498+
for (SequenceReadsetImpl r : readsets)
499+
{
500+
boolean readsetExists = r.getReadsetId() != null && r.getReadsetId() > 0;
501+
SequenceReadsetImpl existingReadset = readsetExists ? ((SequenceReadsetImpl) SequenceAnalysisService.get().getReadset(r.getReadsetId(), getJob().getUser())) : null;
502+
List<ReadDataImpl> preexistingReadData = readsetExists ? existingReadset.getReadDataImpl() : Collections.emptyList();
503+
if (!preexistingReadData.isEmpty())
504+
{
505+
Map<String, File> existingFileNames = new HashMap<>();
506+
preexistingReadData.forEach(rd -> {
507+
existingFileNames.put(rd.getFile1().getName(), rd.getFile1());
508+
if (rd.getFile2() != null)
509+
{
510+
existingFileNames.put(rd.getFile2().getName(), rd.getFile2());
511+
}
512+
});
513+
514+
Map<String, File> sharedFns = new HashMap<>();
515+
for (FileGroup fg : fileGroups)
516+
{
517+
if (r.getFileSetName() != null && r.getFileSetName().equals(fg.name))
518+
{
519+
for (FileGroup.FilePair fp : fg.filePairs)
520+
{
521+
if (existingFileNames.containsKey(fp.file1.getName()))
522+
{
523+
sharedFns.put(fp.file1.getName(), fp.file1);
524+
}
525+
526+
if (fp.file2 != null && existingFileNames.containsKey(fp.file2.getName()))
527+
{
528+
sharedFns.put(fp.file2.getName(), fp.file2);
529+
}
530+
}
531+
}
532+
}
533+
534+
if (!sharedFns.isEmpty())
535+
{
536+
getJob().getLogger().debug("Duplicate file names found between incoming and existing for: " + r.getName());
537+
for (String newFile : sharedFns.keySet())
538+
{
539+
long diff = Math.abs(sharedFns.get(newFile).length() - existingFileNames.get(newFile).length());
540+
getJob().getLogger().debug("File name: " + newFile + ", with size difference: " + diff);
541+
if (diff < 100)
542+
{
543+
throw new PipelineJobException("Identical filenames with nearly identical size detected between existing and new files for readset: " + r.getName());
544+
}
545+
}
546+
}
547+
}
548+
}
549+
}
489550
}
490551

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/VariantProcessingJob.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ else if (_scatterGatherMethod == VariantProcessingStep.ScatterGatherMethod.fixed
159159

160160
public boolean doAllowSplitContigs()
161161
{
162-
return getParameterJson().optBoolean("scatterGather.allowSplitChromosomes", true);
162+
return getParameterJson().optBoolean("scatterGather.allowSplitChromosomes", false);
163163
}
164164

165165
public boolean isScatterJob()

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/PbsvJointCallingHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ private File runPbsvCall(JobContext ctx, List<File> inputs, ReferenceGenome geno
248248

249249
for (File s : inputs)
250250
{
251-
String ret = StringUtils.trimToNull(runner.executeWithOutput(Arrays.asList("/bin/bash", "-c", tabix.getExe().getPath() + " -l '" + s.getPath() + "' | awk ' { $1 == \"" + contig + "\" } ' | wc -l")));
251+
String ret = StringUtils.trimToNull(runner.executeWithOutput(Arrays.asList("/bin/bash", "-c", tabix.getExe().getPath() + " -l '" + s.getPath() + "' | awk ' $1 == \"" + contig + "\" ' | wc -l")));
252252
if ("0".equals(ret))
253253
{
254254
ctx.getLogger().info("Sample is missing contig: " + contig + ", skipping: " + s.getPath());

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/KingInferenceStep.java‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
6464

6565
File plinkOut = new File(outputDirectory, "plink");
6666
output.addIntermediateFile(new File(plinkOut.getPath() + ".bed"));
67-
output.addIntermediateFile(new File(plinkOut.getPath() + ".fam"));
67+
//output.addIntermediateFile(new File(plinkOut.getPath() + ".fam"));
6868
output.addIntermediateFile(new File(plinkOut.getPath() + ".bim"));
6969
output.addIntermediateFile(new File(plinkOut.getPath() + ".log"));
7070
output.addIntermediateFile(new File(plinkOut.getPath() + "-temporary.psam"));
@@ -90,11 +90,8 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
9090
plinkArgs.add("--chr");
9191
plinkArgs.add(StringUtils.join(toKeep, ","));
9292
}
93-
else
94-
{
95-
plinkArgs.add("--allow-extra-chr");
96-
}
9793

94+
plinkArgs.add("--allow-extra-chr");
9895
plinkArgs.add("--silent");
9996

10097
plinkArgs.add("--max-alleles");
@@ -138,7 +135,7 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
138135
kingArgs.add(threads.toString());
139136
}
140137

141-
kingArgs.add("--kinship");
138+
kingArgs.add("--related");
142139

143140
File kinshipOutput = new File(outputDirectory, SequenceAnalysisService.get().getUnzippedBaseName(inputVCF.getName()) + ".kin");
144141
wrapper.execute(kingArgs);

‎jbrowse/test/src/org/labkey/test/tests/external/labModules/JBrowseTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -870,6 +870,7 @@ private void testGridFailureConditions()
870870
private void testVariantTableComparators() throws Exception {
871871
beginAt("/home/jbrowse-variantTable.view?session=mgap&trackId=mgap_hg38&location=1:116589678..117411688");
872872
waitForElement(Locator.tagWithClass("div", "MuiDataGrid-root"));
873+
waitForElement(Locator.tagWithText("div", "116981270")); //proxy for grid loading
873874

874875
// Test filtering AF with wrapped comparators
875876
waitAndClick(Locator.tagWithAttributeContaining("button", "aria-label", "Show filters"));

‎singlecell/resources/chunks/AvgExpression.R‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1+
if (!file.exists('/homeDir/.netrc')) {
2+
print(list.files('/homeDir'))
3+
stop('Unable to find file: /homeDir/.netrc')
4+
}
5+
6+
invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = '/homeDir/.netrc'))
7+
Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
8+
19
GenerateAveragedData <- function(seuratObj, groupFields, addMetadata) {
210
if (addMetadata && !'cDNA_ID' %in% names(seuratObj@meta.data)) {
311
stop('A field names cDNA_ID must exist when addMetadata=TRUE')
@@ -7,25 +15,7 @@ GenerateAveragedData <- function(seuratObj, groupFields, addMetadata) {
715
stop('When addMetadata=TRUE, cDNA_ID must be part of groupFields')
816
}
917

10-
meta <- unique(seuratObj@meta.data[,groupFields, drop = F])
11-
rownames(meta) <- apply(meta, 1, function(y){
12-
return(paste0(y, collapse = '_'))
13-
})
14-
15-
Seurat::Idents(seuratObj) <- rownames(meta)
16-
17-
for (assayName in names(seuratObj@assays)) {
18-
if (!(!identical(seuratObj@assays[[assayName]]@counts, seuratObj@assays[[assayName]]@data))){
19-
print(paste0('Seurat assay', assayName, ' does not appear to be normalized, running now:'))
20-
seuratObj <- Seurat::NormalizeData(seuratObj, verbose = FALSE, assay = assayName)
21-
}
22-
}
23-
24-
a <- Seurat::AverageExpression(seuratObj, return.seurat = T, verbose = F)
25-
a <- Seurat::AddMetaData(a, meta)
26-
27-
totals <- seuratObj@meta.data %>% group_by_at(groupFields) %>% summarise(TotalCells = n())
28-
a$TotalCells <- totals$TotalCells
18+
a <- CellMembrane::PseudobulkSeurat(seuratObj, groupFields = groupFields, assays = assayName)
2919

3020
if (addMetadata) {
3121
a <- Rdiscvr::QueryAndApplyMetadataUsingCDNA(a)

‎singlecell/resources/chunks/ClrNormalizeByGroup.R‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
for (datasetId in names(seuratObjects)) {
22
printName(datasetId)
33
seuratObj <- readRDS(seuratObjects[[datasetId]])
4-
seuratObj <- CellMembrane::ClrNormalizeByGroup(seuratObj, groupingVar = groupingVar, assayName = assayName, targetAssayName = targetAssayName, margin = margin, minCellsPerGroup = minCellsPerGroup, calculatePerFeatureUCell = calculatePerFeatureUCell, featureInclusionList = featureWhitelist, featureExclusionList = featureExclusionList)
4+
seuratObj <- CellMembrane::ClrNormalizeByGroup(seuratObj, groupingVar = groupingVar, assayName = assayName, targetAssayName = targetAssayName, margin = margin, minCellsPerGroup = minCellsPerGroup, calculatePerFeatureUCell = calculatePerFeatureUCell, featureInclusionList = featureWhitelist, featureExclusionList = featureExclusionList, doAsinhTransform = doAsinhTransform)
55

66
saveData(seuratObj, datasetId)
77

0 commit comments

Comments
 (0)