Skip to content

Commit d4cf8a8

Browse files
bbimberbbimber
authored
Merge pull request #403 from LabKey/fb_merge_25.11_to_develop
Merge 25.11 to develop
2 parents 06789e2 + 8d4759a commit d4cf8a8

File tree

14 files changed

+458
-400
lines changed

14 files changed

+458
-400
lines changed

‎SequenceAnalysis/pipeline_code/extra_tools_install.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
358358
echo "Install hifiasm"
359359
echo ""
360360
cd $LKSRC_DIR
361-
if [[ ! -e ${LKTOOLS_DIR}/primer3_core || ! -z $FORCE_REINSTALL ]];
361+
if [[ ! -e ${LKTOOLS_DIR}/hifiasm || ! -z $FORCE_REINSTALL ]];
362362
then
363363
echo "Cleaning up previous installs"
364364
rm -Rf $LKTOOLS_DIR/hifiasm*

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/OutputIntegrationTests.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ public static class VariantProcessingTest extends SequenceIntegrationTests.Abstr
6363
private static final String PROJECT_NAME = "VariantProcessingTestProject";
6464

6565
@BeforeClass
66-
public static void initialSetUp() throws Exception
66+
public static void initialSetUp()
6767
{
6868
doInitialSetUp(PROJECT_NAME);
6969
}
@@ -91,12 +91,12 @@ public void testVariantProcessing() throws Exception
9191

9292
//create VCF, import as outputfile
9393
String basename = "TestFile_" + FileUtil.getTimestamp();
94-
File vcf = new File(_pipelineRoot, basename + ".vcf.gz");
94+
File vcf = FileUtil.appendName(getPipelineRoot(_project), basename + ".vcf.gz");
9595
Integer outputFileId = createTestVcf(genomeId, vcf);
9696

9797
//make job params
9898
String jobName = "TestVariantProcessing";
99-
JSONObject config = substituteParams(new File(_sampleData, VARIANT_JOB), jobName);
99+
JSONObject config = substituteParams(FileUtil.appendName(_sampleData, VARIANT_JOB), jobName);
100100
Set<Integer> outputFileIds = Collections.singleton(outputFileId);
101101

102102
TableInfo ti = QueryService.get().getUserSchema(TestContext.get().getUser(), _project, SequenceAnalysisSchema.SCHEMA_NAME).getTable(SequenceAnalysisSchema.TABLE_OUTPUTFILES, null);
@@ -129,7 +129,7 @@ public void testVariantProcessing() throws Exception
129129
}
130130
}
131131

132-
protected Set<PipelineJob> createOutputHandlerJob(String jobName, JSONObject config, Class handlerClass, Set<Integer> outputFileIDs) throws Exception
132+
protected Set<PipelineJob> createOutputHandlerJob(String jobName, JSONObject config, Class<?> handlerClass, Set<Integer> outputFileIDs) throws Exception
133133
{
134134
Map<String, Object> headers = new HashMap<>();
135135
headers.put("Content-Type", "application/json");
@@ -200,7 +200,7 @@ private int createTestVcf(int genomeId, File vcf)
200200
Integer dataId = new TableSelector(SequenceAnalysisSchema.getTable(SequenceAnalysisSchema.TABLE_REF_LIBRARIES), PageFlowUtil.set("fasta_file"), new SimpleFilter(FieldKey.fromString("rowid"), genomeId), null).getObject(Integer.class);
201201
ExpData data = ExperimentService.get().getExpData(dataId);
202202

203-
File dictFile = new File(data.getFile().getParent(), FileUtil.getBaseName(data.getFile().getName()) + ".dict");
203+
File dictFile = FileUtil.appendName(data.getFile().getParentFile(), FileUtil.getBaseName(data.getFile().getName()) + ".dict");
204204
if (dictFile.exists())
205205
{
206206
SAMSequenceDictionary dict = SAMSequenceDictionaryExtractor.extractDictionary(dictFile.toPath());
@@ -222,7 +222,7 @@ private int createTestVcf(int genomeId, File vcf)
222222
writer.add(vcb.make());
223223
}
224224

225-
ExpData d = createExpData(vcf);
225+
ExpData d = createExpData(vcf, _project);
226226
Map<String, Object> params = new CaseInsensitiveHashMap<>();
227227
params.put("name", "TestVcf");
228228
params.put("description", "Description");

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1924,7 +1924,7 @@ else if (o.has("relPath") || o.has("fileName"))
19241924

19251925
if (f == null || !f.exists())
19261926
{
1927-
throw new PipelineValidationException("Unknown file: " + o.getString("relPath") + " / " + o.getString("fileName"));
1927+
throw new PipelineValidationException("Unknown file: " + o.optString("relPath") + " / " + o.optString("fileName"));
19281928
}
19291929

19301930
ret.add(f);

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceIntegrationTests.java‎

Lines changed: 288 additions & 369 deletions
Large diffs are not rendered by default.

‎SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceRemoteIntegrationTests.java‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.labkey.api.pipeline.TaskId;
1616
import org.labkey.api.pipeline.WorkDirectory;
1717
import org.labkey.api.reader.Readers;
18+
import org.labkey.api.util.FileUtil;
1819
import org.labkey.api.writer.PrintWriters;
1920
import org.labkey.sequenceanalysis.pipeline.AlignmentInitTask;
2021
import org.labkey.sequenceanalysis.pipeline.PrepareAlignerIndexesTask;
@@ -25,6 +26,7 @@
2526
import java.io.IOException;
2627
import java.io.PrintWriter;
2728
import java.util.List;
29+
import java.util.Objects;
2830

2931
import static org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService.SEQUENCE_TOOLS_PARAM;
3032

@@ -40,35 +42,35 @@ public void setUp() throws Exception
4042
}
4143

4244
@BeforeClass
43-
public static void initialSetUp() throws Exception
45+
public static void initialSetUp()
4446
{
4547
doInitialSetUp(PROJECT_NAME);
4648
}
4749

4850
private File setupConfigDir(File outDir) throws IOException
4951
{
50-
File baseDir = new File(outDir, "config");
52+
File baseDir = FileUtil.appendName(outDir, "config");
5153
if (baseDir.exists())
5254
{
5355
FileUtils.deleteDirectory(baseDir);
5456
}
5557

56-
baseDir.mkdirs();
58+
FileUtil.mkdirs(baseDir);
5759

5860
if (_sampleData == null)
5961
{
6062
throw new IOException("_sampleData was null");
6163
}
6264

63-
File source = new File(_sampleData, "remotePipeline");
65+
File source = FileUtil.appendName(_sampleData, "remotePipeline");
6466
if (!source.exists())
6567
{
6668
throw new IOException("Unable to find file: " + source.getPath());
6769
}
6870

69-
FileUtils.copyFile(new File(source, "sequenceanalysisConfig.xml"), new File(baseDir, "sequenceanalysisConfig.xml"));
71+
FileUtils.copyFile(FileUtil.appendName(source, "sequenceanalysisConfig.xml"), FileUtil.appendName(baseDir, "sequenceanalysisConfig.xml"));
7072

71-
try (PrintWriter writer = PrintWriters.getPrintWriter(new File(baseDir, "pipelineConfig.xml")); BufferedReader reader = Readers.getReader(new File(source, "pipelineConfig.xml")))
73+
try (PrintWriter writer = PrintWriters.getPrintWriter(FileUtil.appendName(baseDir, "pipelineConfig.xml")); BufferedReader reader = Readers.getReader(FileUtil.appendName(source, "pipelineConfig.xml")))
7274
{
7375
String line;
7476
while ((line = reader.readLine()) != null)
@@ -83,12 +85,10 @@ private File setupConfigDir(File outDir) throws IOException
8385

8486
path = path.replaceAll("\\\\", "/");
8587
line = line.replaceAll("@@SEQUENCEANALYSIS_TOOLS@@", path);
86-
_log.info("Writing to pipelineConfig.xml: " + line);
8788
}
8889
else if (line.contains("@@WORK_DIR@@"))
8990
{
9091
line = line.replaceAll("@@WORK_DIR@@", outDir.getPath().replaceAll("\\\\", "/"));
91-
_log.info("Writing to pipelineConfig.xml: " + line);
9292
}
9393

9494
writer.println(line);
@@ -113,13 +113,13 @@ protected String getProjectName()
113113
@Test
114114
public void BasicRemoteJob() throws Exception
115115
{
116-
File outDir = new File(_pipelineRoot, "clusterBootstrap");
116+
File outDir = FileUtil.appendName(getPipelineRoot(_project), "clusterBootstrap");
117117
if (outDir.exists())
118118
{
119119
FileUtils.deleteDirectory(outDir);
120120
}
121121

122-
outDir.mkdirs();
122+
FileUtil.mkdirs(outDir);
123123

124124
executeJobRemote(outDir, null);
125125

@@ -143,19 +143,19 @@ public void RunBwaRemote() throws Exception
143143
return;
144144

145145
String jobName = "TestBWAMem_" + System.currentTimeMillis();
146-
JSONObject config = substituteParams(new File(_sampleData, ALIGNMENT_JOB), jobName);
146+
JSONObject config = substituteParams(FileUtil.appendName(_sampleData, ALIGNMENT_JOB), jobName);
147147
config.put("alignment", "BWA-Mem");
148148
appendSamplesForAlignment(config, _readsets);
149149

150150
SequenceAlignmentJob job = SequenceAlignmentJob.createForReadsets(_project, _context.getUser(), "RemoteJob1", "Test of remote pipeline", config, config.getJSONArray("readsetIds"), false).get(0);
151-
File outDir = new File(_pipelineRoot, "remoteBwa");
151+
File outDir = FileUtil.appendName(getPipelineRoot(_project), "remoteBwa");
152152
if (outDir.exists())
153153
{
154154
FileUtils.deleteDirectory(outDir);
155155
}
156156

157-
outDir.mkdirs();
158-
job.getLogFile().getParentFile().mkdirs();
157+
FileUtil.mkdirs(outDir);
158+
FileUtil.mkdirs(job.getLogFile().getParentFile());
159159

160160
_readsets.forEach(rs -> job.getSequenceSupport().cacheReadset(rs));
161161

@@ -171,7 +171,7 @@ public void RunBwaRemote() throws Exception
171171
//Now move to remote tasks
172172
job.setActiveTaskId(new TaskId(PrepareAlignerIndexesTask.class));
173173

174-
File jobFile = new File(outDir, "bwaRemote.job.json.txt");
174+
File jobFile = FileUtil.appendName(outDir, "bwaRemote.job.json.txt");
175175
job.writeToFile(jobFile);
176176

177177
executeJobRemote(outDir, jobFile);
@@ -191,7 +191,7 @@ public void RunBwaRemote() throws Exception
191191
writeJobLogToLog(job);
192192

193193
_log.info("Files in job folder: " + job.getLogFile().getParentFile().getPath());
194-
for (File f : job.getLogFile().getParentFile().listFiles())
194+
for (File f : Objects.requireNonNull(job.getLogFile().getParentFile().listFiles()))
195195
{
196196
_log.info(f.getName());
197197
}
@@ -215,14 +215,14 @@ protected void executeJobRemote(File workDir, @Nullable File jobJson) throws IOE
215215
ProcessBuilder pb = new ProcessBuilder(args);
216216
pb.directory(workDir);
217217

218-
_log.info("Executing job in '" + pb.directory().getAbsolutePath() + "': " + String.join(" ", pb.command()));
218+
_log.info("Executing job in '{}': {}", pb.directory().getAbsolutePath(), String.join(" ", pb.command()));
219219

220220
Process proc;
221221
try
222222
{
223223
pb.redirectErrorStream(true);
224224
proc = pb.start();
225-
File logFile = new File(workDir, "clusterBootstrap.txt");
225+
File logFile = FileUtil.appendName(workDir, "clusterBootstrap.txt");
226226
try (BufferedReader procReader = Readers.getReader(proc.getInputStream());PrintWriter writer = PrintWriters.getPrintWriter(logFile))
227227
{
228228
String line;

‎singlecell/resources/chunks/CalculateUCellScores.R‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ for (datasetId in names(seuratObjects)) {
99
seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
1010

1111
message(paste0('Loading dataset ', datasetId, ', with total cells: ', ncol(seuratObj)))
12-
seuratObj <- RIRA::CalculateUCellScores(seuratObj, storeRanks = storeRanks, assayName = assayName, forceRecalculate = forceRecalculate, ncores = nCores)
12+
seuratObj <- RIRA::CalculateUCellScores(seuratObj, storeRanks = storeRanks, assayName = assayName, forceRecalculate = forceRecalculate, ncores = nCores, dropAllExistingUcells = dropAllExistingUcells)
1313

1414
saveData(seuratObj, datasetId)
1515

‎singlecell/resources/chunks/PerformTcrClustering.R‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
for (datasetId in names(seuratObjects)) {
2+
printName(datasetId)
3+
seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
4+
5+
seuratObj <- tcrClustR::CalculateTcrDistances(
6+
inputData = seuratObj,
7+
chains = c('TRA', 'TRB', 'TRG', 'TRD'),
8+
organism = organism,
9+
minimumCloneSize = 2,
10+
calculateChainPairs = TRUE
11+
)
12+
13+
seuratObj <- tcrClustR::RunTcrClustering(
14+
seuratObj_TCR = seuratObj,
15+
dianaHeight = 20,
16+
clusterSizeThreshold = 1
17+
)
18+
19+
print(paste0('Summary of distances: '))
20+
if (!'TCR_Distances' %in% names(seuratObj@misc)) {
21+
warning('No TCR_Distances were found, this could indicate a problem with processing')
22+
} else {
23+
for (an in names(seuratObj@misc$TCR_Distances)) {
24+
ad <- seuratObj@misc$TCR_Distances[[an]]
25+
print(paste0('Assay: ', an, ', total clones: ', nrow(ad)))
26+
}
27+
}
28+
29+
saveData(seuratObj, datasetId)
30+
31+
# Cleanup
32+
rm(seuratObj)
33+
gc()
34+
}

‎singlecell/src/org/labkey/singlecell/SingleCellModule.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.labkey.api.singlecell.CellHashingService;
3030
import org.labkey.api.singlecell.pipeline.SingleCellRawDataStep;
3131
import org.labkey.api.singlecell.pipeline.SingleCellStep;
32-
import org.labkey.api.util.PageFlowUtil;
3332
import org.labkey.api.view.WebPartFactory;
3433
import org.labkey.singlecell.analysis.AbstractSingleCellHandler;
3534
import org.labkey.singlecell.analysis.CellRangerRawDataHandler;
@@ -76,6 +75,7 @@
7675
import org.labkey.singlecell.pipeline.singlecell.NormalizeAndScale;
7776
import org.labkey.singlecell.pipeline.singlecell.PerformDefaultNimbleAppend;
7877
import org.labkey.singlecell.pipeline.singlecell.PerformMhcDimRedux;
78+
import org.labkey.singlecell.pipeline.singlecell.PerformTcrClustering;
7979
import org.labkey.singlecell.pipeline.singlecell.PhenotypePlots;
8080
import org.labkey.singlecell.pipeline.singlecell.PlotAssayFeatures;
8181
import org.labkey.singlecell.pipeline.singlecell.PlotAverageCiteSeqCounts;
@@ -305,6 +305,7 @@ public static void registerPipelineSteps()
305305
SequencePipelineService.get().registerPipelineStep(new CalculateTcrRepertoireStats.Provider());
306306
SequencePipelineService.get().registerPipelineStep(new PredictTcellActivation.Provider());
307307
SequencePipelineService.get().registerPipelineStep(new IdentifyAndStoreActiveClonotypes.Provider());
308+
SequencePipelineService.get().registerPipelineStep(new PerformTcrClustering.Provider());
308309

309310
SequenceAnalysisService.get().registerReadsetListener(new SingleCellReadsetListener());
310311
}

‎singlecell/src/org/labkey/singlecell/analysis/AbstractSingleCellHandler.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -963,6 +963,7 @@ public static String getOutputDescription(JSONObject jsonParams, Logger log, Fil
963963
int lowOrNegative = 0;
964964
int totalDoublet = 0;
965965
double totalSaturation = 0.0;
966+
Set<String> subjectIds = new HashSet<>();
966967

967968
int hashingIdx = -1;
968969
int saturationIdx = -1;
@@ -972,6 +973,7 @@ public static String getOutputDescription(JSONObject jsonParams, Logger log, Fil
972973
int trbIdx = -1;
973974
int trdIdx = -1;
974975
int trgIdx = -1;
976+
int subjectIdIdx = -1;
975977

976978
int totalTNK = 0;
977979
int cellsWithTRA = 0;
@@ -998,6 +1000,7 @@ public static String getOutputDescription(JSONObject jsonParams, Logger log, Fil
9981000
trdIdx = Arrays.asList(line).indexOf("TRD");
9991001
trgIdx = Arrays.asList(line).indexOf("TRG");
10001002
riraIdx = Arrays.asList(line).indexOf("RIRA_Immune_v2.cellclass");
1003+
subjectIdIdx = Arrays.asList(line).indexOf("SubjectIdId");
10011004
}
10021005
else
10031006
{
@@ -1085,6 +1088,15 @@ else if ("NotUsed".equals(val))
10851088
}
10861089
}
10871090
}
1091+
1092+
if (subjectIdIdx > 0)
1093+
{
1094+
String subjectId = StringUtils.trimToNull(line[subjectIdIdx]);
1095+
if (subjectId != null && !"NA".equals(subjectId))
1096+
{
1097+
subjectIds.add(subjectId);
1098+
}
1099+
}
10881100
}
10891101
}
10901102

@@ -1126,6 +1138,11 @@ else if (riraIdx == -1 || traIdx == -1)
11261138
{
11271139
descriptions.add("TCR information not present");
11281140
}
1141+
1142+
if (!subjectIds.isEmpty())
1143+
{
1144+
descriptions.add("Distinct SubjectIds: " + subjectIds.size());
1145+
}
11291146
}
11301147
catch (IOException e)
11311148
{

‎singlecell/src/org/labkey/singlecell/pipeline/singlecell/AbstractTcrClustRStep.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package org.labkey.singlecell.pipeline.singlecell;
2+
3+
import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
4+
import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider;
5+
import org.labkey.api.singlecell.pipeline.AbstractSingleCellPipelineStep;
6+
import org.labkey.api.util.PageFlowUtil;
7+
8+
import java.util.Collection;
9+
10+
abstract public class AbstractTcrClustRStep extends AbstractSingleCellPipelineStep
11+
{
12+
public static String CONTAINER_NAME = "ghcr.io/bimberlabinternal/tcrclustr:latest";
13+
14+
public AbstractTcrClustRStep(PipelineStepProvider<?> provider, PipelineContext ctx)
15+
{
16+
super(provider, ctx);
17+
}
18+
19+
@Override
20+
public Collection<String> getRLibraries()
21+
{
22+
return PageFlowUtil.set("tcrClustR");
23+
}
24+
25+
@Override
26+
public String getDockerContainerName()
27+
{
28+
return CONTAINER_NAME;
29+
}
30+
31+
// NOTE: ExperimentHub and similar packages default to saving data to the user's home dir. Set a directory, to avoid issues when not running the container as root
32+
@Override
33+
public String getDockerHomeDir()
34+
{
35+
return "/dockerHomeDir";
36+
}
37+
}

0 commit comments

Comments
 (0)