Merge discvr-25.7 to develop

bbimber · bbimber · commit 4493a79a032f · 2025-08-21T13:50:58.000Z
diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java
@@ -91,6 +91,7 @@ public List<String> getBaseArgs(@Nullable String toolName)
         args.add(SequencePipelineService.get().getJavaFilepath());
         args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
         args.add("-DGATK_STACKTRACE_ON_USER_EXCEPTION=true");
+        args.add("-Dsamjdk.optimistic_vcf_4_4=true");
         args.add("-jar");
         args.add(getJAR().getPath());
 
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/FastqcRunner.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/util/FastqcRunner.java
@@ -17,16 +17,16 @@
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.Nullable;
 import org.junit.Assert;
 import org.junit.Test;
 import org.labkey.api.module.Module;
 import org.labkey.api.module.ModuleLoader;
 import org.labkey.api.pipeline.PipelineJobService;
-import org.labkey.api.resource.FileResource;
 import org.labkey.api.resource.DirectoryResource;
+import org.labkey.api.resource.FileResource;
 import org.labkey.api.resource.Resource;
 import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.settings.AppProps;
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/KingInferenceStep.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/KingInferenceStep.java
@@ -115,6 +115,10 @@ public Output processVariants(File inputVCF, File outputDirectory, ReferenceGeno
         plinkArgs.add("--max-alleles");
         plinkArgs.add("2");
 
+        // NOTE: tools like sawfish can report half-called genotypes, like 0/.. For now, be most conservative in PCA:
+        plinkArgs.add("--vcf-half-call");
+        plinkArgs.add("missing");
+
         Integer threads = SequencePipelineService.get().getMaxThreads(getPipelineCtx().getLogger());
         if (threads != null)
         {
diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/PlinkPcaStep.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/variant/PlinkPcaStep.java
@@ -227,6 +227,10 @@ private void runBatch(File inputVCF, File outputDirectory, VariantProcessingStep
             args.add(String.valueOf(maxRam));
         }
 
+        // NOTE: tools like sawfish can report half-called genotypes, like 0/.. For now, be most conservative in PCA:
+        args.add("--vcf-half-call");
+        args.add("missing");
+
         args.addAll(getClientCommandArgs());
 
         getWrapper().execute(args);
diff --git a/Studies/resources/queries/studies/studies/.qview.xml b/Studies/resources/queries/studies/studies/.qview.xml
@@ -0,0 +1,13 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView" hidden="false" canOverride="false">
+    <columns>
+        <column name="rowId"/>
+        <column name="studyName"/>
+        <column name="label"/>
+        <column name="category"/>
+        <column name="description"/>
+        <column name="cohorts"/>
+    </columns>
+    <sorts>
+        <sort column="studyName" descending="false"/>
+    </sorts>
+</customView>
diff --git a/Studies/resources/schemas/studies.xml b/Studies/resources/schemas/studies.xml
@@ -132,6 +132,7 @@
         <auditLogging>DETAILED</auditLogging>
         <columns>
             <column columnName="rowid">
+                <columnTitle>Study Id</columnTitle>
                 <isAutoInc>true</isAutoInc>
                 <shownInUpdateView>false</shownInUpdateView>
                 <shownInInsertView>false</shownInInsertView>
diff --git a/Studies/resources/views/studiesOverview.html b/Studies/resources/views/studiesOverview.html
@@ -12,7 +12,7 @@
                 name: 'query',
                 schemaName: 'studies',
                 queryName: 'studies',
-                maxRows: 20
+                maxRows: 10
             }).render(webpart.wrapperDivId + '-studies');
 
             // LABKEY.Query.selectRows({
@@ -62,4 +62,5 @@
 }(jQuery, LABKEY));
 
 </script>
+<div style="padding-bottom: 20px;">Below is a list of the studies tracked in this folder. Please use the <a href="<%=contextPath%>/project/<%=containerPath%>/begin.view?pageId=dataBrowser">Data Browser</a> tab to browse the dataset tracked in this project.</div>
 
diff --git a/Studies/src/org/labkey/studies/query/LookupSetTable.java b/Studies/src/org/labkey/studies/query/LookupSetTable.java
@@ -22,7 +22,6 @@
 import org.labkey.api.ldk.LDKService;
 import org.labkey.api.ldk.table.AbstractDataDefinedTable;
 import org.labkey.api.query.QueryUpdateService;
-import org.labkey.api.query.SimpleUserSchema;
 
 import java.util.Map;
 
@@ -101,15 +100,7 @@ public LookupSetTable init()
     @Override
     public QueryUpdateService getUpdateService()
     {
-        return new EHRLookupsUpdateService(this);
-    }
-
-    protected class EHRLookupsUpdateService extends UpdateService
-    {
-        public EHRLookupsUpdateService(SimpleUserSchema.SimpleTable<StudiesUserSchema> ti)
-        {
-            super(ti);
-        }
+        return new UpdateService(this);
     }
 }
 
diff --git a/Studies/src/org/labkey/studies/query/StudiesUserSchema.java b/Studies/src/org/labkey/studies/query/StudiesUserSchema.java
@@ -1,18 +1,22 @@
 package org.labkey.studies.query;
 
+import com.google.gwt.user.client.ui.TabBar;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
 import org.labkey.api.collections.CaseInsensitiveTreeSet;
 import org.labkey.api.data.AbstractTableInfo;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerFilter;
 import org.labkey.api.data.DbSchema;
+import org.labkey.api.data.JdbcType;
+import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.SchemaTableInfo;
 import org.labkey.api.data.SimpleFilter;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.ldk.table.ContainerScopedTable;
 import org.labkey.api.ldk.table.CustomPermissionsTable;
+import org.labkey.api.query.ExprColumn;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.QueryDefinition;
 import org.labkey.api.query.QueryException;
@@ -134,7 +138,7 @@ else if (TABLE_LOOKUPS.equalsIgnoreCase(name))
         }
         else if (TABLE_STUDIES.equalsIgnoreCase(name))
         {
-            return createStudyDesignTable(name, cf, false);
+            return createStudiesTable(name, cf, false);
         }
         else if (TABLE_COHORTS.equalsIgnoreCase(name))
         {
@@ -165,7 +169,22 @@ else if (TABLE_EVENT_TYPES.equalsIgnoreCase(name))
         return super.createTable(name, cf);
     }
 
-    private TableInfo createStudyDesignTable(String name, ContainerFilter cf, boolean addTriggers)
+    private TableInfo createStudiesTable(String name, ContainerFilter cf, boolean addTriggers)
+    {
+        CustomPermissionsTable<?> ret = createStudyDesignTable(name, cf, addTriggers);
+
+        final String chr = ret.getSqlDialect().isPostgreSQL() ? "chr" : "char";
+        SQLFragment sql1 = new SQLFragment("(SELECT ").append(ret.getSqlDialect().getGroupConcat(new SQLFragment("c.label"), true, true, new SQLFragment(chr + "(10)"))).append(" as expr FROM " + StudiesSchema.NAME + "." + TABLE_COHORTS + " c WHERE c.studyId = " + ExprColumn.STR_TABLE_ALIAS + ".rowId)");
+        ExprColumn col1 = new ExprColumn(ret, "cohorts", sql1, JdbcType.VARCHAR, ret.getColumn("rowid"));
+        col1.setLabel("Cohort(s)");
+        col1.setDescription("This column lists the cohort labels for this study");
+
+        ret.addColumn(col1);
+
+        return ret;
+    }
+
+    private CustomPermissionsTable<?> createStudyDesignTable(String name, ContainerFilter cf, boolean addTriggers)
     {
         CustomPermissionsTable<SimpleUserSchema> ret = new CustomPermissionsTable<>(this, createSourceTable(name), cf);
         ret.addPermissionMapping(InsertPermission.class, StudiesDataAdminPermission.class);
diff --git a/singlecell/resources/chunks/AppendMetadata.R b/singlecell/resources/chunks/AppendMetadata.R
@@ -4,9 +4,16 @@ if (!file.exists(netRc)) {
     stop(paste0('Unable to find file: ', netRc))
 }
 
-invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
+invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc, timeout = 60, timeout_ms = 60000, connecttimeout = 20, connecttimeout_ms = 20000))
 Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
 
+curlOpt <- curl::curl_options('timeout')
+logger::log_info('Curl options:')
+for (x in names(curlOpt)) {
+    logger::log_info(paste0(x, ': ', curlOpt[x]))
+}
+rm(curlOpt)
+
 for (datasetId in names(seuratObjects)) {
     printName(datasetId)
     seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
diff --git a/singlecell/resources/chunks/ApplyKnownClonotypicData.R b/singlecell/resources/chunks/ApplyKnownClonotypicData.R
@@ -0,0 +1,20 @@
+netRc <- paste0(Sys.getenv('USER_HOME'), '/.netrc')
+if (!file.exists(netRc)) {
+    print(list.files(Sys.getenv('USER_HOME')))
+    stop(paste0('Unable to find file: ', netRc))
+}
+
+invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
+Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
+
+for (datasetId in names(seuratObjects)) {
+    printName(datasetId)
+    seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
+
+    seuratObj <- ApplyKnownClonotypicData(seuratObj)
+    saveData(seuratObj, datasetId)
+
+    # Cleanup
+    rm(seuratObj)
+    gc()
+}
diff --git a/singlecell/resources/chunks/CalculateTcrRepertoireStats.R b/singlecell/resources/chunks/CalculateTcrRepertoireStats.R
@@ -0,0 +1,21 @@
+netRc <- paste0(Sys.getenv('USER_HOME'), '/.netrc')
+if (!file.exists(netRc)) {
+  print(list.files(Sys.getenv('USER_HOME')))
+  stop(paste0('Unable to find file: ', netRc))
+}
+
+invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
+Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
+
+for (datasetId in names(seuratObjects)) {
+  printName(datasetId)
+  seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
+
+  outputFile <- gsub(seuratObjects[[datasetId]], pattern = '.rds', replacement = '.tcrStats.txt')
+  df <- Rdiscvr::CalculateAndStoreTcrRepertoireStats(seuratObj, outputFile = outputFile)
+
+  # Cleanup
+  rm(seuratObj)
+  rm(df)
+  gc()
+}
diff --git a/singlecell/resources/chunks/PredictTcellActivation.R b/singlecell/resources/chunks/PredictTcellActivation.R
@@ -0,0 +1,12 @@
+for (datasetId in names(seuratObjects)) {
+  printName(datasetId)
+  seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
+
+  seuratObj <- RIRA::PredictTcellActivation(seuratObj)
+
+  saveData(seuratObj, datasetId)
+
+  # Cleanup
+  rm(seuratObj)
+  gc()
+}
diff --git a/singlecell/resources/chunks/StudyMetadata.R b/singlecell/resources/chunks/StudyMetadata.R
@@ -4,9 +4,16 @@ if (!file.exists(netRc)) {
     stop(paste0('Unable to find file: ', netRc))
 }
 
-invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
+invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc, timeout = 60, timeout_ms = 60000, connecttimeout = 20, connecttimeout_ms = 20000))
 Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
 
+curlOpt <- curl::curl_options('timeout')
+logger::log_info('Curl options:')
+for (x in names(curlOpt)) {
+    logger::log_info(paste0(x, ': ', curlOpt[x]))
+}
+rm(curlOpt)
+
 for (datasetId in names(seuratObjects)) {
     printName(datasetId)
     seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
diff --git a/singlecell/resources/chunks/UpdateSeuratPrototype.R b/singlecell/resources/chunks/UpdateSeuratPrototype.R
@@ -7,6 +7,12 @@ if (!file.exists(netRc)) {
 invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
 Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
 
+if (Sys.getenv('SEURAT_MAX_THREADS') != '') {
+  nCores <- Sys.getenv('SEURAT_MAX_THREADS')
+} else {
+  nCores <- 1
+}
+
 for (datasetId in names(seuratObjects)) {
   printName(datasetId)
   seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
@@ -34,6 +40,20 @@ for (datasetId in names(seuratObjects)) {
     seuratObj <- Rdiscvr::ClassifyTNKByExpression(seuratObj)
   }
 
+  if (saveRepertoireStats) {
+    seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
+    outputFile <- gsub(seuratObjects[[datasetId]], pattern = '.rds', replacement = '.tcrStats.txt')
+    df <- Rdiscvr::CalculateAndStoreTcrRepertoireStats(seuratObj, outputFile = outputFile)
+  }
+
+  if (scoreActivation) {
+    seuratObj <- RIRA::PredictTcellActivation(seuratObj)
+  }
+
+  if (recalculateUCells) {
+    seuratObj <- RIRA::CalculateUCellScores(seuratObj, storeRanks = FALSE, assayName = 'RNA', forceRecalculate = TRUE, ncores = nCores)
+  }
+
   saveData(seuratObj, datasetId)
 
   # Cleanup
diff --git a/singlecell/src/org/labkey/singlecell/SingleCellModule.java b/singlecell/src/org/labkey/singlecell/SingleCellModule.java
@@ -42,8 +42,10 @@
 import org.labkey.singlecell.pipeline.singlecell.AppendNimble;
 import org.labkey.singlecell.pipeline.singlecell.AppendSaturation;
 import org.labkey.singlecell.pipeline.singlecell.AppendTcr;
+import org.labkey.singlecell.pipeline.singlecell.ApplyKnownClonotypicData;
 import org.labkey.singlecell.pipeline.singlecell.AvgExpression;
 import org.labkey.singlecell.pipeline.singlecell.CalculateGeneComponentScores;
+import org.labkey.singlecell.pipeline.singlecell.CalculateTcrRepertoireStats;
 import org.labkey.singlecell.pipeline.singlecell.CalculateUCellScores;
 import org.labkey.singlecell.pipeline.singlecell.CellBarcodeFilter;
 import org.labkey.singlecell.pipeline.singlecell.CheckExpectations;
@@ -77,6 +79,7 @@
 import org.labkey.singlecell.pipeline.singlecell.PlotAssayFeatures;
 import org.labkey.singlecell.pipeline.singlecell.PlotAverageCiteSeqCounts;
 import org.labkey.singlecell.pipeline.singlecell.PredictScTour;
+import org.labkey.singlecell.pipeline.singlecell.PredictTcellActivation;
 import org.labkey.singlecell.pipeline.singlecell.PrepareRawCounts;
 import org.labkey.singlecell.pipeline.singlecell.RemoveCellCycle;
 import org.labkey.singlecell.pipeline.singlecell.RunCellHashing;
@@ -299,6 +302,9 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new PerformDefaultNimbleAppend.Provider());
         SequencePipelineService.get().registerPipelineStep(new PerformMhcDimRedux.Provider());
         SequencePipelineService.get().registerPipelineStep(new RunTricycle.Provider());
+        SequencePipelineService.get().registerPipelineStep(new ApplyKnownClonotypicData.Provider());
+        SequencePipelineService.get().registerPipelineStep(new CalculateTcrRepertoireStats.Provider());
+        SequencePipelineService.get().registerPipelineStep(new PredictTcellActivation.Provider());
 
         SequenceAnalysisService.get().registerReadsetListener(new SingleCellReadsetListener());
     }
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/ApplyKnownClonotypicData.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/ApplyKnownClonotypicData.java
@@ -0,0 +1,39 @@
+package org.labkey.singlecell.pipeline.singlecell;
+
+import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStepProvider;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
+import org.labkey.api.singlecell.pipeline.SingleCellStep;
+
+import java.util.List;
+
+public class ApplyKnownClonotypicData extends AbstractRDiscvrStep
+{
+    public ApplyKnownClonotypicData(PipelineContext ctx, ApplyKnownClonotypicData.Provider provider)
+    {
+        super(provider, ctx);
+    }
+
+    public static class Provider extends AbstractPipelineStepProvider<SingleCellStep>
+    {
+        public Provider()
+        {
+            super("ApplyKnownClonotypicData", "Append Known Clonotype/Antigen Data", "RDiscvr", "This will query the clone_responses table and append a column tagging each cell for matching antigens (based on clonotype)", List.of(
+
+            ), null, null);
+        }
+
+
+        @Override
+        public ApplyKnownClonotypicData create(PipelineContext ctx)
+        {
+            return new ApplyKnownClonotypicData(ctx, this);
+        }
+    }
+
+    @Override
+    public String getFileSuffix()
+    {
+        return "ctd";
+    }
+}
+
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateTcrRepertoireStats.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateTcrRepertoireStats.java
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/PredictTcellActivation.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/PredictTcellActivation.java
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java

Original file line number	Diff line number	Diff line change
`@@ -227,6 +227,10 @@ private void runBatch(File inputVCF, File outputDirectory, VariantProcessingStep`
`227`	`227`	`args.add(String.valueOf(maxRam));`
`228`	`228`	`}`
`229`	`229`
	`230`	`+ // NOTE: tools like sawfish can report half-called genotypes, like 0/.. For now, be most conservative in PCA:`
	`231`	`+ args.add("--vcf-half-call");`
	`232`	`+ args.add("missing");`
	`233`	`+`
`230`	`234`	`args.addAll(getClientCommandArgs());`
`231`	`235`
`232`	`236`	`getWrapper().execute(args);`