Merge pull request #349 from BimberLab/25.7_fb_merge

bbimber · web-flow · commit d7b3feaee61f · 2025-08-20T08:42:15.000-07:00
Merge discvr-25.3 to discvr-25.7
diff --git a/Studies/resources/queries/studies/studies/.qview.xml b/Studies/resources/queries/studies/studies/.qview.xml
@@ -0,0 +1,13 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView" hidden="false" canOverride="false">
+    <columns>
+        <column name="rowId"/>
+        <column name="studyName"/>
+        <column name="label"/>
+        <column name="category"/>
+        <column name="description"/>
+        <column name="cohorts"/>
+    </columns>
+    <sorts>
+        <sort column="studyName" descending="false"/>
+    </sorts>
+</customView>
diff --git a/Studies/resources/schemas/studies.xml b/Studies/resources/schemas/studies.xml
@@ -132,6 +132,7 @@
         <auditLogging>DETAILED</auditLogging>
         <columns>
             <column columnName="rowid">
+                <columnTitle>Study Id</columnTitle>
                 <isAutoInc>true</isAutoInc>
                 <shownInUpdateView>false</shownInUpdateView>
                 <shownInInsertView>false</shownInInsertView>
diff --git a/Studies/resources/views/studiesOverview.html b/Studies/resources/views/studiesOverview.html
@@ -12,7 +12,7 @@
                 name: 'query',
                 schemaName: 'studies',
                 queryName: 'studies',
-                maxRows: 20
+                maxRows: 10
             }).render(webpart.wrapperDivId + '-studies');
 
             // LABKEY.Query.selectRows({
@@ -62,4 +62,5 @@
 }(jQuery, LABKEY));
 
 </script>
+<div style="padding-bottom: 20px;">Below is a list of the studies tracked in this folder. Please use the <a href="<%=contextPath%>/project/<%=containerPath%>/begin.view?pageId=dataBrowser">Data Browser</a> tab to browse the dataset tracked in this project.</div>
 
diff --git a/Studies/src/org/labkey/studies/query/LookupSetTable.java b/Studies/src/org/labkey/studies/query/LookupSetTable.java
@@ -22,7 +22,6 @@
 import org.labkey.api.ldk.LDKService;
 import org.labkey.api.ldk.table.AbstractDataDefinedTable;
 import org.labkey.api.query.QueryUpdateService;
-import org.labkey.api.query.SimpleUserSchema;
 
 import java.util.Map;
 
@@ -101,15 +100,7 @@ public LookupSetTable init()
     @Override
     public QueryUpdateService getUpdateService()
     {
-        return new EHRLookupsUpdateService(this);
-    }
-
-    protected class EHRLookupsUpdateService extends UpdateService
-    {
-        public EHRLookupsUpdateService(SimpleUserSchema.SimpleTable<StudiesUserSchema> ti)
-        {
-            super(ti);
-        }
+        return new UpdateService(this);
     }
 }
 
diff --git a/Studies/src/org/labkey/studies/query/StudiesUserSchema.java b/Studies/src/org/labkey/studies/query/StudiesUserSchema.java
@@ -1,18 +1,22 @@
 package org.labkey.studies.query;
 
+import com.google.gwt.user.client.ui.TabBar;
 import org.apache.logging.log4j.Logger;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
 import org.labkey.api.collections.CaseInsensitiveTreeSet;
 import org.labkey.api.data.AbstractTableInfo;
 import org.labkey.api.data.Container;
 import org.labkey.api.data.ContainerFilter;
 import org.labkey.api.data.DbSchema;
+import org.labkey.api.data.JdbcType;
+import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.SchemaTableInfo;
 import org.labkey.api.data.SimpleFilter;
 import org.labkey.api.data.TableInfo;
 import org.labkey.api.data.TableSelector;
 import org.labkey.api.ldk.table.ContainerScopedTable;
 import org.labkey.api.ldk.table.CustomPermissionsTable;
+import org.labkey.api.query.ExprColumn;
 import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.QueryDefinition;
 import org.labkey.api.query.QueryException;
@@ -134,7 +138,7 @@ else if (TABLE_LOOKUPS.equalsIgnoreCase(name))
         }
         else if (TABLE_STUDIES.equalsIgnoreCase(name))
         {
-            return createStudyDesignTable(name, cf, false);
+            return createStudiesTable(name, cf, false);
         }
         else if (TABLE_COHORTS.equalsIgnoreCase(name))
         {
@@ -165,7 +169,22 @@ else if (TABLE_EVENT_TYPES.equalsIgnoreCase(name))
         return super.createTable(name, cf);
     }
 
-    private TableInfo createStudyDesignTable(String name, ContainerFilter cf, boolean addTriggers)
+    private TableInfo createStudiesTable(String name, ContainerFilter cf, boolean addTriggers)
+    {
+        CustomPermissionsTable<?> ret = createStudyDesignTable(name, cf, addTriggers);
+
+        final String chr = ret.getSqlDialect().isPostgreSQL() ? "chr" : "char";
+        SQLFragment sql1 = new SQLFragment("(SELECT ").append(ret.getSqlDialect().getGroupConcat(new SQLFragment("c.label"), true, true, new SQLFragment(chr + "(10)"))).append(" as expr FROM " + StudiesSchema.NAME + "." + TABLE_COHORTS + " c WHERE c.studyId = " + ExprColumn.STR_TABLE_ALIAS + ".rowId)");
+        ExprColumn col1 = new ExprColumn(ret, "cohorts", sql1, JdbcType.VARCHAR, ret.getColumn("rowid"));
+        col1.setLabel("Cohort(s)");
+        col1.setDescription("This column lists the cohort labels for this study");
+
+        ret.addColumn(col1);
+
+        return ret;
+    }
+
+    private CustomPermissionsTable<?> createStudyDesignTable(String name, ContainerFilter cf, boolean addTriggers)
     {
         CustomPermissionsTable<SimpleUserSchema> ret = new CustomPermissionsTable<>(this, createSourceTable(name), cf);
         ret.addPermissionMapping(InsertPermission.class, StudiesDataAdminPermission.class);
diff --git a/singlecell/resources/chunks/AppendMetadata.R b/singlecell/resources/chunks/AppendMetadata.R
@@ -4,9 +4,16 @@ if (!file.exists(netRc)) {
     stop(paste0('Unable to find file: ', netRc))
 }
 
-invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
+invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc, timeout = 60, timeout_ms = 60000, connecttimeout = 20, connecttimeout_ms = 20000))
 Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
 
+curlOpt <- curl::curl_options('timeout')
+logger::log_info('Curl options:')
+for (x in names(curlOpt)) {
+    logger::log_info(paste0(x, ': ', curlOpt[x]))
+}
+rm(curlOpt)
+
 for (datasetId in names(seuratObjects)) {
     printName(datasetId)
     seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
diff --git a/singlecell/resources/chunks/ApplyKnownClonotypicData.R b/singlecell/resources/chunks/ApplyKnownClonotypicData.R
@@ -4,14 +4,14 @@ if (!file.exists(netRc)) {
     stop(paste0('Unable to find file: ', netRc))
 }
 
-invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc, connect-timeout = 10))
+invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
 Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
 
 for (datasetId in names(seuratObjects)) {
     printName(datasetId)
     seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
 
-    seuratObj <- ApplyKnownClontypicData(seuratObj, groupFields = groupFields, addMetadata = addMetadata)
+    seuratObj <- ApplyKnownClonotypicData(seuratObj)
     saveData(seuratObj, datasetId)
 
     # Cleanup
diff --git a/singlecell/resources/chunks/CalculateTcrRepertoireStats.R b/singlecell/resources/chunks/CalculateTcrRepertoireStats.R
@@ -0,0 +1,21 @@
+netRc <- paste0(Sys.getenv('USER_HOME'), '/.netrc')
+if (!file.exists(netRc)) {
+  print(list.files(Sys.getenv('USER_HOME')))
+  stop(paste0('Unable to find file: ', netRc))
+}
+
+invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
+Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
+
+for (datasetId in names(seuratObjects)) {
+  printName(datasetId)
+  seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
+
+  outputFile <- gsub(seuratObjects[[datasetId]], pattern = '.rds', replacement = '.tcrStats.txt')
+  df <- Rdiscvr::CalculateAndStoreTcrRepertoireStats(seuratObj, outputFile = outputFile)
+
+  # Cleanup
+  rm(seuratObj)
+  rm(df)
+  gc()
+}
diff --git a/singlecell/resources/chunks/PredictTcellActivation.R b/singlecell/resources/chunks/PredictTcellActivation.R
@@ -0,0 +1,12 @@
+for (datasetId in names(seuratObjects)) {
+  printName(datasetId)
+  seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
+
+  seuratObj <- RIRA::PredictTcellActivation(seuratObj)
+
+  saveData(seuratObj, datasetId)
+
+  # Cleanup
+  rm(seuratObj)
+  gc()
+}
diff --git a/singlecell/resources/chunks/StudyMetadata.R b/singlecell/resources/chunks/StudyMetadata.R
@@ -4,9 +4,16 @@ if (!file.exists(netRc)) {
     stop(paste0('Unable to find file: ', netRc))
 }
 
-invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
+invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc, timeout = 60, timeout_ms = 60000, connecttimeout = 20, connecttimeout_ms = 20000))
 Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
 
+curlOpt <- curl::curl_options('timeout')
+logger::log_info('Curl options:')
+for (x in names(curlOpt)) {
+    logger::log_info(paste0(x, ': ', curlOpt[x]))
+}
+rm(curlOpt)
+
 for (datasetId in names(seuratObjects)) {
     printName(datasetId)
     seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
diff --git a/singlecell/resources/chunks/UpdateSeuratPrototype.R b/singlecell/resources/chunks/UpdateSeuratPrototype.R
@@ -7,6 +7,12 @@ if (!file.exists(netRc)) {
 invisible(Rlabkey::labkey.setCurlOptions(NETRC_FILE = netRc))
 Rdiscvr::SetLabKeyDefaults(baseUrl = serverBaseUrl, defaultFolder = defaultLabKeyFolder)
 
+if (Sys.getenv('SEURAT_MAX_THREADS') != '') {
+  nCores <- Sys.getenv('SEURAT_MAX_THREADS')
+} else {
+  nCores <- 1
+}
+
 for (datasetId in names(seuratObjects)) {
   printName(datasetId)
   seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
@@ -34,6 +40,20 @@ for (datasetId in names(seuratObjects)) {
     seuratObj <- Rdiscvr::ClassifyTNKByExpression(seuratObj)
   }
 
+  if (saveRepertoireStats) {
+    seuratObj <- readSeuratRDS(seuratObjects[[datasetId]])
+    outputFile <- gsub(seuratObjects[[datasetId]], pattern = '.rds', replacement = '.tcrStats.txt')
+    df <- Rdiscvr::CalculateAndStoreTcrRepertoireStats(seuratObj, outputFile = outputFile)
+  }
+
+  if (scoreActivation) {
+    seuratObj <- RIRA::PredictTcellActivation(seuratObj)
+  }
+
+  if (recalculateUCells) {
+    seuratObj <- RIRA::CalculateUCellScores(seuratObj, storeRanks = FALSE, assayName = 'RNA', forceRecalculate = TRUE, ncores = nCores)
+  }
+
   saveData(seuratObj, datasetId)
 
   # Cleanup
diff --git a/singlecell/src/org/labkey/singlecell/SingleCellModule.java b/singlecell/src/org/labkey/singlecell/SingleCellModule.java
@@ -45,6 +45,7 @@
 import org.labkey.singlecell.pipeline.singlecell.ApplyKnownClonotypicData;
 import org.labkey.singlecell.pipeline.singlecell.AvgExpression;
 import org.labkey.singlecell.pipeline.singlecell.CalculateGeneComponentScores;
+import org.labkey.singlecell.pipeline.singlecell.CalculateTcrRepertoireStats;
 import org.labkey.singlecell.pipeline.singlecell.CalculateUCellScores;
 import org.labkey.singlecell.pipeline.singlecell.CellBarcodeFilter;
 import org.labkey.singlecell.pipeline.singlecell.CheckExpectations;
@@ -78,6 +79,7 @@
 import org.labkey.singlecell.pipeline.singlecell.PlotAssayFeatures;
 import org.labkey.singlecell.pipeline.singlecell.PlotAverageCiteSeqCounts;
 import org.labkey.singlecell.pipeline.singlecell.PredictScTour;
+import org.labkey.singlecell.pipeline.singlecell.PredictTcellActivation;
 import org.labkey.singlecell.pipeline.singlecell.PrepareRawCounts;
 import org.labkey.singlecell.pipeline.singlecell.RemoveCellCycle;
 import org.labkey.singlecell.pipeline.singlecell.RunCellHashing;
@@ -301,6 +303,8 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new PerformMhcDimRedux.Provider());
         SequencePipelineService.get().registerPipelineStep(new RunTricycle.Provider());
         SequencePipelineService.get().registerPipelineStep(new ApplyKnownClonotypicData.Provider());
+        SequencePipelineService.get().registerPipelineStep(new CalculateTcrRepertoireStats.Provider());
+        SequencePipelineService.get().registerPipelineStep(new PredictTcellActivation.Provider());
 
         SequenceAnalysisService.get().registerReadsetListener(new SingleCellReadsetListener());
     }
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateTcrRepertoireStats.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/CalculateTcrRepertoireStats.java
@@ -0,0 +1,43 @@
+package org.labkey.singlecell.pipeline.singlecell;
+
+import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStepProvider;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
+import org.labkey.api.singlecell.pipeline.SingleCellStep;
+
+import java.util.Arrays;
+
+public class CalculateTcrRepertoireStats extends AbstractRDiscvrStep
+{
+    public CalculateTcrRepertoireStats(PipelineContext ctx, CalculateTcrRepertoireStats.Provider provider)
+    {
+        super(provider, ctx);
+    }
+
+    public static class Provider extends AbstractPipelineStepProvider<SingleCellStep>
+    {
+        public Provider()
+        {
+            super("CalculateTcrRepertoireStats", "Calculate TCR Repertoire Stats", "CellMembrane/tcrdist3", "This will calculate TCR diversity metrics, and save the results in the database.", Arrays.asList(
+
+            ), null, null);
+        }
+
+        @Override
+        public CalculateTcrRepertoireStats create(PipelineContext ctx)
+        {
+            return new CalculateTcrRepertoireStats(ctx, this);
+        }
+    }
+
+    @Override
+    public boolean createsSeuratObjects()
+    {
+        return false;
+    }
+
+    @Override
+    public String getFileSuffix()
+    {
+        return "ts";
+    }
+}
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/PredictTcellActivation.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/PredictTcellActivation.java
@@ -0,0 +1,37 @@
+package org.labkey.singlecell.pipeline.singlecell;
+
+import org.labkey.api.sequenceanalysis.pipeline.AbstractPipelineStepProvider;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
+import org.labkey.api.singlecell.pipeline.SingleCellStep;
+
+import java.util.List;
+
+public class PredictTcellActivation extends AbstractRDiscvrStep
+{
+    public PredictTcellActivation(PipelineContext ctx, PredictTcellActivation.Provider provider)
+    {
+        super(provider, ctx);
+    }
+
+    public static class Provider extends AbstractPipelineStepProvider<SingleCellStep>
+    {
+        public Provider()
+        {
+            super("PredictTcellActivation", "Predict T cell Activation", "RIRA", "This uses RIRA::PredictTcellActivation to predict TCR-triggered T cells", List.of(), null, null);
+        }
+
+
+        @Override
+        public PredictTcellActivation create(PipelineContext ctx)
+        {
+            return new PredictTcellActivation(ctx, this);
+        }
+    }
+
+    @Override
+    public String getFileSuffix()
+    {
+        return "tca";
+    }
+}
+
diff --git a/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java b/singlecell/src/org/labkey/singlecell/pipeline/singlecell/UpdateSeuratPrototype.java
@@ -48,6 +48,18 @@ public Provider()
                     {{
                         put("checked", false);
                     }}, false),
+                    SeuratToolParameter.create("recalculateUCells", "Recalculate UCells", "If checked, RIRA::CalculateUCellScores will be re-run", "checkbox", new JSONObject()
+                    {{
+                        put("checked", true);
+                    }}, true),
+                    SeuratToolParameter.create("saveRepertoireStats", "Save TCR Repertoire Stats", "If checked, Rdiscvr::CalculateAndStoreTcrRepertoireStats will be run", "checkbox", new JSONObject()
+                    {{
+                        put("checked", true);
+                    }}, true),
+                    SeuratToolParameter.create("scoreActivation", "Score T Cell Activation", "If checked, RIRA::PredictTcellActivation will be run", "checkbox", new JSONObject()
+                    {{
+                        put("checked", true);
+                    }}, true),
                     SeuratToolParameter.create("keepOriginal", "Keep Copy of Original File", "If checked, the original file will be copied with the file extension '.bk'", "checkbox", new JSONObject()
                     {{
                         put("checked", false);
