Add action to allow external server to trigger re-import of genetics data

GeneticsCore/build.gradle

@@ -6,6 +6,8 @@ plugins {
 
 dependencies {
     BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: "apiJarFile")
+    BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: "apiJarFile")
+
     BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: 'published', depExtension: 'module')
     BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:LabDevKitModules:LDK", depProjectConfig: 'published', depExtension: 'module')
     BuildUtils.addLabKeyDependency(project: project, config: "implementation", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "apiJarFile")

GeneticsCore/resources/etls/KinshipDataImport.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml">
+    <name>KinshipDataImport</name>
+    <description>Import PRIMe-seq Kinship Data</description>
+    <transforms>
+        <transform id="kinship" type="TaskRefTransformStep">
+            <taskref ref="org.labkey.GeneticsCore.etl.ImportGeneticsCalculationsStep">
+                <settings>
+
+                </settings>
+            </taskref>
+        </transform>
+    </transforms>
+</etl>

GeneticsCore/resources/module.xml

@@ -21,6 +21,13 @@
             </editPermissions>
             <description>This is the containerPath to the folder holding the primary data for Parentage.  Use of slashes is very important - it should be in the format '/myProject/folder'</description>
         </propertyDescriptor>
+        <propertyDescriptor name="KinshipDataPath">
+            <canSetPerContainer>true</canSetPerContainer>
+            <editPermissions>
+                <permission>ADMIN</permission>
+            </editPermissions>
+            <description>This is the filepath of a folder where externally kinship/inbreeding TSVs calculated externally will be deposited. It is used by the KinshipData ETL to initiate import.</description>
+        </propertyDescriptor>
     </properties>
     <clientDependencies>
         <dependency path="clientapi/ext4.lib.xml"/>

GeneticsCore/src/org/labkey/GeneticsCore/GeneticsCoreController.java

@@ -4,10 +4,15 @@
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.labkey.GeneticsCore.mhc.MhcTaskRef;
+import org.labkey.api.action.ApiResponse;
+import org.labkey.api.action.ApiSimpleResponse;
 import org.labkey.api.action.ConfirmAction;
+import org.labkey.api.action.MutatingApiAction;
 import org.labkey.api.action.SpringActionController;
+import org.labkey.api.di.DataIntegrationService;
 import org.labkey.api.security.RequiresPermission;
 import org.labkey.api.security.permissions.AdminPermission;
+import org.labkey.api.security.permissions.UpdatePermission;
 import org.labkey.api.util.HtmlString;
 import org.labkey.api.util.URLHelper;
 import org.labkey.api.view.HtmlView;
@@ -62,4 +67,26 @@ public URLHelper getSuccessURL(Object o)
             return getContainer().getStartURL(getUser());
         }
     }
+
+
+    @RequiresPermission(UpdatePermission.class)
+    public static class ImportGeneticsDataAction extends MutatingApiAction<Object>
+    {
+        @Override
+        public ApiResponse execute(Object form, BindException errors)
+        {
+            try
+            {
+                DataIntegrationService.get().runTransformNow(getContainer(), getUser(), "{GeneticsCore}/KinshipDataImport");
+
+                return new ApiSimpleResponse("success", true);
+            }
+            catch (Exception e)
+            {
+                _log.error("Unable to initiate genetics data import", e);
+                errors.reject(ERROR_MSG, "Unable to initiate genetics data import");
+                return null;
+            }
+        }
+    }
 }

GeneticsCore/src/org/labkey/GeneticsCore/etl/ImportGeneticsCalculationsStep.java

@@ -0,0 +1,107 @@
+package org.labkey.GeneticsCore.etl;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.xmlbeans.XmlException;
+import org.jetbrains.annotations.NotNull;
+import org.labkey.GeneticsCore.GeneticsCoreModule;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.ContainerManager;
+import org.labkey.api.di.TaskRefTask;
+import org.labkey.api.ehr.EHRService;
+import org.labkey.api.module.Module;
+import org.labkey.api.module.ModuleLoader;
+import org.labkey.api.module.ModuleProperty;
+import org.labkey.api.pipeline.PipelineJob;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.pipeline.RecordedActionSet;
+import org.labkey.api.security.permissions.UpdatePermission;
+import org.labkey.api.view.UnauthorizedException;
+import org.labkey.api.writer.ContainerUser;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class ImportGeneticsCalculationsStep implements TaskRefTask
+{
+    protected ContainerUser _containerUser;
+
+    @Override
+    public RecordedActionSet run(@NotNull PipelineJob job) throws PipelineJobException
+    {
+        Module ehr = ModuleLoader.getInstance().getModule("ehr");
+        Module geneticsCore = ModuleLoader.getInstance().getModule(GeneticsCoreModule.class);
+
+        ModuleProperty mp = ehr.getModuleProperties().get("EHRStudyContainer");
+        String ehrContainerPath = StringUtils.trimToNull(mp.getEffectiveValue(_containerUser.getContainer()));
+        if (ehrContainerPath == null)
+        {
+            throw new PipelineJobException("EHRStudyContainer has not been set");
+        }
+
+        Container ehrContainer = ContainerManager.getForPath(ehrContainerPath);
+        if (ehrContainer == null)
+        {
+            throw new PipelineJobException("Invalid container: " + ehrContainerPath);
+        }
+
+        if (!_containerUser.getContainer().equals(ehrContainer))
+        {
+            throw new PipelineJobException("This ETL can only be run from the EHRStudyContainer");
+        }
+
+        // Downstream import events will get additional permissions checks
+        if (!ehrContainer.hasPermission(_containerUser.getUser(), UpdatePermission.class))
+        {
+            throw new UnauthorizedException();
+        }
+
+        ModuleProperty mp2 = geneticsCore.getModuleProperties().get("KinshipDataPath");
+        String pipeDirPath = StringUtils.trimToNull(mp2.getEffectiveValue(ehrContainer));
+        if (pipeDirPath == null)
+        {
+            throw new PipelineJobException("Must provide the filepath to import data using the KinshipDataPath module property");
+        }
+
+        File pipeDir = new File(pipeDirPath);
+        if (!pipeDir.exists())
+        {
+            throw new PipelineJobException("Path does not exist: " + pipeDir.getPath());
+        }
+
+        File kinship = new File(pipeDir, "kinship.txt");
+        if (!kinship.exists())
+        {
+            throw new PipelineJobException("File does not exist: " + kinship.getPath());
+        }
+
+        File inbreeding = new File(pipeDir, "inbreeding.txt");
+        if (!inbreeding.exists())
+        {
+            throw new PipelineJobException("File does not exist: " + inbreeding.getPath());
+        }
+
+        EHRService.get().standaloneProcessKinshipAndInbreeding(ehrContainer, _containerUser.getUser(), pipeDir, job.getLogger());
+
+        return new RecordedActionSet();
+    }
+
+    @Override
+    public List<String> getRequiredSettings()
+    {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public void setSettings(Map<String, String> settings) throws XmlException
+    {
+
+    }
+
+    @Override
+    public void setContainerUser(ContainerUser containerUser)
+    {
+        _containerUser = containerUser;
+    }
+}

onprc_ehr/resources/queries/onprc_ehr/vet_assignment.js

@@ -1,6 +1,9 @@
-/*created by kolli 2020-11-16*/
-
-
 require("ehr/triggers").initScript(this);
 
+var triggerHelper = new org.labkey.onprc_ehr.query.ONPRC_EHRTriggerHelper(LABKEY.Security.currentUser.id, LABKEY.Security.currentContainer.id);
 
+EHR.Server.TriggerManager.registerHandlerForQuery(EHR.Server.TriggerManager.Events.COMPLETE, 'onprc_ehr', 'vet_assignment', function(event, helper){
+    // NOTE: the rules behind vet assignment are complicated enough that any change to one row here could
+    // impact a lot of records. Therefore, just redo everything in the cache
+    triggerHelper.recalculateAllVetAssignmentRecords()
+});

onprc_ehr/resources/queries/study/CageMateInfant.sql

@@ -14,25 +14,20 @@
  * limitations under the License.
  */
 
+SELECT
 
+    h1.Id,
+    group_concat(h2.Id) as InfantCageMate
 
-select
-
-a.id,
-group_concat(e.Id ) as InfantCageMate,
-a.QCState
-
-
- from study.housing a, study.demographics e
-where  a.Id <> e.Id
-And e.calculated_status.code = 'Alive'
-and a.Enddate  is null
-and a.qcstate = 18 and e.qcstate = 18
-and a.housingType.value = 'Cage Location'
-and (a.room.room = e.Id.curLocation.room and a.cage = e.Id.curLocation.cage)
-and e.Id.age.ageInyears < 1
-
-
-Group by a.id, a.QCState
+FROM study.demographicsCurrentLocation h1
+JOIN study.demographicsCurrentLocation h2 ON (
+    h1.room = h2.room AND
+    h1.cage = h2.cage AND
+    h1.Id != h2.Id
+)
 
+WHERE
+        h1.room.housingType.value = 'Cage Location' AND
+        h2.Id.age.ageInyears < 1
 
+GROUP BY h1.Id

onprc_ehr/src/org/labkey/onprc_ehr/demographics/ActiveDrugsGivenDemographicsProvider.java

@@ -38,6 +38,7 @@ public ActiveDrugsGivenDemographicsProvider(Module module)
         _supportsQCState = false;
     }
 
+    @Override
     protected Set<FieldKey> getFieldKeys()
     {
         Set<FieldKey> keys = new HashSet<FieldKey>();
@@ -59,7 +60,16 @@ protected Set<FieldKey> getFieldKeys()
     @Override
     public boolean requiresRecalc(String schema, String query)
     {
-        return ("study".equalsIgnoreCase(schema) && ("drug".equalsIgnoreCase(query) || "treatment_order".equalsIgnoreCase(query))) ;
+        return ("study".equalsIgnoreCase(schema) && ("drug".equalsIgnoreCase(query) || "treatment_order".equalsIgnoreCase(query)));
+    }
+
+    @Override
+    public Collection<String> getKeysToTest()
+    {
+        //for now, simply skip the whole provider.  because different records can be active from day to day, this makes validation tricky
+        Set<String> keys = new HashSet<>(super.getKeysToTest());
+        keys.remove(_propName);
 
+        return keys;
     }
 }

onprc_ehr/src/org/labkey/onprc_ehr/query/ONPRC_EHRTriggerHelper.java

@@ -2609,4 +2609,8 @@ public void exec(ResultSet object) throws SQLException
         }
     }
 
+    public void recalculateAllVetAssignmentRecords()
+    {
+        EHRDemographicsService.get().recalculateForAllIdsInCache(_container, "onprc_ehr", "vet_assignment", true);
+    }
 }

onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest.java

@@ -1249,6 +1249,7 @@ public void testGeneticsPipeline() throws Exception
         waitAndClickAndWait(Locators.bodyPanel().append(Locator.tagContainingText("a", "EHR Admin Page")));
         waitAndClickAndWait(Locator.tagContainingText("a", "Genetics Calculations"));
         _ext4Helper.checkCheckbox(Ext4Helper.Locators.checkbox(this, "Kinship validation?:"));
+        _ext4Helper.checkCheckbox(Ext4Helper.Locators.checkbox(this, "Allow Import During Business Hours?:"));
         Locator loc = Locator.inputByIdContaining("numberfield");
         waitForElement(loc);
         setFormElement(loc, "23");

onprc_ehr/test/src/org/labkey/test/tests/onprc_ehr/ONPRC_EHRTest2.java

@@ -1193,7 +1193,9 @@ private void updateTotalVolume(Integer quantity)
 
     private void addProjectToTheRow(Ext4GridRef gridRef, int index, String project)
     {
-        gridRef.clickDownArrowOnGrid(index, "project");
+        gridRef.startEditing(index, "project");
+        Ext4FieldRef fieldRef = gridRef.getActiveEditor(index, "project");
+        fieldRef.eval("expand()");
         waitAndClick(Locator.tag("li").append(Locator.tagContainingText("span", "Other")));
         waitForElement(Ext4Helper.Locators.window("Choose Project"));
         _ext4Helper.queryOne("window[title=Choose Project] [fieldLabel='Project']", Ext4ComboRef.class).setComboByDisplayValue(project);