Merge discvr-25.3 to develop

elispot_assay/test/src/org/labkey/test/tests/external/labModules/ELISPOT_AssayTest.java

@@ -177,6 +177,10 @@ private void createPlateTemplate()
         String originalID = TEMPLATE_DATA[1][1];
         TEMPLATE_DATA[1][1] = "";
 
+        // Debug client date parsing
+        String clientFormattedString = (String)executeScript("return Ext4.Date.format(LDK.ConvertUtils.parseDate('2012-02-09'), 'Y-m-d');");
+        assertEquals("Incorrect date parsing", clientFormattedString, "2012-02-09");
+
         _helper.addRecordsToAssayTemplate(TEMPLATE_DATA, expectedCols);
 
         waitForElement(_helper.getAssayWell("A1", LabModuleHelper.UNKNOWN_COLOR), WAIT_FOR_PAGE);

mGAP/resources/module.xml

@@ -25,7 +25,7 @@
     <clientDependencies>
         <dependency path="mGAP/mgap.css"/>
         <dependency path="internal/jQuery"/>
-        <dependency path="https://cdn.plot.ly/plotly-latest.min.js"/>
+        <dependency path="https://cdn.plot.ly/plotly-3.0.1.min.js"/>
         <dependency path="mGAP/Utils.js"/>
     </clientDependencies>
 </module>

mGAP/resources/views/admin.html

@@ -18,6 +18,10 @@
                 xtype: 'ldk-linkbutton',
                 text: 'Manage Genomes',
                 href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'sequenceanalysis', queryName: 'reference_libraries'})
+            },{
+                xtype: 'ldk-linkbutton',
+                text: 'Manage Sequence Outputs',
+                href: LABKEY.ActionURL.buildURL('query', 'executeQuery', null, {schemaName: 'sequenceanalysis', queryName: 'outputfiles'})
             },{
                 xtype: 'ldk-linkbutton',
                 text: 'Manage JBrowse Tracks',

mGAP/resources/views/annotation.html

@@ -18,7 +18,7 @@
     })(jQuery);
 </script>
 
-<div style="float: none;">
+<div style="float: none;margin-bottom: 20px;">
     <div style="float: left; border: 10px; padding-right: 20px;width:40%; text-align: justify" >
         Assigning significance to genetics variants is often a challenge, and this is particularly problematic for model organisms like macaques,
         which have few sources of genomic annotations, relative to human or mouse.
@@ -34,7 +34,6 @@
             <li><a href="http://database.liulab.science/dbNSFP#intro">dbNSFP: Functional Predictions and Annotations for Human Non-synonymous and Splice Site SNVs</a></li>
             <li><a href="https://pcingola.github.io/SnpEff/se_introduction/">SnpEff: variant annotation and effect prediction tool</a></li>
             <li><a href="https://github.com/bimberlabinternal/VariantAnnotation/tree/master">VariantAnnotation: a custom Funcotator-compatible set of annotation sources</a></li>
-
         </ul>
     </div>
     <div>

mGAP/resources/views/overview.html

@@ -46,7 +46,7 @@ <h4><span style="text-decoration: underline;">Structural Variant Catalog:</span>
         We have released a draft dataset with structural variants generated from PacBio sequencing of 44 Rhesus macaques. These data complement the short variant catalog by detecting categories of variants not readily accomplished with short read Illumina data.
         <br>
         <ul style="padding-top: 10px;">
-            <li>Use the <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-genomeBrowser.view?activeTracks=mGAP Structural Variants 1.0&location=11:6,936,982..7,002,783">Genome Browser</a> to view and search structural variant data</li>
+            <li>Use the <a class="mgap-link" href="<%=contextPath%><%=containerPath%>/mGap-genomeBrowser.view?nhpSpecies=Rhesus macaque&activeTracks=mGAP Structural Variants 1.0&location=11:6,936,982..7,002,783">Genome Browser</a> to view and search structural variant data</li>
         </ul>
 
         <h4><span style="text-decoration: underline;">NHP Models of Human Disease:</span> <img style="vertical-align: sub" width="40px" src="<%=contextPath%>/mgap/images/phenotypes.png" alt="Phenotypes"></h4>

mGAP/resources/views/overview.view.xml

@@ -5,7 +5,7 @@
     <dependencies>
         <dependency path="ldk.context"/>
         <dependency path="internal/jQuery"/>
-        <dependency path="https://cdn.plot.ly/plotly-latest.min.js"/>
+        <dependency path="https://cdn.plot.ly/plotly-3.0.1.min.js"/>
         <dependency path="https://code.jquery.com/ui/1.13.2/jquery-ui.min.js"/>
         <dependency path="https://code.jquery.com/ui/1.13.2/themes/base/jquery-ui.css"/>
         <dependency path="https://kit.fontawesome.com/d2fb21907d.js"/>

mGAP/resources/views/quickLinks.html

@@ -18,7 +18,7 @@
                     '<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'variantList', null) + '">Predicted Damaging Variants</a></li>' +
                     (jbrowseId ? '<li><a href="' + LABKEY.ActionURL.buildURL('jbrowse', 'browser', null, {database: jbrowseId}) + '">Genome Browser (Macaque)</a></li>' : '') +
                     (humanJbrowseId ? '<li><a href="' + LABKEY.ActionURL.buildURL('jbrowse', 'browser', null, {database: humanJbrowseId}) + '">Genome Browser (Human)</a></li>' : '') +
-                    (releaseHasLuceneIndex ? '<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'genomeBrowser', null, {target: 'variantSearch'}) + '">Full-text Variant Search (BETA)</a></li>' : '') +
+                    (releaseHasLuceneIndex ? '<li><a href="' + LABKEY.ActionURL.buildURL('mgap', 'genomeBrowser', null, {target: 'variantSearch'}) + '">Full-text Variant Search</a></li>' : '') +
                 '</ul>'
             );
         }

mGAP/resources/views/releaseNotes.html

@@ -1,7 +1,7 @@
 <h4>Release 3.0:</h4>
 <ul>
     <li>This release involves a major change in the processing of variants. All prior releases omitted variants in complex or repetitive regions. We originally excluded these because genotypes can be less accurate; however, we made this change because some repetitive regions overlap coding regions and can contain valuable information. This also results in a significant increase in the total number of variants.</a></li>
-    <li>This is the first release to include a second species. The dataset now contains both Rhesus macaques and Japanese macaques (which are separated into a separate track).</li>
+    <li>This is the first release to include a second species. The dataset now contains both Rhesus macaques and Japanese macaques (which are separated into a separate VCF / browser track).</li>
 </ul>
 
 <h4>Release 2.5:</h4>

mGAP/src/org/labkey/mgap/columnTransforms/AbstractVariantTransform.java

@@ -1,6 +1,9 @@
 package org.labkey.mgap.columnTransforms;
 
 import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
+import org.junit.Test;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
 import org.labkey.api.data.Results;
 import org.labkey.api.data.Selector;
@@ -24,7 +27,10 @@
 import org.labkey.mgap.mGAPManager;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.sql.SQLException;
 import java.util.Date;
 import java.util.HashMap;
@@ -112,51 +118,60 @@ protected Integer getOrCreateOutputFile(Object dataFileUrl, Object folderName, S
             File f = new File(uri);
             if (!f.exists())
             {
-                getStatusLogger().error("File not found: " + uri);
-                return null;
-            }
-            else
-            {
-                File subDir = getLocalSubdir(folderName);
-                File localCopy = doFileCopy(f, subDir, name);
-
-                //first create the ExpData
-                ExpData d = ExperimentService.get().getExpDataByURL(localCopy, getContainerUser().getContainer());
-                if (d == null)
-                {
-                    d = ExperimentService.get().createData(getContainerUser().getContainer(), new DataType("Variant Catalog"));
-                    d.setDataFileURI(localCopy.toURI());
-                    d.setName(localCopy.getName());
-                    d.save(getContainerUser().getUser());
-                }
-
-                //then the outputfile
-                TableSelector ts = new TableSelector(getOutputFilesTableInfo(), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("dataId"), d.getRowId()), null);
-                if (ts.exists())
+                if (!EtlQueueManager.get().isFileInQueue(getContainerUser().getContainer(), f))
                 {
-                    getStatusLogger().info("existing record found for outputfile: " + d.getDataFileUrl());
-                    return ts.getObject(Integer.class);
+                    getStatusLogger().error("File not found: " + uri);
+                    return null;
                 }
                 else
                 {
-                    Map<String, Object> row = new CaseInsensitiveHashMap<>();
-                    row.put("category", getOutputFileCategory());
-                    row.put("dataid", d.getRowId());
-                    row.put("name", name == null ? "mGAP Variants, Version: " + getInputValue("version") : name);
-                    row.put("description", getDescription());
-                    row.put("library_id", getLibraryId());
-                    row.put("container", getContainerUser().getContainer().getId());
-                    row.put("created", new Date());
-                    row.put("createdby", getContainerUser().getUser().getUserId());
-                    row.put("modified", new Date());
-                    row.put("modifiedby", getContainerUser().getUser().getUserId());
-
-                    List<Map<String, Object>> rows = getOutputFilesTableInfo().getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(row), new BatchValidationException(), null, new HashMap<>());
-                    getStatusLogger().info("created outputfile: " + rows.get(0).get("rowid"));
-
-                    return (Integer)rows.get(0).get("rowid");
+                    getStatusLogger().debug("File is in ETL queue: " + f.getPath());
                 }
             }
+
+            File subDir = getLocalSubdir(folderName);
+            File localCopy = doFileCopy(f, subDir, name);
+            if (localCopy == null)
+            {
+                // TODO
+            }
+
+            //first create the ExpData
+            ExpData d = ExperimentService.get().getExpDataByURL(localCopy, getContainerUser().getContainer());
+            if (d == null)
+            {
+                d = ExperimentService.get().createData(getContainerUser().getContainer(), new DataType("Variant Catalog"));
+                d.setDataFileURI(localCopy.toURI());
+                d.setName(localCopy.getName());
+                d.save(getContainerUser().getUser());
+            }
+
+            //then the outputfile
+            TableSelector ts = new TableSelector(getOutputFilesTableInfo(), PageFlowUtil.set("rowid"), new SimpleFilter(FieldKey.fromString("dataId"), d.getRowId()), null);
+            if (ts.exists())
+            {
+                getStatusLogger().info("existing record found for outputfile: " + d.getDataFileUrl());
+                return ts.getObject(Integer.class);
+            }
+            else
+            {
+                Map<String, Object> row = new CaseInsensitiveHashMap<>();
+                row.put("category", getOutputFileCategory());
+                row.put("dataid", d.getRowId());
+                row.put("name", name == null ? "mGAP Variants, Version: " + getInputValue("version") : name);
+                row.put("description", getDescription());
+                row.put("library_id", getLibraryId());
+                row.put("container", getContainerUser().getContainer().getId());
+                row.put("created", new Date());
+                row.put("createdby", getContainerUser().getUser().getUserId());
+                row.put("modified", new Date());
+                row.put("modifiedby", getContainerUser().getUser().getUserId());
+
+                List<Map<String, Object>> rows = getOutputFilesTableInfo().getUpdateService().insertRows(getContainerUser().getUser(), getContainerUser().getContainer(), List.of(row), new BatchValidationException(), null, new HashMap<>());
+                getStatusLogger().info("created outputfile: " + rows.get(0).get("rowid"));
+
+                return (Integer)rows.get(0).get("rowid");
+            }
         }
         catch (Exception e)
         {
@@ -190,12 +205,22 @@ protected File getLocalSubdir(Object folderName) throws PipelineJobException
         return subdir;
     }
 
-    protected File doFileCopy(File f, File subdir, String name) throws PipelineJobException
+    protected File doFileCopy(File f, File subdir, @Nullable String name) throws PipelineJobException
     {
-        getStatusLogger().info("preparing to copy file: " + f.getPath());
+        getStatusLogger().info("preparing to copy file: " + f.getPath() + ", with name: " + name);
+        if (f.getName().equals("write.lock"))
+        {
+            return LuceneIndexTransform.doLuceneCopy(f, subdir, name, getStatusLogger(), getContainerUser().getContainer());
+        }
 
         //Copy file locally, plus index if exists:
         File localCopy = new File(subdir, name == null || f.getName().startsWith("mGap.v") ? f.getName() : FileUtil.makeLegalName(name).replaceAll(" ", "_") + ".vcf.gz");
+        if (f.equals(localCopy))
+        {
+            getStatusLogger().debug("Attempting to copy file that is already a child of the target: " + f.getPath(), new Exception());
+            return localCopy;
+        }
+
         boolean doCopy = true;
         if (localCopy.exists())
         {
@@ -206,15 +231,47 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx
             }
             else
             {
-                getStatusLogger().info("source file has been modified, deleting copy and re-syncing");
-                localCopy.delete();
+                getStatusLogger().info("source file has been modified, deleting copy and re-syncing: " + localCopy.getPath());
+                try
+                {
+                    Files.delete(localCopy.toPath());
+                    if (localCopy.exists())
+                    {
+                        throw new PipelineJobException("Unable to delete file: " + localCopy.getPath());
+                    }
+                }
+                catch (IOException e)
+                {
+                    throw new PipelineJobException("Unable to delete file: " + localCopy.getPath(), e);
+                }
             }
         }
+        else
+        {
+            getStatusLogger().info("existing file not found: " + localCopy.getPath());
+        }
 
         if (doCopy)
         {
-            getStatusLogger().info("queueing file copy: " + localCopy.getPath());
-            EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), f, localCopy);
+            getStatusLogger().info("Creating symlink: " + f.getPath() + " / " + localCopy.getPath());
+            try
+            {
+                if (!Files.isReadable(f.toPath()))
+                {
+                    throw new PipelineJobException("Unable to read file: " + f.getPath());
+                }
+
+                if (localCopy.exists())
+                {
+                    throw new PipelineJobException("File should have been deleted: " + localCopy.getPath());
+                }
+
+                Files.createSymbolicLink(localCopy.toPath(), f.toPath());
+            }
+            catch (IOException e)
+            {
+                throw new PipelineJobException("Failed to create symlink: " + localCopy.getPath(), e);
+            }
         }
 
         File index = new File(f.getPath() + ".tbi");
@@ -223,14 +280,25 @@ protected File doFileCopy(File f, File subdir, String name) throws PipelineJobEx
             File indexLocal = new File(localCopy.getPath() + ".tbi");
             if (doCopy && indexLocal.exists())
             {
-                getStatusLogger().info("deleting local copy of index since file was re-copied");
+                getStatusLogger().info("deleting local copy of index since file will be re-copied: " + indexLocal.getPath());
                 indexLocal.delete();
             }
 
             if (!indexLocal.exists())
             {
-                getStatusLogger().info("queueing copy of index: " + indexLocal.getPath());
-                EtlQueueManager.get().queueFileCopy(getContainerUser().getContainer(), index, indexLocal);
+                getStatusLogger().info("Creating symlink copy of VCF index: " + index.getPath() + " / " + indexLocal.getPath());
+                try
+                {
+                    Files.createSymbolicLink(indexLocal.toPath(), index.toPath());
+                }
+                catch (IOException e)
+                {
+                    getStatusLogger().error("Failed to create symlink: " + indexLocal.getPath(), e);
+                }
+            }
+            else
+            {
+                getStatusLogger().info("Local index already exists: " + indexLocal.getPath());
             }
         }