Support full data file paths in Signal Data metadata

signalData/resources/assay/signalData/domains/result.xml

@@ -7,7 +7,6 @@
     </exp:PropertyDescriptor>
     <exp:PropertyDescriptor>
         <exp:Name>DataFile</exp:Name>
-        <exp:Required>true</exp:Required>
         <exp:RangeURI>http://cpas.fhcrc.org/exp/xml#fileLink</exp:RangeURI>
         <exp:Label>Data File</exp:Label>
     </exp:PropertyDescriptor>

signalData/resources/schemas/dbscripts/postgresql/signaldata-25.000-25.001.sql

@@ -0,0 +1 @@
+SELECT core.executeJavaUpgradeCode('updateDataFileField');

signalData/resources/views/signalDataUpload.html

@@ -1,7 +1,7 @@
     <p>Signal Data Files</p>
     <ol>
         <li>
-            Import (or paste) the results file. Data must include a column that has the signal data filename (case-sensitive).
+            Import (or paste) the results file. Data must include a column that has the signal data filename.
             <br/><a class="signaldata-upload-templatelink labkey-text-link">Download Spreadsheet Template</a><br/>
         </li>
         <li>

signalData/resources/web/signaldata/QCView/DataService.js

@@ -247,6 +247,9 @@ Ext4.define('LABKEY.SignalData.DataService', {
                                     name = name[0];
                                     var filePath = "";
                                     var dataFile = run.dataRows[r]['DataFile'];
+                                    // dataFile values can be empty, indicating no file was uploaded for the results row, ignore these.
+                                    if (!dataFile)
+                                        continue;
 
                                     var osDelimiter = '/';
                                     var fileName = dataFile.split(osDelimiter).pop();  //Hack to make fileLink and pipe resolve file

signalData/resources/web/signaldata/UploadView/UploadLog.js

@@ -16,6 +16,7 @@ Ext4.define('LABKEY.SignalData.UploadLog', {
     DATA_FILE: 'DataFile',
     FILE_URL: 'DataFileURL',
     FILENAME: 'FileName',
+
     constructor: function (config) {
         if (!Ext4.ModelManager.isRegistered(this.modelClass)) {
             Ext4.define(this.modelClass, {
@@ -54,13 +55,22 @@ Ext4.define('LABKEY.SignalData.UploadLog', {
 
     getFields: function (resultFields) {
         if (!this.fields) {
+            // issue 52421 data file metadata can contain full paths
+            var fileNameFromPath = function(v, rec) {
+                if (v.indexOf('/') > -1)
+                    return v.substring(v.lastIndexOf('/') + 1, v.length);
+
+                return v.substring(v.lastIndexOf('\\') + 1, v.length);
+            };
+
             var fields = [];
             resultFields.forEach(function (field) {
                 fields.push({
                     name: field.name,
-                    type: 'string'
+                    type: 'string',
+                    convert: field.fieldKey === this.DATA_FILE ? fileNameFromPath : null
                 });
-            });
+            }, this);
 
             this.fields = fields.concat([
                 {name: this.FILE_URL, type: 'string'},
@@ -158,7 +168,10 @@ Ext4.define('LABKEY.SignalData.UploadLog', {
                         var fileName = row.get(this.DATA_FILE);
                         var me = this;
 
-                        //Delete File
+                        // don't delete if no file has been uploaded
+                        if (!row.get(this.UPLOAD_TIME))
+                            return;
+
                         this.fileSystem.deletePath({
                             path: this.fileSystem.concatPaths(this.getFullWorkingPath(), fileName),
                             isFile: true,

signalData/resources/web/signaldata/UploadView/uploadResultDataForm.js

@@ -140,6 +140,7 @@ LABKEY.SignalData.initializeUploadForm = function(metadataFormElId, metadataFile
             }],
             submit:function(){
                 LABKEY.Ajax.request({
+                    method: 'POST',
                     url: LABKEY.ActionURL.buildURL("assay", "assayFileUpload", LABKEY.ActionURL.getContainer()),
                     params: { protocolId: assay.id, fileName: metadataTSVId + '.txt', fileContent: this.getTsvInput() },
                     success: function(response) {

signalData/resources/web/signaldata/UploadView/uploadResultFiles.js

@@ -376,17 +376,20 @@ LABKEY.SignalData.initializeDataFileUploadForm = function (metadataFormId, eleme
 
         var dataRows = [];
         var dataInputs = [];
-
         var rows = uploadLog.getStore().getRange();
-
         var runFolder = getRunFolderName();
+
         rows.forEach(function (row){
             var dataRow = {};
             row.fields.eachKey(function(key){
-                dataRow[key] = row.get(key);
+
+                // if there was no actual file uploaded, ignore the dataFile field value so we don't
+                // try to validate on the server
+                if (key !== uploadLog.DATA_FILE || row.get('file'))
+                    dataRow[key] = row.get(key);
             });
 
-            if(row.get('file')) {
+            if (row.get('file')) {
                 dataRow[uploadLog.DATA_FILE] = decodeURI(dataRow[uploadLog.FILE_URL]).replace('file:','');
 
                 dataInputs.push({

signalData/src/org/labkey/signaldata/SignalDataController.java

@@ -39,6 +39,7 @@
 import org.labkey.api.webdav.WebdavResource;
 import org.labkey.api.webdav.WebdavService;
 import org.labkey.signaldata.assay.SignalDataAssayDataHandler;
+import org.labkey.vfs.FileLike;
 import org.springframework.validation.BindException;
 
 import java.io.File;
@@ -83,11 +84,10 @@ public ApiResponse execute(Object form, BindException errors) throws Exception
             {
                 containerPath = root.getContainer().getPath();
                 webdavURL = root.getWebdavURL();
-                if (!SignalDataAssayDataHandler.NAMESPACE.isEmpty())
-                    webdavURL = webdavURL.resolve(SignalDataAssayDataHandler.NAMESPACE);
+                webdavURL = webdavURL.resolve(SignalDataAssayDataHandler.NAMESPACE);
 
                 //Create folder if needed
-                File sdFileRoot = new File(root.getRootPath(), SignalDataAssayDataHandler.NAMESPACE);
+                FileLike sdFileRoot = root.getRootFileLike().resolveChild(SignalDataAssayDataHandler.NAMESPACE);
                 if(!sdFileRoot.exists())
                     sdFileRoot.mkdirs();
             }

signalData/src/org/labkey/signaldata/SignalDataModule.java

@@ -17,16 +17,19 @@
 package org.labkey.signaldata;
 
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.Container;
-import org.labkey.api.module.CodeOnlyModule;
+import org.labkey.api.data.UpgradeCode;
+import org.labkey.api.module.DefaultModule;
 import org.labkey.api.module.ModuleContext;
 import org.labkey.api.module.ModuleProperty;
 import org.labkey.api.view.WebPartFactory;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
-public class SignalDataModule extends CodeOnlyModule
+public class SignalDataModule extends DefaultModule
 {
     public static final String NAME = "SignalData";
     public static final String QC_PROVIDER_PROPERTY_NAME = "QCViewProviderModule";
@@ -67,6 +70,30 @@ public void doStartup(ModuleContext moduleContext)
     {
     }
 
+    @Override
+    public @Nullable Double getSchemaVersion()
+    {
+        return 25.001;
+    }
+
+    @Override
+    public boolean hasScripts()
+    {
+        return true;
+    }
+
+    @Override
+    public @NotNull Collection<String> getSchemaNames()
+    {
+        return List.of("signaldata");
+    }
+
+    @Override
+    public @Nullable UpgradeCode getUpgradeCode()
+    {
+        return new SignalDataUpgradeCode();
+    }
+
     @Override
     @NotNull
     public Collection<String> getSummary(Container c)

signalData/src/org/labkey/signaldata/SignalDataUpgradeCode.java

@@ -0,0 +1,53 @@
+package org.labkey.signaldata;
+
+import org.apache.logging.log4j.Logger;
+import org.labkey.api.assay.AssayProvider;
+import org.labkey.api.assay.AssayService;
+import org.labkey.api.data.Container;
+import org.labkey.api.data.ContainerManager;
+import org.labkey.api.data.UpgradeCode;
+import org.labkey.api.exp.api.ExpProtocol;
+import org.labkey.api.exp.api.ExperimentService;
+import org.labkey.api.exp.property.Domain;
+import org.labkey.api.exp.property.DomainProperty;
+import org.labkey.api.module.Module;
+import org.labkey.api.module.ModuleContext;
+import org.labkey.api.module.ModuleLoader;
+import org.labkey.api.security.User;
+import org.labkey.api.util.logging.LogHelper;
+
+public class SignalDataUpgradeCode implements UpgradeCode
+{
+    private static final Logger LOG = LogHelper.getLogger(SignalDataUpgradeCode.class, "SignalData upgrade code");
+
+    /**
+     * Called from signaldata-25.000-25.001.sql
+     * Updates SignalData assay protocols to make the result domain DataFile field not required.
+     */
+    @SuppressWarnings({"UnusedDeclaration"})
+    public static void updateDataFileField(ModuleContext ctx) throws Exception
+    {
+        Module module = ModuleLoader.getInstance().getModule(SignalDataModule.NAME);
+        if (module != null)
+        {
+            for (Container c : ContainerManager.getAllChildrenWithModule(ContainerManager.getRoot(), module))
+            {
+                for (ExpProtocol protocol : ExperimentService.get().getExpProtocols(c))
+                {
+                    AssayProvider provider = AssayService.get().getProvider(protocol);
+                    if (provider != null && provider.getName().equalsIgnoreCase("Signal Data"))
+                    {
+                        Domain domain = provider.getResultsDomain(protocol, true);
+                        DomainProperty dataFile = domain.getPropertyByName("DataFile");
+                        if (dataFile != null && dataFile.isRequired())
+                        {
+                            LOG.info(String.format("Updating Signal Data assay in folder '%s'", c.getPath()));
+                            dataFile.setRequired(false);
+                            domain.save(User.getAdminServiceUser());
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

signalData/test/sampledata/signaldata/RunsMetadata/datafiles.tsv

@@ -1,4 +1,4 @@
-Name	DataFile
-LGC12392.TXT	LGC12392.TXT
-LGC14332.TXT	LGC14332.TXT
-MPP82113.TXT	MPP82113.TXT
+Name	DataFile	StringValue	IntegerValue
+LGC12392.TXT	LGC12392.TXT	StringOne	1
+LGC14332.TXT	LGC14332.TXT	StringTwo	2
+MPP82113.TXT	MPP82113.TXT	StringThree	3

signalData/test/sampledata/signaldata/RunsMetadata/datafiles2.tsv

@@ -0,0 +1,5 @@
+Name	DataFile
+TD789-12.TXT	/Users/testuser/Downloads/td789-12.txt
+TD789-25.TXT	/Users/testuser/Downloads/TD789-25.txt
+QD123-11.TXT	c:\\home\\windows\\QD123-11.TXT
+QuotedPath.TXT	"c:\user\windows\qd123-24.txt"

signalData/test/src/org/labkey/test/pages/signaldata/SignalDataAssayBeginPage.java

@@ -54,7 +54,7 @@ public void waitForGridValue(String value, int qty)
     }
 
     @NotNull
-    private DataRegionTable getDataRegionTable()
+    public DataRegionTable getDataRegionTable()
     {
         return new DataRegionTable("aqwp101", _test.getDriver());
     }

signalData/test/src/org/labkey/test/pages/signaldata/SignalDataUploadPage.java

@@ -25,7 +25,7 @@
 import org.openqa.selenium.support.ui.ExpectedConditions;
 
 import java.io.File;
-import java.util.Arrays;
+import java.util.List;
 
 import static org.labkey.test.WebDriverWrapper.WAIT_FOR_JAVASCRIPT;
 import static org.labkey.test.WebDriverWrapper.sleep;
@@ -50,11 +50,11 @@ public void uploadMetadataFile(File file)
         _test.waitForElement(Locators.runIdentifier);
     }
 
-    public void uploadFile(File... file)
+    public void uploadFile(List<File> files)
     {
         sleep(1_000);
         WebElement dropFileInputEl = Locators.dropFileInput.findElement(_test.getDriver());
-        _test.setInput(dropFileInputEl, Arrays.asList(file));
+        _test.setInput(dropFileInputEl, files);
     }
 
     public void waitForProgressBars(int count)
@@ -68,7 +68,7 @@ public void waitForProgressBars(int count)
 
     public void uploadIncorrectFile(File file)
     {
-        uploadFile(file);
+        uploadFile(List.of(file));
         WebElement msgBox = Locators.fileNotUploadedMsgBox(file.getName()).waitForElement(_test.getDriver(), WAIT_FOR_JAVASCRIPT);
         msgBox.sendKeys(Keys.ESCAPE);
         _test.shortWait().until(ExpectedConditions.invisibilityOf(msgBox));