Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 19 additions & 30 deletions api/src/org/labkey/api/assay/AbstractAssayProvider.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,16 @@
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.json.JSONArray;
import org.labkey.api.assay.actions.AssayRunUploadForm;
import org.labkey.api.assay.actions.DesignerAction;
import org.labkey.api.assay.actions.UploadWizardAction;
import org.labkey.api.assay.pipeline.AssayRunAsyncContext;
import org.labkey.api.assay.plate.FilterCriteria;
import org.labkey.api.assay.security.DesignAssayPermission;
import org.labkey.api.assay.transform.AnalysisScript;
import org.labkey.api.assay.transform.DataExchangeHandler;
import org.labkey.api.assay.transform.DataTransformService;
import org.labkey.api.audit.AuditLogService;
import org.labkey.api.data.ActionButton;
import org.labkey.api.data.ButtonBar;
Expand Down Expand Up @@ -79,7 +83,6 @@
import org.labkey.api.module.Module;
import org.labkey.api.pipeline.PipeRoot;
import org.labkey.api.pipeline.PipelineService;
import org.labkey.api.qc.DataExchangeHandler;
import org.labkey.api.query.FieldKey;
import org.labkey.api.query.FilteredTable;
import org.labkey.api.query.QueryService;
Expand Down Expand Up @@ -1243,16 +1246,15 @@ public boolean hasUsefulDetailsPage()
private static final String SCRIPT_PATH_DELIMITER = "|";

@Override
public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List<File> scripts) throws ExperimentException
public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List<AnalysisScript> scripts) throws ExperimentException
{
Map<String, ObjectProperty> props = new HashMap<>(protocol.getObjectProperties());
String propertyURI = ScriptType.TRANSFORM.getPropertyURI(protocol);

ValidationException validationErrors = new ValidationException();
StringBuilder sb = new StringBuilder();
String separator = "";
for (File scriptFile : scripts)

for (AnalysisScript script : scripts)
{
File scriptFile = script.getScript().toNioPathForRead().toFile();
String ext = FileUtil.getExtension(scriptFile);
if (scriptFile.isFile() && ext != null)
{
Expand All @@ -1274,10 +1276,6 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol,

validationErrors.addErrors(ParamReplacementSvc.get().validateDeprecatedReplacements(scriptText, scriptFile.getName()));
}

sb.append(separator);
sb.append(scriptFile.getAbsolutePath());
separator = SCRIPT_PATH_DELIMITER;
}
else
{
Expand All @@ -1294,20 +1292,17 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol,
if (validationErrors.getErrors().stream().anyMatch(e -> SEVERITY.ERROR == e.getSeverity()))
return validationErrors;

if (sb.length() > 0)
JSONArray json = AnalysisScript.toJson(scripts);
if (json != null)
{
ObjectProperty prop = new ObjectProperty(protocol.getLSID(), protocol.getContainer(),
propertyURI, sb.toString());
propertyURI, json.toString());
props.put(propertyURI, prop);
}
else
{
props.remove(propertyURI);
}

// Be sure to strip out any validation scripts that were stored with the legacy propertyURI. We merge and save
// them as a single list in the TRANSFORM
props.remove(ScriptType.VALIDATION.getPropertyURI(protocol));
protocol.setObjectProperties(props);

return validationErrors;
Expand All @@ -1316,7 +1311,6 @@ public ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol,
/** For migrating legacy assay designs that have separate transform and validation script properties */
private enum ScriptType
{
VALIDATION("ValidationScript"),
TRANSFORM("TransformScript");

private final String _uriSuffix;
Expand All @@ -1334,26 +1328,21 @@ public String getPropertyURI(ExpProtocol protocol)

@NotNull
@Override
public List<File> getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope)
public List<AnalysisScript> getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope)
{
List<File> result = new ArrayList<>();
List<AnalysisScript> result = new ArrayList<>();
if (scope == Scope.ASSAY_DEF || scope == Scope.ALL)
{
ObjectProperty transformScripts = protocol.getObjectProperties().get(ScriptType.TRANSFORM.getPropertyURI(protocol));
if (transformScripts != null)
{
List<AnalysisScript> scripts = AnalysisScript.fromJson(transformScripts.getStringValue());
if (scripts != null)
return scripts;

// try the legacy serialization
for (String scriptPath : transformScripts.getStringValue().split("\\" + SCRIPT_PATH_DELIMITER))
{
result.add(new File(scriptPath));
}
}
ObjectProperty validationScripts = protocol.getObjectProperties().get(ScriptType.VALIDATION.getPropertyURI(protocol));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know we don't expose anything in the UI for defining a validation script, just transform scripts. Was this possible from a file based module? Is there code somewhere that would convert any legacy validation scripts to transform scripts or is that not needed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not that I'm aware of, the file based module code for transform scripts is in ModuleAssayProvider.

if (validationScripts != null)
{
for (String scriptPath : validationScripts.getStringValue().split("\\" + SCRIPT_PATH_DELIMITER))
{
result.add(new File(scriptPath));
}
result.add(new AnalysisScript(new File(scriptPath), Set.of(DataTransformService.TransformOperation.INSERT)));
}
}
return result;
Expand Down
13 changes: 6 additions & 7 deletions api/src/org/labkey/api/assay/AssayProvider.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,14 @@

package org.labkey.api.assay;

import jakarta.servlet.http.HttpServletRequest;
import org.fhcrc.cpas.exp.xml.ExperimentRunType;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.labkey.api.assay.actions.AssayRunUploadForm;
import org.labkey.api.assay.pipeline.AssayRunAsyncContext;
import org.labkey.api.assay.transform.AnalysisScript;
import org.labkey.api.assay.transform.DataExchangeHandler;
import org.labkey.api.assay.plate.FilterCriteria;
import org.labkey.api.data.Container;
import org.labkey.api.data.ContainerFilter;
Expand All @@ -42,11 +45,10 @@
import org.labkey.api.gwt.client.model.GWTPropertyDescriptor;
import org.labkey.api.module.Module;
import org.labkey.api.pipeline.PipelineProvider;
import org.labkey.api.qc.DataExchangeHandler;
import org.labkey.api.query.ValidationException;
import org.labkey.api.security.User;
import org.labkey.api.study.publish.PublishKey;
import org.labkey.api.study.assay.ParticipantVisitResolverType;
import org.labkey.api.study.publish.PublishKey;
import org.labkey.api.util.Pair;
import org.labkey.api.view.ActionURL;
import org.labkey.api.view.HttpView;
Expand All @@ -57,7 +59,6 @@
import org.springframework.web.servlet.ModelAndView;
import org.springframework.web.servlet.mvc.Controller;

import jakarta.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
Expand Down Expand Up @@ -239,12 +240,10 @@ enum Scope
* File based QC and analysis scripts can be added to a protocol and invoked when the validate
* method is called. Set to an empty list if no scripts exist.
*/
// TODO File->FileLike
ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List<File> scripts) throws ExperimentException;
ValidationException setValidationAndAnalysisScripts(ExpProtocol protocol, @NotNull List<AnalysisScript> scripts) throws ExperimentException;

@NotNull
// TODO File->FileLike
List<File> getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope);
List<AnalysisScript> getValidationAndAnalysisScripts(ExpProtocol protocol, Scope scope);

void setSaveScriptFiles(ExpProtocol protocol, boolean save) throws ExperimentException;
boolean isSaveScriptFiles(ExpProtocol protocol);
Expand Down
4 changes: 2 additions & 2 deletions api/src/org/labkey/api/assay/AssayRunDatabaseContext.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
import org.labkey.api.exp.api.ExpRun;
import org.labkey.api.exp.property.Domain;
import org.labkey.api.exp.property.DomainProperty;
import org.labkey.api.qc.DefaultTransformResult;
import org.labkey.api.qc.TransformResult;
import org.labkey.api.assay.transform.DefaultTransformResult;
import org.labkey.api.assay.transform.TransformResult;
import org.labkey.api.security.User;
import org.labkey.api.util.NetworkDrive;
import org.labkey.api.view.ActionURL;
Expand Down
2 changes: 1 addition & 1 deletion api/src/org/labkey/api/assay/AssayRunUploadContext.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.labkey.api.exp.api.ExpProtocol;
import org.labkey.api.exp.api.ExpRun;
import org.labkey.api.exp.property.DomainProperty;
import org.labkey.api.qc.TransformResult;
import org.labkey.api.assay.transform.TransformResult;
import org.labkey.api.security.User;
import org.labkey.api.view.ActionURL;
import org.labkey.api.view.HasHttpRequest;
Expand Down
5 changes: 2 additions & 3 deletions api/src/org/labkey/api/assay/AssayRunUploadContextImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,15 @@
import org.labkey.api.exp.api.ExperimentJSONConverter;
import org.labkey.api.exp.property.Domain;
import org.labkey.api.exp.property.DomainProperty;
import org.labkey.api.qc.DefaultTransformResult;
import org.labkey.api.qc.TransformResult;
import org.labkey.api.assay.transform.DefaultTransformResult;
import org.labkey.api.assay.transform.TransformResult;
import org.labkey.api.security.User;
import org.labkey.api.util.URIUtil;
import org.labkey.api.view.ActionURL;
import org.labkey.api.view.NotFoundException;
import org.labkey.api.view.ViewContext;
import org.labkey.vfs.FileLike;

import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
Expand Down
9 changes: 4 additions & 5 deletions api/src/org/labkey/api/assay/DefaultAssayRunCreator.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@
import org.labkey.api.assay.pipeline.AssayRunAsyncContext;
import org.labkey.api.assay.pipeline.AssayUploadPipelineJob;
import org.labkey.api.assay.sample.AssaySampleLookupContext;
import org.labkey.api.assay.transform.DataTransformService;
import org.labkey.api.assay.transform.TransformDataHandler;
import org.labkey.api.assay.transform.TransformResult;
import org.labkey.api.data.ColumnInfo;
import org.labkey.api.data.Container;
import org.labkey.api.data.ContainerManager;
Expand Down Expand Up @@ -64,9 +67,6 @@
import org.labkey.api.exp.property.ValidatorContext;
import org.labkey.api.pipeline.PipelineService;
import org.labkey.api.pipeline.PipelineValidationException;
import org.labkey.api.qc.DataTransformer;
import org.labkey.api.qc.TransformDataHandler;
import org.labkey.api.qc.TransformResult;
import org.labkey.api.query.BatchValidationException;
import org.labkey.api.query.PropertyValidationError;
import org.labkey.api.query.SimpleValidationError;
Expand Down Expand Up @@ -113,8 +113,7 @@ public DefaultAssayRunCreator(ProviderType provider)

public TransformResult transform(AssayRunUploadContext<ProviderType> context, ExpRun run) throws ValidationException
{
DataTransformer<ProviderType> transformer = new DefaultDataTransformer<>();
return transformer.transformAndValidate(context, run);
return DataTransformService.get().transformAndValidate(context, run, DataTransformService.TransformOperation.INSERT);
}

@Override
Expand Down
3 changes: 1 addition & 2 deletions api/src/org/labkey/api/assay/TsvDataHandler.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.labkey.api.data.SimpleFilter;
import org.labkey.api.data.Sort;
import org.labkey.api.data.TSVGridWriter;
import org.labkey.api.data.TSVWriter;
import org.labkey.api.data.TableInfo;
import org.labkey.api.data.TableSelector;
import org.labkey.api.exp.ExperimentException;
Expand All @@ -38,7 +37,7 @@
import org.labkey.api.exp.api.ExpData;
import org.labkey.api.exp.api.ExpProtocol;
import org.labkey.api.exp.api.ExpRun;
import org.labkey.api.qc.TransformDataHandler;
import org.labkey.api.assay.transform.TransformDataHandler;
import org.labkey.api.query.FieldKey;
import org.labkey.api.security.User;
import org.labkey.api.util.FileType;
Expand Down
4 changes: 2 additions & 2 deletions api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@
import org.labkey.api.exp.property.Domain;
import org.labkey.api.exp.property.DomainProperty;
import org.labkey.api.pipeline.PipelineService;
import org.labkey.api.qc.DefaultTransformResult;
import org.labkey.api.qc.TransformResult;
import org.labkey.api.assay.transform.DefaultTransformResult;
import org.labkey.api.assay.transform.TransformResult;
import org.labkey.api.qc.TsvDataExchangeHandler;
import org.labkey.api.query.DefaultSchema;
import org.labkey.api.query.PdLookupForeignKey;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
import org.labkey.api.exp.property.Domain;
import org.labkey.api.exp.property.DomainProperty;
import org.labkey.api.exp.property.PropertyService;
import org.labkey.api.qc.DefaultTransformResult;
import org.labkey.api.qc.TransformResult;
import org.labkey.api.assay.transform.DefaultTransformResult;
import org.labkey.api.assay.transform.TransformResult;
import org.labkey.api.security.User;
import org.labkey.api.security.UserManager;
import org.labkey.api.view.ActionURL;
Expand Down
108 changes: 108 additions & 0 deletions api/src/org/labkey/api/assay/transform/AnalysisScript.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package org.labkey.api.assay.transform;

import org.jetbrains.annotations.Nullable;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.labkey.vfs.FileLike;
import org.labkey.vfs.FileSystemLike;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class AnalysisScript
{
FileLike _script;
Set<DataTransformService.TransformOperation> _operations = new HashSet<>();

public AnalysisScript(File script, Set<DataTransformService.TransformOperation> operations)
{
_script = new FileSystemLike.Builder(script).build().getRoot();
_operations = operations;
}

private AnalysisScript(File script, List<String> operations)
{
_script = new FileSystemLike.Builder(script).build().getRoot();
for (String op : operations)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume it is not possible to have a null operations param here, but thoughts on adding in a check and default case for setting operations to insert?

Copy link
Contributor Author

@labkey-klum labkey-klum Dec 19, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good suggestion, done.
15e395a

{
if (op != null)
_operations.add(DataTransformService.TransformOperation.valueOf(op));
}

// default to insert only
if (_operations.isEmpty())
_operations.add(DataTransformService.TransformOperation.INSERT);
}

public FileLike getScript()
{
return _script;
}

public String getScriptPath()
{
return _script.toNioPathForRead().toString();
}

public boolean canExecute(DataTransformService.TransformOperation operation)
{
return _operations.contains(operation);
}

private static JSONObject toJson(AnalysisScript script)
{
JSONObject json = new JSONObject();

json.put("operations", script._operations);
json.put("script", script._script.toNioPathForRead());

return json;
}

@Nullable
public static JSONArray toJson(Collection<AnalysisScript> scripts)
{
if (!scripts.isEmpty())
{
JSONArray json = new JSONArray();
for (AnalysisScript script : scripts)
{
json.put(toJson(script));
}
return json;
}
return null;
}

@Nullable
public static List<AnalysisScript> fromJson(String jsonStr)
{
try
{
List<AnalysisScript> scripts = new ArrayList<>();
JSONArray json = new JSONArray(jsonStr);
for (Object o : json.toList())
{
if (o instanceof Map<?,?> props)
{
File script = new File(String.valueOf(props.get("script")));
List<String> ops = (List<String>) props.get("operations");

scripts.add(new AnalysisScript(script, ops));
}
}
return scripts;
}
catch (JSONException e)
{
// ignore
return null;
}
}
}
Loading