Skip to content

Commit b2bd997

Browse files
labkey-jeckelslabkey-jeckels
authored
Adopt FileLike for more pipeline APIs (#576)
1 parent 9e1c240 commit b2bd997

File tree

13 files changed

+65
-64
lines changed

13 files changed

+65
-64
lines changed

‎lincs/src/org/labkey/lincs/psp/LincsPspPipelineJob.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.labkey.api.util.FileUtil;
1313
import org.labkey.api.util.URLHelper;
1414
import org.labkey.api.view.ViewBackgroundInfo;
15-
import org.labkey.lincs.LincsModule;
1615

1716
public class LincsPspPipelineJob extends PipelineJob implements LincsPspJobSupport
1817
{
@@ -43,7 +42,7 @@ public LincsPspPipelineJob(ViewBackgroundInfo info, PipeRoot root, ITargetedMSRu
4342

4443
String baseLogFileName = FileUtil.makeFileNameWithTimestamp("LincsPSP_" + (_oldPspJob != null ? "rerun_" : "") + run.getBaseName().replace(" ", "_"));
4544

46-
LocalDirectory localDirectory = LocalDirectory.create(root, LincsModule.NAME, baseLogFileName,
45+
LocalDirectory localDirectory = LocalDirectory.create(root, baseLogFileName,
4746
!root.isCloudRoot() ? root.getRootPath().getAbsolutePath() : FileUtil.getTempDirectory().getPath());
4847
setLocalDirectory(localDirectory);
4948
setLogFile(localDirectory.determineLogFile());

‎nextflow/src/org/labkey/nextflow/NextFlowController.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.labkey.api.view.ViewBackgroundInfo;
4040
import org.labkey.nextflow.pipeline.NextFlowPipelineJob;
4141
import org.labkey.nextflow.pipeline.NextFlowProtocol;
42+
import org.labkey.vfs.FileLike;
4243
import org.springframework.validation.BindException;
4344
import org.springframework.validation.Errors;
4445
import org.springframework.web.servlet.ModelAndView;
@@ -261,17 +262,17 @@ public void validateCommand(AnalyzeForm o, Errors errors)
261262
@Override
262263
public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors)
263264
{
264-
List<File> selectedFiles = o.getValidatedFiles(getContainer(), false);
265+
List<FileLike> selectedFiles = o.getValidatedFiles(getContainer(), false);
265266
if (selectedFiles.isEmpty())
266267
{
267268
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
268269
}
269270
// NextFlow operates on the full directory so show the list to the user, regardless of what they selected
270271
// from the file listing
271-
File inputDir = selectedFiles.get(0).getParentFile();
272+
FileLike inputDir = selectedFiles.get(0).getParent();
272273

273-
File[] inputFiles = inputDir.listFiles(new PipelineProvider.FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES));
274-
if (inputFiles == null || inputFiles.length == 0)
274+
List<FileLike> inputFiles = inputDir.getChildren().stream().filter(new PipelineProvider.FileTypesEntryFilter(NextFlowProtocol.INPUT_TYPES)).toList();
275+
if (inputFiles.isEmpty())
275276
{
276277
return new HtmlView(HtmlString.of("Couldn't find input file(s)"));
277278
}
@@ -290,7 +291,7 @@ public ModelAndView getView(AnalyzeForm o, boolean b, BindException errors)
290291
INPUT(at(hidden, true, name, "launch", value, true)),
291292
Arrays.stream(o.getFile()).map(f -> INPUT(at(hidden, true, name, "file", value, f))).toList(),
292293
"Files: ",
293-
UL(Arrays.stream(inputFiles).map(File::getName).map(DOM::LI)),
294+
UL(inputFiles.stream().map(FileLike::getName).map(DOM::LI)),
294295
"Config: ",
295296
new SelectBuilder().name("configFile").addOptions(Arrays.stream(configFiles).filter(f -> f.isFile() && f.getName().toLowerCase().endsWith(".config")).map(File::getName).sorted(String.CASE_INSENSITIVE_ORDER).toList()).build(),
296297
DOM.BR(),
@@ -318,7 +319,7 @@ public boolean handlePost(AnalyzeForm form, BindException errors) throws Excepti
318319
}
319320
else
320321
{
321-
List<File> inputFiles = form.getValidatedFiles(getContainer());
322+
List<FileLike> inputFiles = form.getValidatedFiles(getContainer());
322323
if (inputFiles.isEmpty())
323324
{
324325
errors.reject(ERROR_MSG, "No input files");
@@ -327,7 +328,7 @@ public boolean handlePost(AnalyzeForm form, BindException errors) throws Excepti
327328
{
328329
ViewBackgroundInfo info = getViewBackgroundInfo();
329330
PipeRoot root = PipelineService.get().findPipelineRoot(info.getContainer());
330-
NextFlowPipelineJob job = NextFlowPipelineJob.create(info, root, configFile.toPath(), inputFiles.stream().map(File::toPath).toList());
331+
NextFlowPipelineJob job = NextFlowPipelineJob.create(info, root, configFile.toPath(), inputFiles);
331332
PipelineService.get().queueJob(job);
332333
LOG.info("NextFlow job queued: {}", job.getJsonJobInfo(false));
333334
}

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowPipelineJob.java‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
import org.labkey.api.util.StringUtilsLabKey;
2121
import org.labkey.api.util.logging.LogHelper;
2222
import org.labkey.api.view.ViewBackgroundInfo;
23+
import org.labkey.api.writer.PrintWriters;
2324
import org.labkey.nextflow.NextFlowManager;
25+
import org.labkey.vfs.FileLike;
2426

2527
import java.io.BufferedWriter;
2628
import java.io.File;
@@ -35,29 +37,29 @@ public class NextFlowPipelineJob extends AbstractFileAnalysisJob
3537
{
3638
protected static final Logger LOG = LogHelper.getLogger(NextFlowPipelineJob.class, "NextFlow jobs");
3739

38-
private Path config;
40+
private FileLike config;
3941

4042
@SuppressWarnings("unused") // For serialization
4143
protected NextFlowPipelineJob()
4244
{}
4345

44-
public static NextFlowPipelineJob create(ViewBackgroundInfo info, @NotNull PipeRoot root, Path templateConfig, List<Path> inputFiles) throws IOException
46+
public static NextFlowPipelineJob create(ViewBackgroundInfo info, @NotNull PipeRoot root, Path templateConfig, List<FileLike> inputFiles) throws IOException
4547
{
46-
Path parentDir = inputFiles.get(0).getParent();
48+
FileLike parentDir = inputFiles.get(0).getParent();
4749

4850
String jobName = FileUtil.makeFileNameWithTimestamp("NextFlow");
49-
Path jobDir = parentDir.resolve(jobName);
50-
Path log = jobDir.resolve(jobName + ".log");
51+
FileLike jobDir = parentDir.resolveChild(jobName);
52+
FileLike log = jobDir.resolveChild(jobName + ".log");
5153
FileUtil.createDirectory(jobDir);
5254

53-
Path config = createConfig(templateConfig, parentDir, jobDir, info.getContainer());
55+
FileLike config = createConfig(templateConfig, parentDir, jobDir, info.getContainer());
5456

5557
return new NextFlowPipelineJob(info, root, config, inputFiles, log);
5658
}
5759

58-
public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, Path config, List<Path> inputFiles, Path log) throws IOException
60+
public NextFlowPipelineJob(ViewBackgroundInfo info, @NotNull PipeRoot root, FileLike config, List<FileLike> inputFiles, FileLike log) throws IOException
5961
{
60-
super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getFileName().toString(), config, inputFiles, false, false);
62+
super(new NextFlowProtocol(), NextFlowPipelineProvider.NAME, info, root, config.getName(), config, inputFiles, false);
6163
this.config = config;
6264
setLogFile(log);
6365
}
@@ -69,7 +71,7 @@ public JSONObject getJsonJobInfo(boolean includeInvocationCount)
6971
result.put("container", getContainer().getPath());
7072
result.put("filePath", getLogFilePath().getParent().toString());
7173
result.put("runName", getNextFlowRunName(includeInvocationCount));
72-
result.put("configFile", getConfig().getFileName().toString());
74+
result.put("configFile", getConfig().getName());
7375
return result;
7476
}
7577

@@ -88,7 +90,7 @@ public ParamParser getInputParameters()
8890
}
8991

9092
/** Take the template config file and substitute in the values for this job */
91-
private static Path createConfig(Path configTemplate, Path parentDir, Path jobDir, Container container) throws IOException
93+
private static FileLike createConfig(Path configTemplate, FileLike parentDir, FileLike jobDir, Container container) throws IOException
9294
{
9395
String template;
9496
try (InputStream in = Files.newInputStream(configTemplate))
@@ -104,8 +106,8 @@ private static Path createConfig(Path configTemplate, Path parentDir, Path jobDi
104106
uploadUrl = StringUtils.stripEnd(uploadUrl, "/");
105107
substitutedContent = substitutedContent.replace("${panorama.upload_url}", "panorama.upload_url = '" + uploadUrl + "'");
106108

107-
Path substitutedFile = jobDir.resolve(configTemplate.getFileName());
108-
try (BufferedWriter writer = Files.newBufferedWriter(substitutedFile))
109+
FileLike substitutedFile = jobDir.resolveChild(configTemplate.getFileName().toString());
110+
try (BufferedWriter writer = new BufferedWriter(PrintWriters.getPrintWriter(substitutedFile.openOutputStream())))
109111
{
110112
writer.write(substitutedContent);
111113
}
@@ -115,7 +117,7 @@ private static Path createConfig(Path configTemplate, Path parentDir, Path jobDi
115117
@Override
116118
public String getDescription()
117119
{
118-
return "NextFlow analysis of " + StringUtilsLabKey.pluralize(getInputFilePaths().size(), "file") + " using config: " + config.getFileName();
120+
return "NextFlow analysis of " + StringUtilsLabKey.pluralize(getInputFilePaths().size(), "file") + " using config: " + config.getName();
119121
}
120122

121123
@Override
@@ -131,7 +133,7 @@ public TaskId getTaskPipelineId()
131133
}
132134

133135
@Override
134-
public AbstractFileAnalysisJob createSingleFileJob(File file)
136+
public AbstractFileAnalysisJob createSingleFileJob(FileLike file)
135137
{
136138
throw new UnsupportedOperationException();
137139
}

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowProtocol.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
import org.labkey.api.pipeline.file.AbstractFileAnalysisProtocolFactory;
88
import org.labkey.api.util.FileType;
99
import org.labkey.api.view.ViewBackgroundInfo;
10+
import org.labkey.vfs.FileLike;
1011

11-
import java.nio.file.Path;
1212
import java.util.List;
1313
import java.util.Map;
1414

@@ -47,7 +47,7 @@ public NextFlowProtocol createProtocolInstance(String name, String description,
4747
}
4848

4949
@Override
50-
public Path getDefaultParametersFile(PipeRoot root)
50+
public FileLike getDefaultParametersFile(PipeRoot root)
5151
{
5252
return null;
5353
}
@@ -61,7 +61,7 @@ public String getName()
6161
}
6262

6363
@Override
64-
public NextFlowPipelineJob createPipelineJob(ViewBackgroundInfo info, PipeRoot root, List<Path> filesInput, Path fileParameters, @Nullable Map<String, String> variableMap)
64+
public NextFlowPipelineJob createPipelineJob(ViewBackgroundInfo info, PipeRoot root, List<FileLike> filesInput, FileLike fileParameters, @Nullable Map<String, String> variableMap)
6565
{
6666
throw new UnsupportedOperationException();
6767
}

‎nextflow/src/org/labkey/nextflow/pipeline/NextFlowRunTask.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.labkey.api.util.FileType;
1717
import org.labkey.nextflow.NextFlowConfiguration;
1818
import org.labkey.nextflow.NextFlowManager;
19+
import org.labkey.vfs.FileLike;
1920

2021
import java.io.BufferedReader;
2122
import java.io.File;
@@ -143,9 +144,9 @@ else if (Files.isDirectory(path))
143144
}
144145
}
145146

146-
private boolean hasAwsSection(Path configFile) throws PipelineJobException
147+
private boolean hasAwsSection(FileLike configFile) throws PipelineJobException
147148
{
148-
try (InputStream in = Files.newInputStream(configFile);
149+
try (InputStream in = configFile.openInputStream();
149150
InputStreamReader isReader = new InputStreamReader(in, StandardCharsets.UTF_8);
150151
BufferedReader reader = new BufferedReader(isReader))
151152
{
@@ -174,7 +175,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
174175
private @NotNull List<String> getArgs() throws PipelineJobException
175176
{
176177
NextFlowConfiguration config = NextFlowManager.get().getConfiguration();
177-
Path configFile = getJob().getConfig();
178+
FileLike configFile = getJob().getConfig();
178179

179180
boolean aws = hasAwsSection(configFile);
180181

@@ -194,7 +195,7 @@ private boolean hasAwsSection(Path configFile) throws PipelineJobException
194195
args.add(s3Path);
195196
}
196197
args.add("-c");
197-
args.add(configFile.toAbsolutePath().toString());
198+
args.add(configFile.toNioPathForRead().toAbsolutePath().toString());
198199
args.add("-name");
199200
args.add(getJob().getNextFlowRunName(true));
200201
return args;

‎panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,8 @@
215215
import org.labkey.panoramapublic.view.expannotations.TargetedMSExperimentWebPart;
216216
import org.labkey.panoramapublic.view.expannotations.TargetedMSExperimentsWebPart;
217217
import org.labkey.panoramapublic.view.publish.CatalogEntryWebPart;
218+
import org.labkey.vfs.FileLike;
219+
import org.labkey.vfs.FileSystemLike;
218220
import org.springframework.validation.BindException;
219221
import org.springframework.validation.Errors;
220222
import org.springframework.web.servlet.ModelAndView;
@@ -1748,15 +1750,15 @@ private boolean validateAction(CopyExperimentForm form, BindException errors)
17481750
return true;
17491751
}
17501752

1751-
private Path getExportFilesDir(Container c)
1753+
private FileLike getExportFilesDir(Container c)
17521754
{
17531755
FileContentService fcs = FileContentService.get();
17541756
if(fcs != null)
17551757
{
17561758
Path fileRoot = fcs.getFileRootPath(c, FileContentService.ContentType.files);
17571759
if (fileRoot != null)
17581760
{
1759-
return fileRoot.resolve(PipelineService.EXPORT_DIR);
1761+
return FileSystemLike.wrapFile(fileRoot.resolve(PipelineService.EXPORT_DIR));
17601762
}
17611763
}
17621764
return null;
@@ -4710,7 +4712,7 @@ private static File getLocalFile(Container container, String fileName) throws Px
47104712
PipeRoot root = PipelineService.get().getPipelineRootSetting(container);
47114713
if (root != null)
47124714
{
4713-
LocalDirectory localDirectory = LocalDirectory.create(root, PanoramaPublicModule.NAME);
4715+
LocalDirectory localDirectory = LocalDirectory.create(root);
47144716
return new File(localDirectory.getLocalDirectoryFile(), fileName);
47154717
}
47164718
else

‎panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
import org.labkey.panoramapublic.query.JournalManager;
8080
import org.labkey.panoramapublic.query.SubmissionManager;
8181
import org.labkey.panoramapublic.security.PanoramaPublicSubmitterRole;
82+
import org.labkey.vfs.FileLike;
8283

8384
import java.io.File;
8485
import java.io.IOException;
@@ -254,9 +255,9 @@ private void verifySymlinks(Container source, Container target, boolean matching
254255
}
255256
}
256257

257-
private void cleanupExportDirectory(User user, File directory)
258+
private void cleanupExportDirectory(User user, FileLike directory)
258259
{
259-
List<? extends ExpData> datas = ExperimentService.get().getExpDatasUnderPath(directory.toPath(), null, true);
260+
List<? extends ExpData> datas = ExperimentService.get().getExpDatasUnderPath(directory.toNioPathForRead(), null, true);
260261
for (ExpData data : datas)
261262
{
262263
data.delete(user);

‎panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentJobSupport.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@
1717

1818
import org.labkey.panoramapublic.model.ExperimentAnnotations;
1919
import org.labkey.panoramapublic.model.Journal;
20-
21-
import java.io.File;
22-
import java.nio.file.Path;
20+
import org.labkey.vfs.FileLike;
2321

2422
/**
2523
* User: vsharma
@@ -32,7 +30,7 @@ public interface CopyExperimentJobSupport
3230

3331
Journal getJournal();
3432

35-
File getExportDir();
33+
FileLike getExportDir();
3634

3735
String getReviewerEmailPrefix();
3836

@@ -48,5 +46,5 @@ public interface CopyExperimentJobSupport
4846

4947
String getPreviousVersionName();
5048

51-
void setExportTargetPath(Path exportTargetPath);
49+
void setExportTargetPath(FileLike exportTargetPath);
5250
}

‎panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentPipelineJob.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,12 @@
3030
import org.labkey.api.view.ActionURL;
3131
import org.labkey.api.view.NotFoundException;
3232
import org.labkey.api.view.ViewBackgroundInfo;
33-
import org.labkey.panoramapublic.PanoramaPublicModule;
3433
import org.labkey.panoramapublic.model.ExperimentAnnotations;
3534
import org.labkey.panoramapublic.model.Journal;
35+
import org.labkey.vfs.FileLike;
3636

3737
import java.io.File;
3838
import java.io.IOException;
39-
import java.nio.file.Path;
4039

4140
/**
4241
* User: vsharma
@@ -63,7 +62,7 @@ public class CopyExperimentPipelineJob extends PipelineJob implements CopyExperi
6362

6463
private String _previousVersionName;
6564

66-
private Path _exportTargetPath;
65+
private FileLike _exportTargetPath;
6766

6867
private Container _exportSourceContainer;
6968

@@ -93,7 +92,7 @@ public CopyExperimentPipelineJob(ViewBackgroundInfo info, PipeRoot root, Experim
9392

9493
// CONSIDER: Add a static factory method to LocalDirectory instead of using the constructor.
9594
// create(@NotNull PipeRoot root, @NotNull String moduleName, @NotNull String baseLogFileName, @NotNull String localDirPath, boolean temporary)
96-
LocalDirectory localDirectory = new LocalDirectory(targetRoot.getContainer(), PanoramaPublicModule.NAME, root, baseLogFileName);
95+
LocalDirectory localDirectory = new LocalDirectory(targetRoot.getContainer(), root, baseLogFileName);
9796

9897
setLocalDirectory(localDirectory);
9998
setLogFile(localDirectory.determineLogFile());
@@ -160,9 +159,9 @@ public Journal getJournal()
160159
}
161160

162161
@Override
163-
public File getExportDir()
162+
public FileLike getExportDir()
164163
{
165-
return _exportTargetPath.toFile();
164+
return _exportTargetPath;
166165
}
167166

168167
@Override
@@ -254,7 +253,7 @@ public void setPreviousVersionName(String previousVersionName)
254253
}
255254

256255
@Override
257-
public void setExportTargetPath(Path exportTargetPath)
256+
public void setExportTargetPath(FileLike exportTargetPath)
258257
{
259258
_exportTargetPath = exportTargetPath;
260259
}

0 commit comments

Comments
 (0)