Skip to content

Commit

Permalink
fix: set the vfs_cache in the task working directory to work with wor…
Browse files Browse the repository at this point in the history
…ker isolation

Worker isolation restricts file creating inside the task working directory. By default Common VFS will create a cache directory /tmp/vfs_cache which will not be allowed by Kestra's security manager.

To fix that we need to init the FileRecplicator by ourselves and pass it a cache directory created inside the task working directory.
  • Loading branch information
loicmathieu committed Aug 14, 2024
1 parent 54485a0 commit 95b1119
Show file tree
Hide file tree
Showing 9 changed files with 48 additions and 8 deletions.
3 changes: 2 additions & 1 deletion src/main/java/io/kestra/plugin/fs/vfs/Delete.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ public abstract class Delete extends AbstractVfsTask implements RunnableTask<Del
private final Boolean errorOnMissing = false;

public Output run(RunContext runContext) throws Exception {
try (StandardFileSystemManager fsm = new StandardFileSystemManager()) {
try (StandardFileSystemManager fsm = new KestraStandardFileSystemManager(runContext)) {
fsm.setConfiguration(StandardFileSystemManager.class.getResource(KestraStandardFileSystemManager.CONFIG_RESOURCE));
fsm.init();

return VfsService.delete(
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/io/kestra/plugin/fs/vfs/Download.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ public abstract class Download extends AbstractVfsTask implements RunnableTask<D
protected String from;

public Output run(RunContext runContext) throws Exception {
try (StandardFileSystemManager fsm = new StandardFileSystemManager()) {
try (StandardFileSystemManager fsm = new KestraStandardFileSystemManager(runContext)) {
fsm.setConfiguration(StandardFileSystemManager.class.getResource(KestraStandardFileSystemManager.CONFIG_RESOURCE));
fsm.init();

return VfsService.download(
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/io/kestra/plugin/fs/vfs/Downloads.java
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ public abstract class Downloads extends AbstractVfsTask implements RunnableTask<
public Output run(RunContext runContext) throws Exception {
Logger logger = runContext.logger();

try (StandardFileSystemManager fsm = new StandardFileSystemManager()) {
try (StandardFileSystemManager fsm = new KestraStandardFileSystemManager(runContext)) {
fsm.setConfiguration(StandardFileSystemManager.class.getResource(KestraStandardFileSystemManager.CONFIG_RESOURCE));
fsm.init();

// path
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package io.kestra.plugin.fs.vfs;

import io.kestra.core.runners.RunContext;
import org.apache.commons.vfs2.impl.DefaultFileReplicator;
import org.apache.commons.vfs2.impl.StandardFileSystemManager;

import java.io.File;
import java.nio.file.Path;

class KestraStandardFileSystemManager extends StandardFileSystemManager {
static final String CONFIG_RESOURCE = "providers.xml"; // same as StandardFileSystemManager.CONFIG_RESOURCE

private final RunContext runContext;

KestraStandardFileSystemManager(RunContext runContext) {
super();

this.runContext = runContext;
}

@Override
protected DefaultFileReplicator createDefaultFileReplicator() {
// By default, the file replicator uses /tmp as the base temp directory; we create it manually to use the task working directory.
File vfsCache = this.runContext.workingDir().resolve(Path.of("vfs_cache")).toFile();
if (!vfsCache.mkdirs()) {
throw new RuntimeException("Unable to create directory " + vfsCache.getPath());
}

return new DefaultFileReplicator(vfsCache);
}
}
3 changes: 2 additions & 1 deletion src/main/java/io/kestra/plugin/fs/vfs/List.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ public abstract class List extends AbstractVfsTask implements RunnableTask<List.
private boolean recursive = false;

public Output run(RunContext runContext) throws Exception {
try (StandardFileSystemManager fsm = new StandardFileSystemManager()) {
try (StandardFileSystemManager fsm = new KestraStandardFileSystemManager(runContext)) {
fsm.setConfiguration(StandardFileSystemManager.class.getResource(KestraStandardFileSystemManager.CONFIG_RESOURCE));
fsm.init();

return VfsService.list(
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/io/kestra/plugin/fs/vfs/Move.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ public abstract class Move extends AbstractVfsTask implements RunnableTask<Move.
private String to;

public Output run(RunContext runContext) throws Exception {
try (StandardFileSystemManager fsm = new StandardFileSystemManager()) {
try (StandardFileSystemManager fsm = new KestraStandardFileSystemManager(runContext)) {
fsm.setConfiguration(StandardFileSystemManager.class.getResource(KestraStandardFileSystemManager.CONFIG_RESOURCE));
fsm.init();

return VfsService.move(
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/io/kestra/plugin/fs/vfs/Trigger.java
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ public Optional<Execution> evaluate(ConditionContext conditionContext, TriggerCo
// connection options
FileSystemOptions fileSystemOptions = this.fsOptions(runContext);

try (StandardFileSystemManager fsm = new StandardFileSystemManager()) {
try (StandardFileSystemManager fsm = new KestraStandardFileSystemManager(runContext)) {
fsm.setConfiguration(StandardFileSystemManager.class.getResource(KestraStandardFileSystemManager.CONFIG_RESOURCE));
fsm.init();

List.Output run;
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/io/kestra/plugin/fs/vfs/Upload.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.apache.commons.vfs2.impl.StandardFileSystemManager;

import java.net.URI;

import jakarta.validation.constraints.NotNull;

@SuperBuilder
Expand All @@ -31,7 +32,8 @@ public abstract class Upload extends AbstractVfsTask implements RunnableTask<Upl
private String to;

public Upload.Output run(RunContext runContext) throws Exception {
try (StandardFileSystemManager fsm = new StandardFileSystemManager()) {
try (StandardFileSystemManager fsm = new KestraStandardFileSystemManager(runContext)) {
fsm.setConfiguration(StandardFileSystemManager.class.getResource(KestraStandardFileSystemManager.CONFIG_RESOURCE));
fsm.init();

var renderedFrom = runContext.render(this.from);
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/io/kestra/plugin/fs/vfs/Uploads.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ public abstract class Uploads extends AbstractVfsTask implements RunnableTask<Up
private String to;

public Output run(RunContext runContext) throws Exception {
try (StandardFileSystemManager fsm = new StandardFileSystemManager()) {
try (StandardFileSystemManager fsm = new KestraStandardFileSystemManager(runContext)) {
fsm.setConfiguration(StandardFileSystemManager.class.getResource(KestraStandardFileSystemManager.CONFIG_RESOURCE));
fsm.init();

String[] renderedFrom;
Expand Down

0 comments on commit 95b1119

Please sign in to comment.