diff --git a/src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java b/src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java index ce62318..c9d5371 100644 --- a/src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java +++ b/src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java @@ -9,6 +9,7 @@ import io.kestra.core.models.tasks.runners.ScriptService; import io.kestra.core.models.tasks.runners.TaskRunner; import io.kestra.core.runners.RunContext; +import io.kestra.core.serializers.JacksonMapper; import io.kestra.plugin.dbt.ResultParser; import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions; import io.kestra.plugin.scripts.exec.scripts.models.RunnerType; @@ -174,18 +175,15 @@ public void accept(String line, Boolean isStdErr) { ); } - List commandsArgs = ScriptService.scriptCommands( - List.of("/bin/sh", "-c"), - null, - List.of(createDbtCommand(runContext)) - ); - ScriptOutput run = commandsWrapper .addEnv(Map.of( "PYTHONUNBUFFERED", "true", "PIP_ROOT_USER_ACTION", "ignore" )) - .withCommands(commandsArgs) + .withInterpreter(Property.of(List.of("/bin/sh", "-c"))) + .withCommands(new Property<>(JacksonMapper.ofJson().writeValueAsString( + List.of(createDbtCommand(runContext))) + )) .run(); parseResults(runContext, workingDirectory, run); diff --git a/src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java b/src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java index 35e13b9..15d5414 100644 --- a/src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java +++ b/src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java @@ -282,9 +282,7 @@ public class DbtCLI extends AbstractExecScript { title = "The list of dbt CLI commands to run." ) @NotNull - @NotEmpty - @PluginProperty(dynamic = true) - private List commands; + private Property> commands; @Schema( title = "The `profiles.yml` file content.", @@ -396,15 +394,10 @@ public void accept(String line, Boolean isStdErr) { ); } - //Create and run commands - List commandsArgs = ScriptService.scriptCommands( - runContext.render(this.interpreter).asList(String.class), - this.getBeforeCommandsWithOptions(runContext), - runContext.render(this.commands).stream().map(command -> command.startsWith("dbt") ? command.concat(" --log-format json") : command).toList() - ); + var renderedCommands = runContext.render(this.commands).asList(String.class); // check that if a command uses --project-dir, the projectDir must be set - if (commandsArgs.stream().anyMatch(cmd -> cmd.contains("--project-dir")) && this.projectDir == null) { + if (renderedCommands.stream().anyMatch(cmd -> cmd.contains("--project-dir")) && this.projectDir == null) { runContext.logger().warn("One of the dbt CLI commands uses the `--project-dir` flag, but the `projectDir` task property is not set. Make sure to set the `projectDir` property."); } @@ -413,7 +406,13 @@ public void accept(String line, Boolean isStdErr) { "PYTHONUNBUFFERED", "true", "PIP_ROOT_USER_ACTION", "ignore" )) - .withCommands(commandsArgs) + .withInterpreter(this.interpreter) + .withBeforeCommands(Property.of(this.getBeforeCommandsWithOptions(runContext))) + .withCommands(Property.of( + renderedCommands.stream() + .map(command -> command.startsWith("dbt") ? command.concat(" --log-format json") : command) + .toList()) + ) .run(); //Parse run results diff --git a/src/main/java/io/kestra/plugin/dbt/cli/Setup.java b/src/main/java/io/kestra/plugin/dbt/cli/Setup.java index a9c1b39..b0afa2b 100644 --- a/src/main/java/io/kestra/plugin/dbt/cli/Setup.java +++ b/src/main/java/io/kestra/plugin/dbt/cli/Setup.java @@ -195,18 +195,14 @@ public ScriptOutput run(RunContext runContext) throws Exception { new HashMap<>() ); - List commandsArgs = ScriptService.scriptCommands( - runContext.render(this.interpreter).asList(String.class), - this.getBeforeCommandsWithOptions(runContext), - commands - ); - return commandsWrapper .addEnv(Map.of( "PYTHONUNBUFFERED", "true", "PIP_ROOT_USER_ACTION", "ignore" )) - .withCommands(commandsArgs) + .withInterpreter(this.interpreter) + .withBeforeCommands(Property.of(this.getBeforeCommandsWithOptions(runContext))) + .withCommands(Property.of(commands)) .run(); } diff --git a/src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java b/src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java index 92cbe4e..84f35bf 100644 --- a/src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java +++ b/src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java @@ -76,7 +76,7 @@ void run() throws Exception { .profiles(Property.of(PROFILES) ) .containerImage(new Property<>("ghcr.io/kestra-io/dbt-bigquery:latest")) - .commands(List.of("dbt build")) + .commands(Property.of(List.of("dbt build"))) .build(); RunContext runContext = TestsUtils.mockRunContext(runContextFactory, execute, Map.of()); @@ -98,7 +98,7 @@ void testDbtCliWithStoreManifest_manifestShouldBePresentInKvStore() throws Excep .profiles(Property.of(PROFILES) ) .containerImage(new Property<>("ghcr.io/kestra-io/dbt-bigquery:latest")) - .commands(List.of("dbt build")) + .commands(Property.of(List.of("dbt build"))) .storeManifest( DbtCLI.KvStoreManifest.builder() .key(Property.of(MANIFEST_KEY)) @@ -131,7 +131,7 @@ void testDbtWithLoadManifest_manifestShouldBeLoadedFromKvStore() throws Exceptio .profiles(Property.of(PROFILES)) .projectDir(Property.of("unit-kestra")) .containerImage(new Property<>("ghcr.io/kestra-io/dbt-bigquery:latest")) - .commands(List.of("dbt build --project-dir unit-kestra")) + .commands(Property.of(List.of("dbt build --project-dir unit-kestra"))) .loadManifest( DbtCLI.KvStoreManifest.builder() .key(Property.of(MANIFEST_KEY))