From e86144d2d5d78d1c683c00988810ceed4e419b45 Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle Date: Tue, 17 Dec 2024 12:03:27 +0100 Subject: [PATCH 1/3] refactor: migrate to dynamic properties migrate abstract python singer to dynamic properties and BigQuery tap --- .../plugin/singer/AbstractPythonSinger.java | 31 ++++++++++--------- .../plugin/singer/taps/AbstractPythonTap.java | 4 +-- .../kestra/plugin/singer/taps/BigQuery.java | 22 ++++++------- .../io/kestra/plugin/singer/taps/BingAds.java | 9 +++--- .../kestra/plugin/singer/taps/ChargeBee.java | 9 +++--- .../plugin/singer/taps/ExchangeRateHost.java | 9 +++--- .../plugin/singer/taps/FacebookAds.java | 9 +++--- .../io/kestra/plugin/singer/taps/Fastly.java | 9 +++--- .../kestra/plugin/singer/taps/GenericTap.java | 11 +++---- .../io/kestra/plugin/singer/taps/GitHub.java | 9 +++--- .../io/kestra/plugin/singer/taps/Gitlab.java | 9 +++--- .../plugin/singer/taps/GoogleAdwords.java | 9 +++--- .../plugin/singer/taps/GoogleAnalytics.java | 9 +++--- .../singer/taps/GoogleSearchConsole.java | 9 +++--- .../io/kestra/plugin/singer/taps/HubSpot.java | 9 +++--- .../io/kestra/plugin/singer/taps/Marketo.java | 9 +++--- .../kestra/plugin/singer/taps/Netsuite.java | 9 +++--- .../singer/taps/PipelinewiseMongoDb.java | 9 +++--- .../plugin/singer/taps/PipelinewiseMysql.java | 9 +++--- .../singer/taps/PipelinewiseOracle.java | 9 +++--- .../singer/taps/PipelinewisePostgres.java | 9 +++--- .../singer/taps/PipelinewiseSqlServer.java | 9 +++--- .../kestra/plugin/singer/taps/Quickbooks.java | 9 +++--- .../kestra/plugin/singer/taps/Recharge.java | 9 +++--- .../plugin/singer/taps/SageIntacct.java | 9 +++--- .../kestra/plugin/singer/taps/Salesforce.java | 9 +++--- .../io/kestra/plugin/singer/taps/Shopify.java | 9 +++--- .../io/kestra/plugin/singer/taps/Slack.java | 9 +++--- .../io/kestra/plugin/singer/taps/Stripe.java | 9 +++--- .../io/kestra/plugin/singer/taps/Zendesk.java | 9 +++--- .../io/kestra/plugin/singer/taps/Zoom.java | 9 +++--- .../singer/targets/AbstractPythonTarget.java | 2 +- .../singer/targets/AdswerveBigQuery.java | 9 +++--- .../io/kestra/plugin/singer/targets/Csv.java | 9 +++--- .../singer/targets/DatamillCoPostgres.java | 9 +++--- .../plugin/singer/targets/GenericTarget.java | 11 +++---- .../io/kestra/plugin/singer/targets/Json.java | 9 +++--- .../singer/targets/MeltanoSnowflake.java | 9 +++--- .../kestra/plugin/singer/targets/Oracle.java | 9 +++--- .../singer/targets/PipelinewisePostgres.java | 9 +++--- .../singer/targets/PipelinewiseRedshift.java | 9 +++--- .../singer/targets/PipelinewiseSnowflake.java | 9 +++--- .../plugin/singer/targets/SqlServer.java | 9 +++--- .../plugin/singer/taps/BigQueryTest.java | 7 +++-- .../taps/PipelinewiseSqlServerTest.java | 3 +- .../singer/targets/AdswerveBigQueryTest.java | 5 +-- .../kestra/plugin/singer/targets/CsvTest.java | 5 +-- .../targets/DatamillCoPostgresTest.java | 5 +-- .../plugin/singer/targets/JsonTest.java | 5 +-- .../plugin/singer/targets/SqlServerTest.java | 5 +-- 50 files changed, 246 insertions(+), 203 deletions(-) diff --git a/src/main/java/io/kestra/plugin/singer/AbstractPythonSinger.java b/src/main/java/io/kestra/plugin/singer/AbstractPythonSinger.java index b06cd21..ba3a3a2 100644 --- a/src/main/java/io/kestra/plugin/singer/AbstractPythonSinger.java +++ b/src/main/java/io/kestra/plugin/singer/AbstractPythonSinger.java @@ -8,6 +8,7 @@ import io.kestra.core.models.annotations.PluginProperty; import io.kestra.core.models.executions.metrics.Counter; import io.kestra.core.models.executions.metrics.Timer; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.Task; import io.kestra.core.models.tasks.runners.AbstractLogConsumer; import io.kestra.core.models.tasks.runners.ScriptService; @@ -65,22 +66,19 @@ public abstract class AbstractPythonSinger extends Task { @Schema( title = "The name of Singer state file stored in KV Store." ) - @PluginProperty(dynamic = true) @NotNull @Builder.Default - protected String stateName = "singer-state"; + protected Property stateName = Property.of("singer-state"); @Schema( title = "Override default pip packages to use a specific version." ) - @PluginProperty(dynamic = true) - protected List pipPackages; + protected Property> pipPackages; @Schema( title = "Override default singer command." ) - @PluginProperty(dynamic = true) - protected String command; + protected Property command; @Schema( title = "Deprecated, use 'taskRunner' instead" @@ -98,9 +96,8 @@ public abstract class AbstractPythonSinger extends Task { private TaskRunner taskRunner = Docker.instance(); @Schema(title = "The task runner container image, only used if the task runner is container-based.") - @PluginProperty(dynamic = true) @Builder.Default - private String containerImage = DEFAULT_IMAGE; + private Property containerImage = Property.of(DEFAULT_IMAGE); protected DockerOptions injectDefaults(DockerOptions original) { if (original == null) { @@ -115,14 +112,16 @@ protected DockerOptions injectDefaults(DockerOptions original) { return builder.build(); } - abstract public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException, IOException; + public abstract Map configuration(RunContext runContext) throws IllegalVariableEvaluationException, IOException; - abstract public List pipPackages(); + public abstract Property> pipPackages(); - abstract protected String command(); + protected abstract Property command(); protected String finalCommand(RunContext runContext) throws IllegalVariableEvaluationException { - return this.command != null ? runContext.render(this.command) : this.command(); + return this.command != null ? + runContext.render(this.command).as(String.class).orElseThrow() : + runContext.render(this.command()).as(String.class).orElse(null); } protected void run(RunContext runContext, String command, AbstractLogConsumer logConsumer) throws Exception { @@ -141,7 +140,7 @@ protected void run(RunContext runContext, String command, AbstractLogConsumer lo .withWarningOnStdErr(true) .withDockerOptions(this.injectDefaults(getDocker())) .withTaskRunner(taskRunner) - .withContainerImage(this.containerImage) + .withContainerImage(runContext.render(this.containerImage).as(String.class).orElseThrow()) .withLogConsumer(logConsumer) .withCommands(ScriptService.scriptCommands( List.of("/bin/sh", "-c"), @@ -156,7 +155,11 @@ protected void run(RunContext runContext, String command, AbstractLogConsumer lo } protected Stream pipInstallCommands(RunContext runContext) throws Exception { - ArrayList finalRequirements = new ArrayList<>(this.pipPackages != null ? runContext.render(this.pipPackages) : this.pipPackages()); + ArrayList finalRequirements = new ArrayList<>( + this.pipPackages != null ? + runContext.render(this.pipPackages).asList(String.class) : + runContext.render(this.pipPackages()).asList(String.class) + ); finalRequirements.add("python-json-logger"); return Stream.of( diff --git a/src/main/java/io/kestra/plugin/singer/taps/AbstractPythonTap.java b/src/main/java/io/kestra/plugin/singer/taps/AbstractPythonTap.java index cb38afa..40afc85 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/AbstractPythonTap.java +++ b/src/main/java/io/kestra/plugin/singer/taps/AbstractPythonTap.java @@ -69,7 +69,7 @@ public void initEnvDiscoveryAndState(RunContext runContext) throws Exception { if (this.features().contains(Feature.STATE)) { try { InputStream taskStateFile = runContext.stateStore().getState( - runContext.render(this.stateName), + runContext.render(this.stateName).as(String.class).orElseThrow(), "state.json", runContext.storage().getTaskStorageContext().map(StorageContext.Task::getTaskRunValue).orElse(null) ); @@ -102,7 +102,7 @@ public Output run(RunContext runContext) throws Exception { .raw(runContext.storage().putFile(this.rawSingerStream.getLeft())); if (this.features().contains(Feature.STATE)) { - this.saveState(runContext, runContext.render(this.stateName), this.stateRecords); + this.saveState(runContext, runContext.render(this.stateName).as(String.class).orElseThrow(), this.stateRecords); } return outputBuilder diff --git a/src/main/java/io/kestra/plugin/singer/taps/BigQuery.java b/src/main/java/io/kestra/plugin/singer/taps/BigQuery.java index df0a7cc..8d93d67 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/BigQuery.java +++ b/src/main/java/io/kestra/plugin/singer/taps/BigQuery.java @@ -4,6 +4,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -30,8 +31,7 @@ public class BigQuery extends AbstractPythonTap implements RunnableTask serviceAccount; @NotNull @NotEmpty @@ -45,17 +45,15 @@ public class BigQuery extends AbstractPythonTap implements RunnableTask limit; @NotNull @Schema( title = "When replicating incrementally, disable to only select records whose `datetime_key` is greater than the maximum value replicated in the last run, by excluding records whose timestamps match exactly.", description = "This could cause records to be missed that were created after the last run finished, but during the same second and with the same timestamp." ) - @PluginProperty @Builder.Default - private Boolean startAlwaysInclusive = true; + private Property startAlwaysInclusive = Property.of(true); @NotNull @Schema( @@ -83,7 +81,7 @@ public List features() { public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("streams", this.streams) - .put("start_always_inclusive", this.startAlwaysInclusive); + .put("start_always_inclusive", runContext.render(this.startAlwaysInclusive).as(Boolean.class).orElseThrow()); if (this.startDateTime != null) { builder.put("start_datetime", runContext.render(this.startDateTime.toString())); @@ -97,13 +95,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return List.of("git+https://github.com/kestra-io/tap-bigquery.git@fix"); + public Property> pipPackages() { + return Property.of(List.of("git+https://github.com/kestra-io/tap-bigquery.git@fix")); } @Override - protected String command() { - return "tap-bigquery"; + protected Property command() { + return Property.of("tap-bigquery"); } @SuppressWarnings("DuplicatedCode") @@ -112,7 +110,7 @@ protected Map environmentVariables(RunContext runContext) throws HashMap env = new HashMap<>(super.environmentVariables(runContext)); if (this.serviceAccount != null) { - this.writeSingerFiles("google-credentials.json", runContext.render(this.serviceAccount)); + this.writeSingerFiles("google-credentials.json", runContext.render(this.serviceAccount).as(String.class).orElseThrow()); env.put("GOOGLE_APPLICATION_CREDENTIALS", workingDirectory.toAbsolutePath() + "/google-credentials.json"); } diff --git a/src/main/java/io/kestra/plugin/singer/taps/BingAds.java b/src/main/java/io/kestra/plugin/singer/taps/BingAds.java index 9623f08..65c0d0b 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/BingAds.java +++ b/src/main/java/io/kestra/plugin/singer/taps/BingAds.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -107,12 +108,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-bing-ads"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-bing-ads")); } @Override - protected String command() { - return "tap-bing-ads"; + protected Property command() { + return Property.of("tap-bing-ads"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/ChargeBee.java b/src/main/java/io/kestra/plugin/singer/taps/ChargeBee.java index 67c0b53..e605219 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/ChargeBee.java +++ b/src/main/java/io/kestra/plugin/singer/taps/ChargeBee.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -84,12 +85,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/hotgluexyz/tap-chargebee.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/hotgluexyz/tap-chargebee.git")); } @Override - protected String command() { - return "tap-chargebee"; + protected Property command() { + return Property.of("tap-chargebee"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/ExchangeRateHost.java b/src/main/java/io/kestra/plugin/singer/taps/ExchangeRateHost.java index f36e421..4ce4e54 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/ExchangeRateHost.java +++ b/src/main/java/io/kestra/plugin/singer/taps/ExchangeRateHost.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -69,12 +70,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-exchangeratehost"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-exchangeratehost")); } @Override - protected String command() { - return "tap-exchangeratehost"; + protected Property command() { + return Property.of("tap-exchangeratehost"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/FacebookAds.java b/src/main/java/io/kestra/plugin/singer/taps/FacebookAds.java index 4fe0659..9394693 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/FacebookAds.java +++ b/src/main/java/io/kestra/plugin/singer/taps/FacebookAds.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -90,12 +91,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-facebook"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-facebook")); } @Override - protected String command() { - return "tap-facebook"; + protected Property command() { + return Property.of("tap-facebook"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Fastly.java b/src/main/java/io/kestra/plugin/singer/taps/Fastly.java index 9996cc8..1ef3a45 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Fastly.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Fastly.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -65,12 +66,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://gitlab.com/meltano/tap-fastly.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://gitlab.com/meltano/tap-fastly.git")); } @Override - protected String command() { - return "tap-fastly"; + protected Property command() { + return Property.of("tap-fastly"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/GenericTap.java b/src/main/java/io/kestra/plugin/singer/taps/GenericTap.java index 34961db..0e74b46 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GenericTap.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GenericTap.java @@ -2,6 +2,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -27,15 +28,13 @@ public class GenericTap extends AbstractPythonTap implements RunnableTask pipPackages; + private Property> pipPackages; @NotNull @Schema( title = "The command to start." ) - @PluginProperty - private String command; + private Property command; @NotNull @Schema( @@ -66,12 +65,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { + public Property> pipPackages() { return this.pipPackages; } @Override - protected String command() { + protected Property command() { return this.command; } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/GitHub.java b/src/main/java/io/kestra/plugin/singer/taps/GitHub.java index d35fdf1..2ac19fc 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GitHub.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GitHub.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.JacksonMapper; @@ -107,12 +108,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-github"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-github")); } @Override - protected String command() { - return "tap-github"; + protected Property command() { + return Property.of("tap-github"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Gitlab.java b/src/main/java/io/kestra/plugin/singer/taps/Gitlab.java index 6042142..63033b0 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Gitlab.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Gitlab.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -123,12 +124,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://gitlab.com/meltano/tap-gitlab.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://gitlab.com/meltano/tap-gitlab.git")); } @Override - protected String command() { - return "tap-gitlab"; + protected Property command() { + return Property.of("tap-gitlab"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/GoogleAdwords.java b/src/main/java/io/kestra/plugin/singer/taps/GoogleAdwords.java index 65cdd5e..11bcede 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GoogleAdwords.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GoogleAdwords.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -137,12 +138,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://gitlab.com/meltano/tap-adwords.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://gitlab.com/meltano/tap-adwords.git")); } @Override - protected String command() { - return "tap-adwords"; + protected Property command() { + return Property.of("tap-adwords"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/GoogleAnalytics.java b/src/main/java/io/kestra/plugin/singer/taps/GoogleAnalytics.java index 2619086..7862791 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GoogleAnalytics.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GoogleAnalytics.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -138,13 +139,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://gitlab.com/meltano/tap-google-analytics.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://gitlab.com/meltano/tap-google-analytics.git")); } @Override - protected String command() { - return "tap-google-analytics"; + protected Property command() { + return Property.of("tap-google-analytics"); } @Getter diff --git a/src/main/java/io/kestra/plugin/singer/taps/GoogleSearchConsole.java b/src/main/java/io/kestra/plugin/singer/taps/GoogleSearchConsole.java index 2648bad..053dd56 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GoogleSearchConsole.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GoogleSearchConsole.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -101,12 +102,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-google-search-console"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-google-search-console")); } @Override - protected String command() { - return "tap-google-search-console"; + protected Property command() { + return Property.of("tap-google-search-console"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/HubSpot.java b/src/main/java/io/kestra/plugin/singer/taps/HubSpot.java index ef16a53..57b7ec6 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/HubSpot.java +++ b/src/main/java/io/kestra/plugin/singer/taps/HubSpot.java @@ -4,6 +4,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.JacksonMapper; @@ -110,12 +111,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/potloc/tap-hubspot.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/potloc/tap-hubspot.git")); } @Override - protected String command() { - return "tap-hubspot"; + protected Property command() { + return Property.of("tap-hubspot"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Marketo.java b/src/main/java/io/kestra/plugin/singer/taps/Marketo.java index e70ac8f..1749798 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Marketo.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Marketo.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -93,12 +94,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://gitlab.com/meltano/tap-marketo.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://gitlab.com/meltano/tap-marketo.git")); } @Override - protected String command() { - return "tap-marketo"; + protected Property command() { + return Property.of("tap-marketo"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Netsuite.java b/src/main/java/io/kestra/plugin/singer/taps/Netsuite.java index 6d7700d..1c5441f 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Netsuite.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Netsuite.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -124,12 +125,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/hotgluexyz/tap-netsuite"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/hotgluexyz/tap-netsuite")); } @Override - protected String command() { - return "tap-netsuite"; + protected Property command() { + return Property.of("tap-netsuite"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDb.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDb.java index 57b2f49..466d15c 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDb.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDb.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -122,13 +123,13 @@ public List features() { } @Override - public List pipPackages() { - return Collections.singletonList("pipelinewise-tap-mongodb"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("pipelinewise-tap-mongodb")); } @Override - protected String command() { - return "tap-mongodb"; + protected Property command() { + return Property.of("tap-mongodb"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMysql.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMysql.java index 79fba1a..20c21d7 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMysql.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMysql.java @@ -5,6 +5,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -140,12 +141,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("pipelinewise-tap-mysql"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("pipelinewise-tap-mysql")); } @Override - protected String command() { - return "tap-mysql"; + protected Property command() { + return Property.of("tap-mysql"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseOracle.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseOracle.java index fe879fd..3d7bd6a 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseOracle.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseOracle.java @@ -5,6 +5,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -117,12 +118,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("pipelinewise-tap-oracle"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("pipelinewise-tap-oracle")); } @Override - protected String command() { - return "tap-oracle"; + protected Property command() { + return Property.of("tap-oracle"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewisePostgres.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewisePostgres.java index 15a37cc..baf845f 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewisePostgres.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewisePostgres.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -132,12 +133,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return List.of("pipelinewise-tap-postgres"); + public Property> pipPackages() { + return Property.of(List.of("pipelinewise-tap-postgres")); } @Override - protected String command() { - return "tap-postgres"; + protected Property command() { + return Property.of("tap-postgres"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServer.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServer.java index 8e674c1..9671198 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServer.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServer.java @@ -5,6 +5,7 @@ import io.kestra.core.models.annotations.Example; import io.kestra.core.models.annotations.Plugin; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -168,12 +169,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-mssql==2.3.1"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-mssql==2.3.1")); } @Override - protected String command() { - return "tap-mssql"; + protected Property command() { + return Property.of("tap-mssql"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Quickbooks.java b/src/main/java/io/kestra/plugin/singer/taps/Quickbooks.java index 22bfbc1..d33e149 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Quickbooks.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Quickbooks.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -125,12 +126,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-quickbooks"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-quickbooks")); } @Override - protected String command() { - return "tap-quickbooks"; + protected Property command() { + return Property.of("tap-quickbooks"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Recharge.java b/src/main/java/io/kestra/plugin/singer/taps/Recharge.java index 47f627c..f88737c 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Recharge.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Recharge.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -74,12 +75,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-recharge"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-recharge")); } @Override - protected String command() { - return "tap-recharge"; + protected Property command() { + return Property.of("tap-recharge"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/SageIntacct.java b/src/main/java/io/kestra/plugin/singer/taps/SageIntacct.java index b260257..f481898 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/SageIntacct.java +++ b/src/main/java/io/kestra/plugin/singer/taps/SageIntacct.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -98,12 +99,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/hotgluexyz/tap-intacct.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/hotgluexyz/tap-intacct.git")); } @Override - protected String command() { - return "tap-intacct"; + protected Property command() { + return Property.of("tap-intacct"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Salesforce.java b/src/main/java/io/kestra/plugin/singer/taps/Salesforce.java index aba0ac3..43cab16 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Salesforce.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Salesforce.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -166,13 +167,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://gitlab.com/meltano/tap-salesforce.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://gitlab.com/meltano/tap-salesforce.git")); } @Override - protected String command() { - return "tap-salesforce"; + protected Property command() { + return Property.of("tap-salesforce"); } public enum ApiType { diff --git a/src/main/java/io/kestra/plugin/singer/taps/Shopify.java b/src/main/java/io/kestra/plugin/singer/taps/Shopify.java index 6e776f8..2d46401 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Shopify.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Shopify.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -72,12 +73,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-shopify"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-shopify")); } @Override - protected String command() { - return "tap-shopify"; + protected Property command() { + return Property.of("tap-shopify"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Slack.java b/src/main/java/io/kestra/plugin/singer/taps/Slack.java index 9557b07..da7bbd9 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Slack.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Slack.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -119,12 +120,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/Mashey/tap-slack.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/Mashey/tap-slack.git")); } @Override - protected String command() { - return "tap-slack"; + protected Property command() { + return Property.of("tap-slack"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Stripe.java b/src/main/java/io/kestra/plugin/singer/taps/Stripe.java index a499f5c..1657066 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Stripe.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Stripe.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -75,12 +76,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/meltano/tap-stripe.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/meltano/tap-stripe.git")); } @Override - protected String command() { - return "tap-stripe"; + protected Property command() { + return Property.of("tap-stripe"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Zendesk.java b/src/main/java/io/kestra/plugin/singer/taps/Zendesk.java index b76e67b..616a7a1 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Zendesk.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Zendesk.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -97,12 +98,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("tap-zendesk"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("tap-zendesk")); } @Override - protected String command() { - return "tap-zendesk"; + protected Property command() { + return Property.of("tap-zendesk"); } } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Zoom.java b/src/main/java/io/kestra/plugin/singer/taps/Zoom.java index 421cda6..d207117 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Zoom.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Zoom.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.models.Feature; @@ -83,12 +84,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/mashey/tap-zoom.git"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/mashey/tap-zoom.git")); } @Override - protected String command() { - return "tap-zoom"; + protected Property command() { + return Property.of("tap-zoom"); } } diff --git a/src/main/java/io/kestra/plugin/singer/targets/AbstractPythonTarget.java b/src/main/java/io/kestra/plugin/singer/targets/AbstractPythonTarget.java index f6a2a50..1d7c57e 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/AbstractPythonTarget.java +++ b/src/main/java/io/kestra/plugin/singer/targets/AbstractPythonTarget.java @@ -53,7 +53,7 @@ protected AbstractPythonTarget.Output runTarget(RunContext runContext) throws Ex AbstractPythonTarget.Output.OutputBuilder builder = AbstractPythonTarget.Output.builder(); if (!this.stateRecords.isEmpty()) { - builder.stateKey(this.saveState(runContext, runContext.render(this.stateName), this.stateRecords)); + builder.stateKey(this.saveState(runContext, runContext.render(this.stateName).as(String.class).orElseThrow(), this.stateRecords)); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/targets/AdswerveBigQuery.java b/src/main/java/io/kestra/plugin/singer/targets/AdswerveBigQuery.java index 0cfe8d2..12c7a20 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/AdswerveBigQuery.java +++ b/src/main/java/io/kestra/plugin/singer/targets/AdswerveBigQuery.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -161,13 +162,13 @@ protected Map environmentVariables(RunContext runContext) throws } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/kestra-io/target-bigquery.git@fix"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/kestra-io/target-bigquery.git@fix")); } @Override - protected String command() { - return "target-bigquery"; + protected Property command() { + return Property.of("target-bigquery"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/Csv.java b/src/main/java/io/kestra/plugin/singer/targets/Csv.java index 8df4d32..7f645ca 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/Csv.java +++ b/src/main/java/io/kestra/plugin/singer/targets/Csv.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -66,13 +67,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("git+https://github.com/hotgluexyz/target-csv.git@0.3.6"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("git+https://github.com/hotgluexyz/target-csv.git@0.3.6")); } @Override - protected String command() { - return "target-csv"; + protected Property command() { + return Property.of("target-csv"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/DatamillCoPostgres.java b/src/main/java/io/kestra/plugin/singer/targets/DatamillCoPostgres.java index 12de343..40e5250 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/DatamillCoPostgres.java +++ b/src/main/java/io/kestra/plugin/singer/targets/DatamillCoPostgres.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -184,13 +185,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("singer-target-postgres"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("singer-target-postgres")); } @Override - protected String command() { - return "target-postgres"; + protected Property command() { + return Property.of("target-postgres"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/GenericTarget.java b/src/main/java/io/kestra/plugin/singer/targets/GenericTarget.java index fe384c1..de7cd96 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/GenericTarget.java +++ b/src/main/java/io/kestra/plugin/singer/targets/GenericTarget.java @@ -2,6 +2,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -29,15 +30,13 @@ public class GenericTarget extends AbstractPythonTarget implements RunnableTask< @Schema( title = "The list of pip package to install." ) - @PluginProperty - private List pipPackages; + private Property> pipPackages; @NotNull @Schema( title = "The command to start." ) - @PluginProperty - private String command; + private Property command; @NotNull @Schema( @@ -53,12 +52,12 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { + public Property> pipPackages() { return this.pipPackages; } @Override - protected String command() { + protected Property command() { return this.command; } diff --git a/src/main/java/io/kestra/plugin/singer/targets/Json.java b/src/main/java/io/kestra/plugin/singer/targets/Json.java index abe4d03..303e937 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/Json.java +++ b/src/main/java/io/kestra/plugin/singer/targets/Json.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -45,13 +46,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("target-jsonl"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("target-jsonl")); } @Override - protected String command() { - return "target-jsonl"; + protected Property command() { + return Property.of("target-jsonl"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java b/src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java index 57ea324..a637085 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java +++ b/src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -134,13 +135,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("meltanolabs-target-snowflake"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("meltanolabs-target-snowflake")); } @Override - protected String command() { - return "target-snowflake"; + protected Property command() { + return Property.of("target-snowflake"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/Oracle.java b/src/main/java/io/kestra/plugin/singer/targets/Oracle.java index 96b2cb6..311f332 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/Oracle.java +++ b/src/main/java/io/kestra/plugin/singer/targets/Oracle.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -135,13 +136,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return List.of("oracledb", "git+https://github.com/kestra-io/target-oracle.git"); + public Property> pipPackages() { + return Property.of(List.of("oracledb", "git+https://github.com/kestra-io/target-oracle.git")); } @Override - protected String command() { - return "target-oracle"; + protected Property command() { + return Property.of("target-oracle"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/PipelinewisePostgres.java b/src/main/java/io/kestra/plugin/singer/targets/PipelinewisePostgres.java index e6df7fe..38d96dd 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/PipelinewisePostgres.java +++ b/src/main/java/io/kestra/plugin/singer/targets/PipelinewisePostgres.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -184,13 +185,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("pipelinewise-target-postgres"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("pipelinewise-target-postgres")); } @Override - protected String command() { - return "target-postgres"; + protected Property command() { + return Property.of("target-postgres"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseRedshift.java b/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseRedshift.java index 85c803a..763e733 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseRedshift.java +++ b/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseRedshift.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -328,13 +329,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("pipelinewise-target-redshift"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("pipelinewise-target-redshift")); } @Override - protected String command() { - return "target-redshift"; + protected Property command() { + return Property.of("target-redshift"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseSnowflake.java b/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseSnowflake.java index 147dacd..a19e1df 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseSnowflake.java +++ b/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseSnowflake.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -411,13 +412,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("pipelinewise-target-snowflake\n"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("pipelinewise-target-snowflake\n")); } @Override - protected String command() { - return "target-snowflake"; + protected Property command() { + return Property.of("target-snowflake"); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/SqlServer.java b/src/main/java/io/kestra/plugin/singer/targets/SqlServer.java index 7843dff..c8df58c 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/SqlServer.java +++ b/src/main/java/io/kestra/plugin/singer/targets/SqlServer.java @@ -3,6 +3,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.models.property.Property; import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; @@ -147,13 +148,13 @@ public Map configuration(RunContext runContext) throws IllegalVa } @Override - public List pipPackages() { - return Collections.singletonList("target-mssql"); + public Property> pipPackages() { + return Property.of(Collections.singletonList("target-mssql")); } @Override - protected String command() { - return "target-mssql"; + protected Property command() { + return Property.of("target-mssql"); } @Override diff --git a/src/test/java/io/kestra/plugin/singer/taps/BigQueryTest.java b/src/test/java/io/kestra/plugin/singer/taps/BigQueryTest.java index 9d5dd5d..f478b38 100644 --- a/src/test/java/io/kestra/plugin/singer/taps/BigQueryTest.java +++ b/src/test/java/io/kestra/plugin/singer/taps/BigQueryTest.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.CharStreams; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -35,10 +36,10 @@ void run() throws Exception { .id(IdUtils.create()) .taskRunner(Docker.instance()) .type(BigQuery.class.getName()) - .serviceAccount(serviceAccount) - .startAlwaysInclusive(false) + .serviceAccount(Property.of(serviceAccount)) + .startAlwaysInclusive(Property.of(false)) .startDateTime(Instant.parse("2013-09-08T16:19:12Z")) - .limit(1) + .limit(Property.of(1)) .streams(Collections.singletonList( BigQuery.Stream.builder() .name("covid19_nyt_us_states") diff --git a/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServerTest.java b/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServerTest.java index 0a79cec..1d28301 100644 --- a/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServerTest.java +++ b/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServerTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.taps; import com.google.common.collect.ImmutableMap; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -34,7 +35,7 @@ void run() throws Exception { .password("SQLServer_Passwd") .port(57037) .filterDbs(Collections.singletonList("dbo")) - .stateName("tap-test") + .stateName(Property.of("tap-test")) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("Categories") diff --git a/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java b/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java index ca67c23..e207faa 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.CharStreams; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -52,7 +53,7 @@ void run() throws Exception { .username("root") .password("mysql_passwd") .port(63306) - .stateName(stateName) + .stateName(Property.of(stateName)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("Category") @@ -81,7 +82,7 @@ void run() throws Exception { .id(IdUtils.create() + "_bq") .type(io.kestra.plugin.singer.targets.AdswerveBigQuery.class.getName()) .from(tapOutput.getRaw().toString()) - .stateName(stateName) + .stateName(Property.of(stateName)) .serviceAccount(serviceAccount) .projectId(project) .datasetId(dataset) diff --git a/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java b/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java index 082664b..d77d9e1 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.targets; import com.google.common.collect.ImmutableMap; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -36,7 +37,7 @@ void run() throws Exception { .username("root") .password("mysql_passwd") .port(63306) - .stateName(stateName) + .stateName(Property.of(stateName)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("Category") @@ -63,7 +64,7 @@ void run() throws Exception { .id(IdUtils.create()) .type(DatamillCoPostgres.class.getName()) .from(tapOutput.getRaw().toString()) - .stateName(stateName) + .stateName(Property.of(stateName)) .delimiter(";"); Csv target = builder.build(); diff --git a/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java b/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java index f58572c..b0e02f4 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.targets; import com.google.common.collect.ImmutableMap; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -37,7 +38,7 @@ void run() throws Exception { .username("root") .password("mysql_passwd") .port(63306) - .stateName(stateName) + .stateName(Property.of(stateName)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("Category") @@ -66,7 +67,7 @@ void run() throws Exception { .id(IdUtils.create()) .type(io.kestra.plugin.singer.targets.DatamillCoPostgres.class.getName()) .from(tapOutput.getRaw().toString()) - .stateName(stateName) + .stateName(Property.of(stateName)) .host("172.17.0.1") .username("postgres") .password("pg_passwd") diff --git a/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java b/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java index a641690..c561a47 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.targets; import com.google.common.collect.ImmutableMap; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -36,7 +37,7 @@ void run() throws Exception { .username("root") .password("mysql_passwd") .port(63306) - .stateName(stateName) + .stateName(Property.of(stateName)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("Category") @@ -65,7 +66,7 @@ void run() throws Exception { .id(IdUtils.create()) .type(DatamillCoPostgres.class.getName()) .from(tapOutput.getRaw().toString()) - .stateName(stateName) + .stateName(Property.of(stateName)) .build(); runContext = TestsUtils.mockRunContext(runContextFactory, task, ImmutableMap.of()); diff --git a/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java b/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java index a68376e..d0dd406 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.models.executions.AbstractMetricEntry; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -38,7 +39,7 @@ void run() throws Exception { .password("SQLServer_Passwd") .port(57037) .filterDbs(Collections.singletonList("dbo")) - .stateName("before-target-test") + .stateName(Property.of("before-target-test")) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("Categories") @@ -84,7 +85,7 @@ void run() throws Exception { tap = tapBuilder .filterDbs(Collections.singletonList("target")) - .stateName("after-target-test") + .stateName(Property.of("after-target-test")) .streamsConfigurations(Collections.singletonList( StreamsConfiguration.builder() // SQL Server target transforms table & columns names to snake_case From 665964bbd9ad34422c137d6db3e911bcecfaf7e3 Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle Date: Tue, 17 Dec 2024 14:06:53 +0100 Subject: [PATCH 2/3] refactor: migrate target migrate abstract target AdswerveBigQuery DatamillCoPostgres GenericTarget MeltanoSnowflake Oracle --- .../singer/targets/AbstractPythonTarget.java | 7 +- .../singer/targets/AdswerveBigQuery.java | 52 +++--- .../singer/targets/DatamillCoPostgres.java | 75 ++++---- .../plugin/singer/targets/GenericTarget.java | 4 +- .../singer/targets/MeltanoSnowflake.java | 30 ++-- .../kestra/plugin/singer/targets/Oracle.java | 40 ++--- .../singer/targets/PipelinewisePostgres.java | 70 +++----- .../singer/targets/PipelinewiseRedshift.java | 123 ++++++------- .../singer/targets/PipelinewiseSnowflake.java | 165 +++++++----------- .../plugin/singer/targets/SqlServer.java | 40 ++--- .../singer/targets/AdswerveBigQueryTest.java | 6 +- .../kestra/plugin/singer/targets/CsvTest.java | 2 +- .../targets/DatamillCoPostgresTest.java | 8 +- .../plugin/singer/targets/JsonTest.java | 2 +- .../plugin/singer/targets/OracleTest.java | 7 +- .../plugin/singer/targets/SqlServerTest.java | 6 +- 16 files changed, 261 insertions(+), 376 deletions(-) diff --git a/src/main/java/io/kestra/plugin/singer/targets/AbstractPythonTarget.java b/src/main/java/io/kestra/plugin/singer/targets/AbstractPythonTarget.java index 1d7c57e..304ccbf 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/AbstractPythonTarget.java +++ b/src/main/java/io/kestra/plugin/singer/targets/AbstractPythonTarget.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.targets; import com.fasterxml.jackson.core.type.TypeReference; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.plugin.singer.AbstractPythonSinger; import io.swagger.v3.oas.annotations.media.Schema; @@ -37,11 +38,11 @@ public abstract class AbstractPythonTarget extends AbstractPythonSinger { ) @NotNull @Valid - private String from; + private Property from; protected AbstractPythonTarget.Output runTarget(RunContext runContext) throws Exception { // from - URI from = new URI(runContext.render(this.from)); + URI from = new URI(runContext.render(this.from).as(String.class).orElseThrow()); Path tempFile = runContext.workingDir().createTempFile(); Files.copy(runContext.storage().getFile(from), tempFile, StandardCopyOption.REPLACE_EXISTING); @@ -68,7 +69,7 @@ protected void tapsSync(Path tempFile, RunContext runContext) throws Exception { this.runSinger(commands, runContext); } - protected void runSinger(List commands, RunContext runContext) throws Exception { + protected void runSinger(List commands, RunContext runContext) throws Exception { Flux .create( throwConsumer(emitter -> { diff --git a/src/main/java/io/kestra/plugin/singer/targets/AdswerveBigQuery.java b/src/main/java/io/kestra/plugin/singer/targets/AdswerveBigQuery.java index 12c7a20..bdadf77 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/AdswerveBigQuery.java +++ b/src/main/java/io/kestra/plugin/singer/targets/AdswerveBigQuery.java @@ -47,8 +47,7 @@ public class AdswerveBigQuery extends AbstractPythonTarget implements RunnableTa @Schema( title = "The Dataset location." ) - @PluginProperty(dynamic = true) - private String location; + private Property location; @NotNull @Schema( @@ -56,62 +55,54 @@ public class AdswerveBigQuery extends AbstractPythonTarget implements RunnableTa description = "This option is disabled by default and invalid RECORD messages will fail only at load time by " + "Postgres. Enabling this option will detect invalid records earlier but could cause performance degradation.." ) - @PluginProperty - private final Boolean validateRecords = false; + private final Property validateRecords = Property.of(false); @Schema( title = "Add singer Metadata columns.", description = "Add `_time_extracted` and `_time_loaded` metadata columns." ) - @PluginProperty @Builder.Default - private final Boolean addMetadataColumns = false; + private final Property addMetadataColumns = Property.of(false); @Schema( title = "The replication method, `append` or `truncate`." ) - @PluginProperty @Builder.Default - private final ReplicationMethod replicationMethod = ReplicationMethod.append; + private final Property replicationMethod = Property.of(ReplicationMethod.append); @Schema( title = "Add prefix to table name." ) - @PluginProperty(dynamic = true) - private String tablePrefix; + private Property tablePrefix; @Schema( title = "Add suffix to table name." ) - @PluginProperty(dynamic = true) - private String tableSuffix; + private Property tableSuffix; @Schema( title = "Maximum cache size in MB." ) @PluginProperty @Builder.Default - private final Integer maxCache = 50; + private final Property maxCache = Property.of(50); @Schema( title = "The JSON service account key as string." ) - @PluginProperty(dynamic = true) - protected String serviceAccount; + protected Property serviceAccount; @Schema( title = "Enable control state flush.", description = "default: merges multiple state messages from the tap into the state file, if true : uses the last state message as the state file." ) - @PluginProperty @Builder.Default - protected Boolean mergeStateMessages = false; + protected Property mergeStateMessages = Property.of(false); @Schema( title = "Table configs." ) - @PluginProperty - protected Map tableConfigs; + protected Property> tableConfigs; @SneakyThrows @Override @@ -119,29 +110,30 @@ public Map configuration(RunContext runContext) throws IllegalVa ImmutableMap.Builder builder = ImmutableMap.builder() .put("project_id", runContext.render(this.projectId)) .put("dataset_id", runContext.render(this.datasetId)) - .put("validate_records", this.validateRecords) - .put("add_metadata_columns", this.addMetadataColumns) - .put("replication_method", this.replicationMethod) - .put("max_cache", this.maxCache); + .put("validate_records", runContext.render(this.validateRecords).as(Boolean.class).orElseThrow()) + .put("add_metadata_columns", runContext.render(this.addMetadataColumns).as(Boolean.class).orElseThrow()) + .put("replication_method", runContext.render(this.replicationMethod).as(ReplicationMethod.class).orElseThrow()) + .put("max_cache", runContext.render(this.maxCache).as(Integer.class).orElseThrow()); if (this.location != null) { - builder.put("location", runContext.render(this.location)); + builder.put("location", runContext.render(this.location).as(String.class).orElseThrow()); } if (this.tablePrefix != null) { - builder.put("table_prefix", runContext.render(this.tablePrefix)); + builder.put("table_prefix", runContext.render(this.tablePrefix).as(String.class).orElseThrow()); } if (this.tableSuffix != null) { - builder.put("table_suffix", runContext.render(this.tableSuffix)); + builder.put("table_suffix", runContext.render(this.tableSuffix).as(String.class).orElseThrow()); } - if (this.mergeStateMessages) { + if (runContext.render(this.mergeStateMessages).as(Boolean.class).orElseThrow()) { builder.put("merge_state_messages", "0"); } - if (this.tableConfigs != null) { - this.writeSingerFiles("table-config.json", runContext.render(this.tableConfigs)); + var renderedConfigs = runContext.render(this.tableConfigs).asMap(String.class, Object.class); + if (!renderedConfigs.isEmpty()) { + this.writeSingerFiles("table-config.json", renderedConfigs); builder.put("table_config", workingDirectory.toAbsolutePath() + "/table-config.json"); } @@ -154,7 +146,7 @@ protected Map environmentVariables(RunContext runContext) throws HashMap env = new HashMap<>(super.environmentVariables(runContext)); if (this.serviceAccount != null) { - this.writeSingerFiles("google-credentials.json", runContext.render(this.serviceAccount)); + this.writeSingerFiles("google-credentials.json", runContext.render(this.serviceAccount).as(String.class).orElseThrow()); env.put("GOOGLE_APPLICATION_CREDENTIALS", workingDirectory.toAbsolutePath() + "/google-credentials.json"); } diff --git a/src/main/java/io/kestra/plugin/singer/targets/DatamillCoPostgres.java b/src/main/java/io/kestra/plugin/singer/targets/DatamillCoPostgres.java index 40e5250..b199902 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/DatamillCoPostgres.java +++ b/src/main/java/io/kestra/plugin/singer/targets/DatamillCoPostgres.java @@ -45,78 +45,67 @@ public class DatamillCoPostgres extends AbstractPythonTarget implements Runnable @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @Schema( title = "The database name." ) - @PluginProperty(dynamic = true) - private String dbName; + private Property dbName; @NotNull @Schema( title = "The database port" ) - @PluginProperty - private Integer port; + private Property port; @Schema( title = "The database schema." ) - @PluginProperty(dynamic = true) @Builder.Default - private final String schema = "public"; + private final Property schema = Property.of("public"); @Schema( title = "Refer to the [libpq](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS) docs for more information about SSL." ) - @PluginProperty(dynamic = true) @Builder.Default - private final String sslMode = "prefer"; + private final Property sslMode = Property.of("prefer"); @Schema( title = "Crash on invalid records." ) - @PluginProperty @Builder.Default - private final Boolean invalidRecordsDetect = true; + private final Property invalidRecordsDetect = Property.of(true); @Schema( title = "Include a positive value n in your config to allow to encounter at most n invalid records per stream before giving up." ) - @PluginProperty @Builder.Default - private final Integer invalidRecordsThreshold = 0; + private final Property invalidRecordsThreshold = Property.of(0); @Schema( title = "The level for logging.", description = "Set to DEBUG to get things like queries executed, timing of those queries, etc. See Python's Logger Levels for information about valid values." ) - @PluginProperty(dynamic = true) @Builder.Default - private final String loggingLevel = "INFO"; + private final Property loggingLevel = Property.of("INFO"); @Schema( title = "Whether the Target should create tables which have no records present in Remote." ) - @PluginProperty @Builder.Default - private final Boolean persistEmptyTables = false; + private final Property persistEmptyTables = Property.of(false); @Schema( title = "The maximum number of rows to buffer in memory before writing to the destination table in Postgres." ) - @PluginProperty @Builder.Default - private final Integer maxBatchRows = 200000; + private final Property maxBatchRows = Property.of(200000); @Schema( title = "The maximum number of bytes to buffer in memory before writing to the destination table in Postgres." ) - @PluginProperty @Builder.Default - private final Integer maxBufferSize = 104857600; + private final Property maxBufferSize = Property.of(104857600); @Schema( title = "How often, in rows received, to count the buffered rows and bytes to check if a flush is necessary.", @@ -125,60 +114,56 @@ public class DatamillCoPostgres extends AbstractPythonTarget implements Runnable " to set as the default is dynamically adjusted to check reasonably often. \n\n" + "Default is 5000, or 1/40th `maxBatchRows`" ) - @PluginProperty - private Integer batchDetectionThreshold; + private Property batchDetectionThreshold; @Schema( title = "Whether the Target should create column indexes on the important columns used during data loading.", description = "These indexes will make data loading slightly slower but the deduplication phase much faster. Defaults to on for better baseline performance." ) - @PluginProperty @Builder.Default - private final Boolean addUpsertIndexes = true; + private final Property addUpsertIndexes = Property.of(true); @Schema( title = "Raw SQL statement(s) to execute as soon as the connection to Postgres is opened by the target.", description = "Useful for setup like SET ROLE or other connection state that is important." ) - @PluginProperty - private String beforeRunSql; + private Property beforeRunSql; @Schema( title = "Raw SQL statement(s) to before closing the connection to Postgres." ) - @PluginProperty - private String afterRunSql; + private Property afterRunSql; @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("postgres_username", runContext.render(this.username)) - .put("postgres_password", runContext.render(this.password)) + .put("postgres_password", runContext.render(this.password).as(String.class).orElse(null)) .put("postgres_host", runContext.render(this.host)) - .put("postgres_port", this.port) - .put("postgres_database", runContext.render(this.dbName)) - .put("postgres_schema", runContext.render(this.schema)) - .put("postgres_sslmode", this.sslMode) - .put("invalid_records_detect", this.invalidRecordsDetect) - .put("invalid_records_threshold", this.invalidRecordsThreshold) + .put("postgres_port", runContext.render(this.port).as(Integer.class).orElseThrow()) + .put("postgres_database", runContext.render(this.dbName).as(String.class).orElse(null)) + .put("postgres_schema", runContext.render(this.schema).as(String.class).orElseThrow()) + .put("postgres_sslmode", runContext.render(this.sslMode).as(String.class).orElseThrow()) + .put("invalid_records_detect", runContext.render(this.invalidRecordsDetect).as(Boolean.class).orElseThrow()) + .put("invalid_records_threshold", runContext.render(this.invalidRecordsThreshold).as(Integer.class).orElseThrow()) .put("disable_collection", true) - .put("logging_level", this.loggingLevel) - .put("persist_empty_tables", this.persistEmptyTables) - .put("max_batch_rows", this.maxBatchRows) - .put("max_buffer_size", this.maxBufferSize) + .put("logging_level", runContext.render(this.loggingLevel).as(String.class).orElseThrow()) + .put("persist_empty_tables", runContext.render(this.persistEmptyTables).as(Boolean.class).orElseThrow()) + .put("max_batch_rows", runContext.render(this.maxBatchRows).as(Integer.class).orElseThrow()) + .put("max_buffer_size", runContext.render(this.maxBufferSize).as(Integer.class).orElseThrow()) .put("state_support", true) - .put("add_upsert_indexes", this.addUpsertIndexes); + .put("add_upsert_indexes", runContext.render(this.addUpsertIndexes).as(Boolean.class).orElseThrow()); if (this.batchDetectionThreshold != null) { - builder.put("batch_detection_threshold", this.batchDetectionThreshold); + builder.put("batch_detection_threshold", runContext.render(this.batchDetectionThreshold).as(Integer.class).orElseThrow()); } if (this.beforeRunSql != null) { - builder.put("before_run_sql", runContext.render(this.beforeRunSql)); + builder.put("before_run_sql", runContext.render(this.beforeRunSql).as(String.class).orElseThrow()); } if (this.afterRunSql != null) { - builder.put("after_run_sql", runContext.render(this.afterRunSql)); + builder.put("after_run_sql", runContext.render(this.afterRunSql).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/targets/GenericTarget.java b/src/main/java/io/kestra/plugin/singer/targets/GenericTarget.java index de7cd96..f58c26e 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/GenericTarget.java +++ b/src/main/java/io/kestra/plugin/singer/targets/GenericTarget.java @@ -44,11 +44,11 @@ public class GenericTarget extends AbstractPythonTarget implements RunnableTask< description = "Will be save on config.json and used as arguments" ) @PluginProperty(dynamic = true) - private Map configs; + private Property> configs; @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException, IOException { - return runContext.render(this.configs); + return runContext.render(this.configs).asMap(String.class, Object.class); } @Override diff --git a/src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java b/src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java index a637085..2d28d1a 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java +++ b/src/main/java/io/kestra/plugin/singer/targets/MeltanoSnowflake.java @@ -54,8 +54,7 @@ public class MeltanoSnowflake extends AbstractPythonTarget implements RunnableTa @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @NotNull @NotEmpty @@ -76,33 +75,28 @@ public class MeltanoSnowflake extends AbstractPythonTarget implements RunnableTa @Schema( title = "The initial role for the session." ) - @PluginProperty(dynamic = true) - private String role; + private Property role; @Schema( title = "Whether to add metadata columns." ) - @PluginProperty @Builder.Default - private Boolean addRecordMetadata = true; + private Property addRecordMetadata = Property.of(true); @Schema( title = "The default target database schema name to use for all streams." ) - @PluginProperty(dynamic = true) - private String defaultTargetSchema; + private Property defaultTargetSchema; @Schema( title = "'True' to enable schema flattening and automatically expand nested properties." ) - @PluginProperty - private Boolean flatteningEnabled; + private Property flatteningEnabled; @Schema( title = "The max depth to flatten schemas." ) - @PluginProperty - private Integer flatteningMaxDepth; + private Property flatteningMaxDepth; @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { @@ -110,25 +104,25 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("account", runContext.render(this.account)) .put("database", runContext.render(this.database)) .put("username", runContext.render(this.username)) - .put("password", runContext.render(this.password)) + .put("password", runContext.render(this.password).as(String.class).orElseThrow()) .put("warehouse", runContext.render(this.warehouse)) .put("schema", runContext.render(this.schema)) - .put("add_record_metadata", this.addRecordMetadata.toString()); + .put("add_record_metadata", runContext.render(this.addRecordMetadata).as(Boolean.class).orElseThrow().toString()); if (this.role != null) { - builder.put("role", runContext.render(this.role)); + builder.put("role", runContext.render(this.role).as(String.class).orElseThrow()); } if (this.defaultTargetSchema != null) { - builder.put("default_target_schema", runContext.render(this.defaultTargetSchema)); + builder.put("default_target_schema", runContext.render(this.defaultTargetSchema).as(String.class).orElseThrow()); } if (this.flatteningEnabled != null) { - builder.put("flattening_enabled", this.flatteningEnabled.toString()); + builder.put("flattening_enabled", runContext.render(this.flatteningEnabled).as(Boolean.class).orElseThrow().toString()); } if (this.flatteningMaxDepth != null) { - builder.put("flattening_max_depth", this.flatteningMaxDepth); + builder.put("flattening_max_depth", runContext.render(this.flatteningMaxDepth).as(Integer.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/targets/Oracle.java b/src/main/java/io/kestra/plugin/singer/targets/Oracle.java index 311f332..08f2e80 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/Oracle.java +++ b/src/main/java/io/kestra/plugin/singer/targets/Oracle.java @@ -39,15 +39,13 @@ public class Oracle extends AbstractPythonTarget implements RunnableTask database; @NotNull @Schema( title = "The database port." ) - @PluginProperty - private Integer port; + private Property port; @NotEmpty @Schema( @@ -66,38 +64,32 @@ public class Oracle extends AbstractPythonTarget implements RunnableTask driverName; @Schema( title = "Use float data type for numbers (otherwise number type is used)." ) - @PluginProperty - private Boolean preferFloatOverNumeric; + private Property preferFloatOverNumeric; @Schema( title = "Config object for stream maps capability." ) - @PluginProperty(dynamic = true) - private String streamMaps; + private Property streamMaps; @Schema( title = "User-defined config values to be used within map expressions." ) - @PluginProperty(dynamic = true) - private String streamMapConfig; + private Property streamMapConfig; @Schema( title = "Enable schema flattening and automatically expand nested properties." ) - @PluginProperty - private Boolean flatteningEnabled; + private Property flatteningEnabled; @Schema( title = "The max depth to flatten schemas." ) - @PluginProperty - private Integer flatteningMaxDepth; + private Property flatteningMaxDepth; @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { @@ -105,31 +97,31 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("user", runContext.render(this.username)) .put("password", runContext.render(this.password)) .put("host", runContext.render(this.host)) - .put("port", String.valueOf(this.port)) - .put("database", runContext.render(this.database)); + .put("port", String.valueOf(runContext.render(this.port).as(Integer.class).orElseThrow())) + .put("database", runContext.render(this.database).as(String.class).orElse(null)); if (this.driverName != null) { - builder.put("driver_name", runContext.render(this.driverName)); + builder.put("driver_name", runContext.render(this.driverName).as(String.class).orElseThrow()); } if (this.preferFloatOverNumeric != null) { - builder.put("prefer_float_over_numeric", StringUtils.capitalize(this.preferFloatOverNumeric.toString())); + builder.put("prefer_float_over_numeric", StringUtils.capitalize(runContext.render(this.preferFloatOverNumeric).as(Boolean.class).orElseThrow().toString())); } if (this.streamMaps != null) { - builder.put("stream_maps", runContext.render(this.streamMaps)); + builder.put("stream_maps", runContext.render(this.streamMaps).as(String.class).orElseThrow()); } if (this.streamMapConfig != null) { - builder.put("stream_map_config", runContext.render(this.streamMapConfig)); + builder.put("stream_map_config", runContext.render(this.streamMapConfig).as(String.class).orElseThrow()); } if (this.flatteningEnabled != null) { - builder.put("flattening_enabled", StringUtils.capitalize(this.flatteningEnabled.toString())); + builder.put("flattening_enabled", StringUtils.capitalize(runContext.render(this.flatteningEnabled).as(Boolean.class).orElseThrow().toString())); } if (this.flatteningMaxDepth != null) { - builder.put("flattening_max_depth", this.flatteningMaxDepth); + builder.put("flattening_max_depth", runContext.render(this.flatteningMaxDepth).as(Integer.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/targets/PipelinewisePostgres.java b/src/main/java/io/kestra/plugin/singer/targets/PipelinewisePostgres.java index 38d96dd..416d40f 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/PipelinewisePostgres.java +++ b/src/main/java/io/kestra/plugin/singer/targets/PipelinewisePostgres.java @@ -45,53 +45,46 @@ public class PipelinewisePostgres extends AbstractPythonTarget implements Runnab @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @Schema( title = "The database name." ) - @PluginProperty(dynamic = true) - private String dbName; + private Property dbName; @NotNull @Schema( title = "The database port." ) - @PluginProperty - private Integer port; + private Property port; @Schema( title = "Maximum number of rows in each batch.", description = "At the end of each batch, the rows in the batch are loaded into Postgres." ) - @PluginProperty @Builder.Default - private final Integer batchSizeRows = 100000; + private final Property batchSizeRows = Property.of(100000); @Schema( title = "Flush and load every stream into Postgres when one batch is full.", description = "Warning: This may trigger the COPY command to use files with low number of records.." ) - @PluginProperty @Builder.Default - private final Boolean flushAllStreams = false; + private final Property flushAllStreams = Property.of(false); @Schema( title = "The number of threads used to flush tables.", description = "0 will create a thread for each stream, up to parallelism_max. -1 will create a thread for " + "each CPU core. Any other positive number will create that number of threads, up to parallelism_max." ) - @PluginProperty @Builder.Default - private final Integer parallelism = 0; + private final Property parallelism = Property.of(0); @Schema( title = "Max number of parallel threads to use when flushing tables." ) - @PluginProperty @Builder.Default - private final Integer maxParallelism = 16; + private final Property maxParallelism = Property.of(16); @Schema( title = "Add singer Metadata columns.", @@ -102,9 +95,8 @@ public class PipelinewisePostgres extends AbstractPythonTarget implements Runnab "Enabling metadata columns will flag the deleted rows by setting the `_SDC_DELETED_AT` metadata column. " + "Without the `add_metadata_columns` option the deleted rows from singer taps will not be recognisable in Postgres." ) - @PluginProperty @Builder.Default - private final Boolean addMetadataColumns = false; + private final Property addMetadataColumns = Property.of(false); @Schema( title = "Delete rows on Postgres.", @@ -113,72 +105,66 @@ public class PipelinewisePostgres extends AbstractPythonTarget implements Runnab "by the singer tap. Due to deleting rows requires metadata columns, hard_delete option automatically " + "enables the add_metadata_columns option as well." ) - @PluginProperty @Builder.Default - private final Boolean hardDelete = false; + private final Property hardDelete = Property.of(false); @Schema( title = "Object type RECORD items from taps can be transformed to flattened columns by creating columns automatically.", description = "When value is 0 (default) then flattening functionality is turned off." ) - @PluginProperty @Builder.Default - private final Integer dataFlatteningMaxLevel = 0; + private final Property dataFlatteningMaxLevel = Property.of(0); @Schema( title = "Log based and Incremental replications on tables with no Primary Key cause duplicates when merging UPDATE events.", description = "When set to true, stop loading data if no Primary Key is defined." ) - @PluginProperty @Builder.Default - private final Boolean primaryKeyRequired = true; + private final Property primaryKeyRequired = Property.of(true); @Schema( title = "Validate every single record message to the corresponding JSON schema.", description = "This option is disabled by default and invalid RECORD messages will fail only at load time by " + "Postgres. Enabling this option will detect invalid records earlier but could cause performance degradation.." ) - @PluginProperty @Builder.Default - private final Boolean validateRecords = false; + private final Property validateRecords = Property.of(false); @Schema( title = "Name of the schema where the tables will be created.", description = "If `schemaMapping` is not defined then every stream sent by the tap is loaded into this schema." ) - @PluginProperty(dynamic = true) - private String defaultTargetSchema; + private Property defaultTargetSchema; @Schema( title = "Grant USAGE privilege on newly created schemas and grant SELECT privilege on newly created." ) - @PluginProperty(dynamic = true) - private String defaultTargetSchemaSelectPermission; + private Property defaultTargetSchemaSelectPermission; @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("user", runContext.render(this.username)) - .put("password", runContext.render(this.password)) + .put("password", runContext.render(this.password).as(String.class).orElse(null)) .put("host", runContext.render(this.host)) - .put("port", this.port) - .put("dbname", runContext.render(this.dbName)) - .put("batch_size_rows", this.batchSizeRows) - .put("flush_all_streams", this.flushAllStreams) - .put("parallelism", this.parallelism) - .put("max_parallelism", this.maxParallelism) - .put("add_metadata_columns", this.addMetadataColumns) - .put("hard_delete", this.hardDelete) - .put("data_flattening_max_level", this.dataFlatteningMaxLevel) - .put("primary_key_required", this.primaryKeyRequired) - .put("validate_records", this.validateRecords); + .put("port", runContext.render(this.port).as(Integer.class).orElseThrow()) + .put("dbname", runContext.render(this.dbName).as(String.class).orElse(null)) + .put("batch_size_rows", runContext.render(this.batchSizeRows).as(Integer.class).orElseThrow()) + .put("flush_all_streams", runContext.render(this.flushAllStreams).as(Boolean.class).orElseThrow()) + .put("parallelism", runContext.render(this.parallelism).as(Integer.class).orElseThrow()) + .put("max_parallelism", runContext.render(this.maxParallelism).as(Integer.class).orElseThrow()) + .put("add_metadata_columns", runContext.render(this.addMetadataColumns).as(Boolean.class).orElseThrow()) + .put("hard_delete", runContext.render(this.hardDelete).as(Boolean.class).orElseThrow()) + .put("data_flattening_max_level", runContext.render(this.dataFlatteningMaxLevel).as(Integer.class).orElseThrow()) + .put("primary_key_required", runContext.render(this.primaryKeyRequired).as(Boolean.class).orElseThrow()) + .put("validate_records", runContext.render(this.validateRecords).as(Boolean.class).orElseThrow()); if (this.defaultTargetSchema != null) { - builder.put("default_target_schema", runContext.render(this.defaultTargetSchema)); + builder.put("default_target_schema", runContext.render(this.defaultTargetSchema).as(String.class).orElseThrow()); } if (this.defaultTargetSchema != null) { - builder.put("default_target_schema_select_permission", runContext.render(this.defaultTargetSchemaSelectPermission)); + builder.put("default_target_schema_select_permission", runContext.render(this.defaultTargetSchemaSelectPermission).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseRedshift.java b/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseRedshift.java index 763e733..933c67d 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseRedshift.java +++ b/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseRedshift.java @@ -45,21 +45,18 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @Schema( title = "The database name." ) - @PluginProperty(dynamic = true) - private String dbName; + private Property dbName; @NotNull @Schema( title = "The database port." ) - @PluginProperty - private Integer port; + private Property port; @NotEmpty @NotNull @@ -82,22 +79,19 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab title = "S3 Access Key ID.", description = "Used for S3 and Redshift copy operations." ) - @PluginProperty(dynamic = true) - private String accessKeyId; + private Property accessKeyId; @Schema( title = "S3 Secret Access Key.", description = "Used for S3 and Redshift copy operations." ) - @PluginProperty(dynamic = true) - private String secretAccessKey; + private Property secretAccessKey; @Schema( title = "AWS S3 Session Token.", description = "S3 AWS STS token for temporary credentials." ) - @PluginProperty(dynamic = true) - private String sessionToken; + private Property sessionToken; @Schema( title = "AWS Redshift COPY role ARN.", @@ -105,23 +99,20 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab "Used instead of the given AWS keys for the COPY operation if provided - " + "the keys are still used for other S3 operations." ) - @PluginProperty(dynamic = true) - private String redshiftCopyRoleArn; + private Property redshiftCopyRoleArn; @Schema( title = "AWS S3 ACL.", description = "S3 Object ACL." ) - @PluginProperty(dynamic = true) - private String s3Acl; + private Property s3Acl; @Schema( title = "S3 Key Prefix.", description = "A static prefix before the generated S3 key names. " + "Using prefixes you can upload files into specific directories in the S3 bucket. Default(None)." ) - @PluginProperty(dynamic = true) - private String s3KeyPrefix; + private Property s3KeyPrefix; @Schema( title = "COPY options.", @@ -129,47 +120,41 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab "Some basic file formatting parameters are fixed values and not recommended overriding them by custom values. " + "They are like: `CSV GZIP DELIMITER ',' REMOVEQUOTES ESCAPE`." ) - @PluginProperty(dynamic = true) - private String copyOptions; + private Property copyOptions; @Schema( title = "Maximum number of rows in each batch.", description = "At the end of each batch, the rows in the batch are loaded into Redshift." ) - @PluginProperty @Builder.Default - private final Integer batchSizeRows = 100000; + private final Property batchSizeRows = Property.of(100000); @Schema( title = "Flush and load every stream into Redshift when one batch is full.", description = "Warning: This may trigger the COPY command to use files with low number of records.." ) - @PluginProperty @Builder.Default - private final Boolean flushAllStreams = false; + private final Property flushAllStreams = Property.of(false); @Schema( title = "The number of threads used to flush tables.", description = "0 will create a thread for each stream, up to parallelism_max. -1 will create a thread for " + "each CPU core. Any other positive number will create that number of threads, up to parallelism_max." ) - @PluginProperty @Builder.Default - private final Integer parallelism = 0; + private final Property parallelism = Property.of(0); @Schema( title = "Max number of parallel threads to use when flushing tables." ) - @PluginProperty @Builder.Default - private final Integer maxParallelism = 16; + private final Property maxParallelism = Property.of(16); @Schema( title = "Grant USAGE privilege on newly created schemas and grant SELECT privilege on newly created tables to a specific list of users or groups.", description = "If `schemaMapping` is not defined then every stream sent by the tap is granted accordingly." ) - @PluginProperty(dynamic = true) - private String defaultTargetSchemaSelectPermissions; + private Property defaultTargetSchemaSelectPermissions; @Schema( title = "Schema mapping.", @@ -179,8 +164,7 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab "`default_target_schema_select_permissions` value to grant SELECT permissions to different groups per " + "schemas or optionally you can create indices automatically for the replicated tables." ) - @PluginProperty - private String schema_mapping; + private Property schema_mapping; @Schema( title = "Disable table cache.", @@ -189,9 +173,8 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab "tables is required. With disable_table_cache option you can turn off this caching. You will always " + "see the most recent table structures but will cause an extra query runtime." ) - @PluginProperty @Builder.Default - private final Boolean disableTableCache = false; + private final Property disableTableCache = Property.of(false); @Schema( title = "Add metadata columns.", @@ -202,9 +185,8 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab "Enabling metadata columns will flag the deleted rows by setting the _SDC_DELETED_AT metadata column. " + "Without the `addMetadataColumns` option the deleted rows from singer taps will not be recongisable in Redshift." ) - @PluginProperty @Builder.Default - private final Boolean addMetadataColumns = false; + private final Property addMetadataColumns = Property.of(false); @Schema( title = "Delete rows on Redshift.", @@ -213,9 +195,8 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab "singer tap. Due to deleting rows requires metadata columns, `hardDelete` option automatically enables " + "the `addMetadataColumns` option as well." ) - @PluginProperty @Builder.Default - private final Boolean hardDelete = false; + private final Property hardDelete = Property.of(false); @Schema( title = "Object type RECORD items from taps can be transformed to flattened columns by creating columns automatically.", @@ -224,105 +205,99 @@ public class PipelinewiseRedshift extends AbstractPythonTarget implements Runnab "singer tap. Due to deleting rows requires metadata columns, `hardDelete` option automatically " + "enables the `addMetadataColumns` option as well.." ) - @PluginProperty @Builder.Default - private final Integer dataFlatteningMaxLevel = 0; + private final Property dataFlatteningMaxLevel = Property.of(0); @Schema( title = "Log based and Incremental replications on tables with no Primary Key cause duplicates when merging UPDATE events.", description = "When set to true, stop loading data if no Primary Key is defined.." ) - @PluginProperty @Builder.Default - private final Boolean primaryKeyRequired = true; + private final Property primaryKeyRequired = Property.of(true); @Schema( title = "Validate every single record message to the corresponding JSON schema.", description = "This option is disabled by default and invalid RECORD messages will fail only at load time by " + "Redshift. Enabling this option will detect invalid records earlier but could cause performance degradation.." ) - @PluginProperty @Builder.Default - private final Boolean validateRecords = false; + private final Property validateRecords = Property.of(false); @Schema( title = "Do not update existing records when Primary Key is defined. ", description = "Useful to improve performance when records are immutable, e.g. events." ) - @PluginProperty @Builder.Default - private final Boolean skipUpdates = false; + private final Property skipUpdates = Property.of(false); @Schema( title = "The compression method to use when writing files to S3 and running Redshift COPY." ) - @PluginProperty @Builder.Default - private final Compression compression = Compression.bzip2; + private final Property compression = Property.of(Compression.bzip2); @Schema( title = "number of slices to split files into prior to running COPY on Redshift.", description = "This should be set to the number of Redshift slices. The number of slices per node depends on " + "the node size of the cluster - run SELECT COUNT(DISTINCT slice) slices FROM stv_slices to calculate this." ) - @PluginProperty @Builder.Default - private final Integer slices = 1; + private final Property slices = Property.of(1); @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("user", runContext.render(this.username)) - .put("password", runContext.render(this.password)) + .put("password", runContext.render(this.password).as(String.class).orElse(null)) .put("host", runContext.render(this.host)) - .put("port", this.port) - .put("dbname", runContext.render(this.dbName)) + .put("port", runContext.render(this.port).as(Integer.class).orElseThrow()) + .put("dbname", runContext.render(this.dbName).as(String.class).orElseThrow()) .put("s3_bucket", runContext.render(this.s3Bucket)) .put("default_target_schema", runContext.render(this.defaultTargetSchema)) - .put("batch_size_rows", this.batchSizeRows) - .put("flush_all_streams", this.flushAllStreams) - .put("parallelism", this.parallelism) - .put("max_parallelism", this.maxParallelism) - .put("disable_table_cache", this.disableTableCache) - .put("addMetadataColumns", this.addMetadataColumns) - .put("hardDelete", this.hardDelete) - .put("data_flattening_max_level", this.dataFlatteningMaxLevel) - .put("primary_key_required", this.primaryKeyRequired) - .put("validate_records", this.validateRecords) - .put("skip_updates", this.skipUpdates) - .put("compression", this.compression) - .put("slices", this.slices); + .put("batch_size_rows", runContext.render(this.batchSizeRows).as(Integer.class).orElseThrow()) + .put("flush_all_streams", runContext.render(this.flushAllStreams).as(Boolean.class).orElseThrow()) + .put("parallelism", runContext.render(this.parallelism).as(Integer.class).orElseThrow()) + .put("max_parallelism", runContext.render(this.maxParallelism).as(Integer.class).orElseThrow()) + .put("disable_table_cache", runContext.render(this.disableTableCache).as(Boolean.class).orElseThrow()) + .put("addMetadataColumns", runContext.render(this.addMetadataColumns).as(Boolean.class).orElseThrow()) + .put("hardDelete", runContext.render(this.hardDelete).as(Boolean.class).orElseThrow()) + .put("data_flattening_max_level", runContext.render(this.dataFlatteningMaxLevel).as(Integer.class).orElseThrow()) + .put("primary_key_required", runContext.render(this.primaryKeyRequired).as(Boolean.class).orElseThrow()) + .put("validate_records", runContext.render(this.validateRecords).as(Boolean.class).orElseThrow()) + .put("skip_updates", runContext.render(this.skipUpdates).as(Boolean.class).orElseThrow()) + .put("compression", runContext.render(this.compression).as(Compression.class).orElseThrow()) + .put("slices", runContext.render(this.slices).as(Integer.class).orElseThrow()); if (this.accessKeyId != null) { - builder.put("aws_access_key_id", runContext.render(this.accessKeyId)); + builder.put("aws_access_key_id", runContext.render(this.accessKeyId).as(String.class).orElseThrow()); } if (this.secretAccessKey != null) { - builder.put("aws_secret_access_key", runContext.render(this.secretAccessKey)); + builder.put("aws_secret_access_key", runContext.render(this.secretAccessKey).as(String.class).orElseThrow()); } if (this.sessionToken != null) { - builder.put("aws_session_token", runContext.render(this.sessionToken)); + builder.put("aws_session_token", runContext.render(this.sessionToken).as(String.class).orElseThrow()); } if (this.redshiftCopyRoleArn != null) { - builder.put("aws_redshift_copy_role_arn", runContext.render(this.redshiftCopyRoleArn)); + builder.put("aws_redshift_copy_role_arn", runContext.render(this.redshiftCopyRoleArn).as(String.class).orElseThrow()); } if (this.s3Acl != null) { - builder.put("s3_acl", runContext.render(this.s3Acl)); + builder.put("s3_acl", runContext.render(this.s3Acl).as(String.class).orElseThrow()); } if (this.s3KeyPrefix != null) { - builder.put("s3_key_prefix", runContext.render(this.s3KeyPrefix)); + builder.put("s3_key_prefix", runContext.render(this.s3KeyPrefix).as(String.class).orElseThrow()); } if (this.copyOptions != null) { - builder.put("copy_options", runContext.render(this.copyOptions)); + builder.put("copy_options", runContext.render(this.copyOptions).as(String.class).orElseThrow()); } if (this.defaultTargetSchemaSelectPermissions != null) { - builder.put("default_target_schema_select_permissions", runContext.render(this.defaultTargetSchemaSelectPermissions)); + builder.put("default_target_schema_select_permissions", runContext.render(this.defaultTargetSchemaSelectPermissions).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseSnowflake.java b/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseSnowflake.java index a19e1df..df7032c 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseSnowflake.java +++ b/src/main/java/io/kestra/plugin/singer/targets/PipelinewiseSnowflake.java @@ -55,8 +55,7 @@ public class PipelinewiseSnowflake extends AbstractPythonTarget implements Runna @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @NotNull @NotEmpty @@ -69,207 +68,175 @@ public class PipelinewiseSnowflake extends AbstractPythonTarget implements Runna @Schema( title = "Snowflake role to use. If not defined then the user's default role will be used." ) - @PluginProperty(dynamic = true) - private String role; + private Property role; @Schema( title = "S3 Access Key ID. If not provided, `AWS_ACCESS_KEY_ID` environment variable or IAM role will be used." ) - @PluginProperty(dynamic = true) - private String awsAccessKeyId; + private Property awsAccessKeyId; @Schema( title = "S3 Secret Access Key. If not provided, `AWS_SECRET_ACCESS_KEY` environment variable or IAM role will be used." ) - @PluginProperty(dynamic = true) - private String awsSecretAccessKey; + private Property awsSecretAccessKey; @Schema( title = "AWS Session token. If not provided, `AWS_SESSION_TOKEN` environment variable will be used." ) - @PluginProperty(dynamic = true) - private String awsSessionToken; + private Property awsSessionToken; @Schema( title = "AWS profile name for profile based authentication. If not provided, `AWS_PROFILE` environment variable will be used." ) - @PluginProperty(dynamic = true) - private String awsProfile; + private Property awsProfile; @Schema( title = "S3 Bucket name. Required if to use S3 External stage. When this is defined then `stage` has to be defined as well." ) - @PluginProperty(dynamic = true) - private String s3Bucket; + private Property s3Bucket; @Schema( title = "A static prefix before the generated S3 key names. Using prefixes you can upload files into specific directories in the S3 bucket." ) - @PluginProperty(dynamic = true) - private String s3KeyPrefix; + private Property s3KeyPrefix; @Schema( title = "The complete URL to use for the constructed client. This is allowing to use non-native s3 account. " ) - @PluginProperty(dynamic = true) - private String s3EndpointUrl; + private Property s3EndpointUrl; @Schema( title = "Default region when creating new connections." ) - @PluginProperty(dynamic = true) - private String s3RegionName; + private Property s3RegionName; @Schema( title = "S3 ACL name to set on the uploaded files." ) - @PluginProperty(dynamic = true) - private String s3Acl; + private Property s3Acl; @Schema( title = "Named external stage name created at pre-requirements section. Has to be a fully qualified name including the schema name. If not specified, table internal stage are used. When this is defined then `s3_bucket` has to be defined as well. " ) - @PluginProperty(dynamic = true) - private String stage; + private Property stage; @Schema( title = "Named file format name created at pre-requirements section. Has to be a fully qualified name including the schema name. " ) - @PluginProperty(dynamic = true) - private String fileFormat; + private Property fileFormat; @Schema( title = "Maximum number of rows in each batch. At the end of each batch, the rows in the batch are loaded into Snowflake. " ) - @PluginProperty @Builder.Default - private Integer batchSizeRows = 100000; + private Property batchSizeRows = Property.of(100000); @Schema( title = "Maximum time to wait for batch to reach `batch_size_rows`. " ) - @PluginProperty - private Duration batchWaitLimit; + private Property batchWaitLimit; @Builder.Default @Schema( title = "Flush and load every stream into Snowflake when one batch is full. Warning: This may trigger the COPY command to use files with low number of records, and may cause performance problems. " ) - @PluginProperty(dynamic = true) - private Boolean flushAllStreams = false; + private Property flushAllStreams = Property.of(false); @Schema( title = "The number of threads used to flush tables. 0 will create a thread for each stream, up to parallelism_max. -1 will create a thread for each CPU core. Any other positive number will create that number of threads, up to parallelism_max. " ) - @PluginProperty @Builder.Default - private Integer parallelism = 0; + private Property parallelism = Property.of(0); @Schema( title = "Max number of parallel threads to use when flushing tables. " ) - @PluginProperty @Builder.Default - private Integer parallelismMax = 16; + private Property parallelismMax = Property.of(16); @Schema( title = "Name of the schema where the tables will be created, **without** database prefix. If `schema_mapping` is not defined then every stream sent by the tap is loaded into this schema. " ) - @PluginProperty(dynamic = true) - private String defaultTargetSchema; + private Property defaultTargetSchema; @Schema( title = "Grant USAGE privilege on newly created schemas and grant SELECT privilege on newly created tables to a specific role or a list of roles. If `schema_mapping` is not defined then every stream sent by the tap is granted accordingly. " ) - @PluginProperty(dynamic = true) - private String defaultTargetSchemaSelectPermission; + private Property defaultTargetSchemaSelectPermission; @Schema( title = "Useful if you want to load multiple streams from one tap to multiple Snowflake schemas.

If the tap sends the `stream_id` in `-` format then this option overwrites the `default_target_schema` value. Note, that using `schema_mapping` you can overwrite the `default_target_schema_select_permission` value to grant SELECT permissions to different groups per schemas or optionally you can create indices automatically for the replicated tables.

**Note**: This is an experimental feature and recommended to use via PipelineWise YAML files that will generate the object mapping in the right JSON format. For further info check a PipelineWise YAML Example" ) - @PluginProperty(dynamic = true) - private String schemaMapping; + private Property schemaMapping; @Schema( title = "By default the connector caches the available table structures in Snowflake at startup. In this way it doesn't need to run additional queries when ingesting data to check if altering the target tables is required. With `disable_table_cache` option you can turn off this caching. You will always see the most recent table structures but will cause an extra query runtime." ) - @PluginProperty(dynamic = true) @Builder.Default - private Boolean disableTableCache = false; + private Property disableTableCache = Property.of(false); @Schema( title = "When this is defined, Client-Side Encryption is enabled. The data in S3 will be encrypted. No third parties, including Amazon AWS and any ISPs, can see data in the clear. Snowflake COPY command will decrypt the data once it is in Snowflake. The master key must be 256-bit length and must be encoded as base64 string." ) - @PluginProperty(dynamic = true) - private String clientSideEncryptionMasterKey; + private Property clientSideEncryptionMasterKey; @Schema( title = "Metadata columns add extra row level information about data ingestions, (i.e. when was the row read in source, when was inserted or deleted in snowflake etc.) Metadata columns are creating automatically by adding extra columns to the tables with a column prefix `_SDC_`. The column names are following the stitch naming conventions documented at https://www.stitchdata.com/docs/data-structure/integration-schemas#sdc-columns. Enabling metadata columns will flag the deleted rows by setting the `_SDC_DELETED_AT` metadata column. Without the `add_metadata_columns` option the deleted rows from singer taps will not be recongisable in Snowflake. " ) @Builder.Default - @PluginProperty - private Boolean addMetadataColumns = false; + private Property addMetadataColumns = Property.of(false); @Schema( title = "When `hardDelete` option is true, then DELETE SQL commands will be performed in Snowflake to delete rows in tables. It is achieved by continuously checking the `_SDC_DELETED_AT` metadata column sent by the singer tap. Due to deleting rows requiring metadata columns, `hard_delete` option automatically enables the `add_metadata_columns` option as well." ) - @PluginProperty @Builder.Default - private Boolean hardDelete = false; + private Property hardDelete = Property.of(false); @Schema( title = "(Default: 0) Object type RECORD items from taps can be loaded into VARIANT columns as JSON (default) or we can flatten the schema by creating columns automatically.

When value is 0 (default) then flattening functionality is turned off." ) - @PluginProperty @Builder.Default - private Integer dataFlatteningMaxLevel = 0; + private Property dataFlatteningMaxLevel = Property.of(0); @Schema( title = "Log based and Incremental replications on tables with no Primary Key cause duplicates when merging UPDATE events. When set to true, stop loading data if no Primary Key is defined." ) - @PluginProperty @Builder.Default - private Boolean primaryKeyRequired = true; + private Property primaryKeyRequired = Property.of(true); @Schema( title = "Validate every single record message to the corresponding JSON schema. This option is disabled by default and invalid RECORD messages will fail only at load time by Snowflake. Enabling this option will detect invalid records earlier but could cause performance degradation." ) - @PluginProperty(dynamic = true) @Builder.Default - private Boolean validateRecords = false; + private Property validateRecords = Property.of(false); @Schema( title = "Generate uncompressed files when loading to Snowflake. Normally, by default GZIP compressed files are generated. " ) - @PluginProperty @Builder.Default - private Boolean noCompression = false; + private Property noCompression = Property.of(false); @Schema( title = "Optional string to tag executed queries in Snowflake. Replaces tokens `{{database}}`, `{{schema}}` and `{{table}}` with the appropriate values. The tags are displayed in the output of the Snowflake `QUERY_HISTORY`, `QUERY_HISTORY_BY_*` functions. " ) - @PluginProperty(dynamic = true) - private String queryTag; + private Property queryTag; @Schema( title = "When enabled, the files loaded to Snowflake will also be stored in `archive_load_files_s3_bucket` under the key `/{archive_load_files_s3_prefix}/{schema_name}/{table_name}/`. All archived files will have `tap`, `schema`, `table` and `archived-by` as S3 metadata keys. When incremental replication is used, the archived files will also have the following S3 metadata keys: `incremental-key`, `incremental-key-min` and `incremental-key-max`" ) - @PluginProperty @Builder.Default - private Boolean archiveLoadFiles = false; + private Property archiveLoadFiles = Property.of(false); @Schema( title = "(Default: `archive`) When `archive_load_files` is enabled, the archived files will be placed in the archive S3 bucket under this prefix." ) - @PluginProperty(dynamic = true) - private String archiveLoadFilesS3Prefix; + private Property archiveLoadFilesS3Prefix; @Schema( title = "(Default: Value of `s3_bucket`) When `archive_load_files` is enabled, the archived files will be placed in this bucket." ) - @PluginProperty(dynamic = true) - private String archiveLoadFilesS3Bucket; + private Property archiveLoadFilesS3Bucket; @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { @@ -277,135 +244,135 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("account", runContext.render(this.account)) .put("dbname", runContext.render(this.database)) .put("user", runContext.render(this.username)) - .put("password", runContext.render(this.password)) + .put("password", runContext.render(this.password).as(String.class).orElse(null)) .put("warehouse", runContext.render(this.warehouse)); if (this.role != null) { - builder.put("role", runContext.render(this.role)); + builder.put("role", runContext.render(this.role).as(String.class).orElseThrow()); } if (this.awsAccessKeyId != null) { - builder.put("aws_access_key_id", runContext.render(this.awsAccessKeyId)); + builder.put("aws_access_key_id", runContext.render(this.awsAccessKeyId).as(String.class).orElseThrow()); } if (this.awsSecretAccessKey != null) { - builder.put("aws_secret_access_key", runContext.render(this.awsSecretAccessKey)); + builder.put("aws_secret_access_key", runContext.render(this.awsSecretAccessKey).as(String.class).orElseThrow()); } if (this.awsSessionToken != null) { - builder.put("aws_session_token", runContext.render(this.awsSessionToken)); + builder.put("aws_session_token", runContext.render(this.awsSessionToken).as(String.class).orElseThrow()); } if (this.awsProfile != null) { - builder.put("aws_profile", runContext.render(this.awsProfile)); + builder.put("aws_profile", runContext.render(this.awsProfile).as(String.class).orElseThrow()); } if (this.s3Bucket != null) { - builder.put("s3_bucket", runContext.render(this.s3Bucket)); + builder.put("s3_bucket", runContext.render(this.s3Bucket).as(String.class).orElseThrow()); } if (this.s3KeyPrefix != null) { - builder.put("s3_key_prefix", runContext.render(this.s3KeyPrefix)); + builder.put("s3_key_prefix", runContext.render(this.s3KeyPrefix).as(String.class).orElseThrow()); } if (this.s3EndpointUrl != null) { - builder.put("s3_endpoint_url", runContext.render(this.s3EndpointUrl)); + builder.put("s3_endpoint_url", runContext.render(this.s3EndpointUrl).as(String.class).orElseThrow()); } if (this.s3RegionName != null) { - builder.put("s3_region_name", runContext.render(this.s3RegionName)); + builder.put("s3_region_name", runContext.render(this.s3RegionName).as(String.class).orElseThrow()); } if (this.s3Acl != null) { - builder.put("s3_acl", runContext.render(this.s3Acl)); + builder.put("s3_acl", runContext.render(this.s3Acl).as(String.class).orElseThrow()); } if (this.stage != null) { - builder.put("stage", runContext.render(this.stage)); + builder.put("stage", runContext.render(this.stage).as(String.class).orElseThrow()); } if (this.fileFormat != null) { - builder.put("file_format", runContext.render(this.fileFormat)); + builder.put("file_format", runContext.render(this.fileFormat).as(String.class).orElseThrow()); } if (this.batchSizeRows != null) { - builder.put("batch_size_rows", this.batchSizeRows); + builder.put("batch_size_rows", runContext.render(this.batchSizeRows).as(Integer.class).orElseThrow()); } if (this.batchWaitLimit != null) { - builder.put("batch_wait_limit_seconds", this.batchWaitLimit.toSeconds()); + builder.put("batch_wait_limit_seconds", runContext.render(this.batchWaitLimit).as(Duration.class).orElseThrow().toSeconds()); } if (this.flushAllStreams != null) { - builder.put("flush_all_streams", this.flushAllStreams.toString()); + builder.put("flush_all_streams", runContext.render(this.flushAllStreams).as(Boolean.class).orElseThrow().toString()); } if (this.parallelism != null) { - builder.put("parallelism", this.parallelism); + builder.put("parallelism", runContext.render(this.parallelism).as(Integer.class).orElseThrow()); } if (this.parallelismMax != null) { - builder.put("parallelism_max", this.parallelismMax); + builder.put("parallelism_max", runContext.render(this.parallelismMax).as(Integer.class).orElseThrow()); } if (this.defaultTargetSchema != null) { - builder.put("default_target_schema", runContext.render(this.defaultTargetSchema)); + builder.put("default_target_schema", runContext.render(this.defaultTargetSchema).as(String.class).orElseThrow()); } if (this.defaultTargetSchemaSelectPermission != null) { - builder.put("default_target_schema_select_permission", runContext.render(this.defaultTargetSchemaSelectPermission)); + builder.put("default_target_schema_select_permission", runContext.render(this.defaultTargetSchemaSelectPermission).as(String.class).orElseThrow()); } if (this.schemaMapping != null) { - builder.put("schema_mapping", runContext.render(this.schemaMapping)); + builder.put("schema_mapping", runContext.render(this.schemaMapping).as(String.class).orElseThrow()); } if (this.disableTableCache != null) { - builder.put("disable_table_cache", this.disableTableCache.toString()); + builder.put("disable_table_cache", runContext.render(this.disableTableCache).as(Boolean.class).orElseThrow().toString()); } if (this.clientSideEncryptionMasterKey != null) { - builder.put("client_side_encryption_master_key", runContext.render(this.clientSideEncryptionMasterKey)); + builder.put("client_side_encryption_master_key", runContext.render(this.clientSideEncryptionMasterKey).as(String.class).orElseThrow()); } if (this.addMetadataColumns != null) { - builder.put("add_metadata_columns", this.addMetadataColumns.toString()); + builder.put("add_metadata_columns", runContext.render(this.addMetadataColumns).as(Boolean.class).orElseThrow().toString()); } if (this.hardDelete != null) { - builder.put("hard_delete", this.hardDelete.toString()); + builder.put("hard_delete", runContext.render(this.hardDelete).as(Boolean.class).orElseThrow().toString()); } if (this.dataFlatteningMaxLevel != null) { - builder.put("data_flattening_max_level", this.dataFlatteningMaxLevel); + builder.put("data_flattening_max_level", runContext.render(this.dataFlatteningMaxLevel).as(Integer.class).orElseThrow()); } if (this.primaryKeyRequired != null) { - builder.put("primary_key_required", this.primaryKeyRequired.toString()); + builder.put("primary_key_required", runContext.render(this.primaryKeyRequired).as(Boolean.class).orElseThrow().toString()); } if (this.validateRecords != null) { - builder.put("validate_records", this.validateRecords.toString()); + builder.put("validate_records", runContext.render(this.validateRecords).as(Boolean.class).orElseThrow().toString()); } if (this.noCompression != null) { - builder.put("no_compression", this.noCompression.toString()); + builder.put("no_compression", runContext.render(this.noCompression).as(Boolean.class).orElseThrow().toString()); } if (this.queryTag != null) { - builder.put("query_tag", runContext.render(this.queryTag)); + builder.put("query_tag", runContext.render(this.queryTag).as(String.class).orElseThrow()); } if (this.archiveLoadFiles != null) { - builder.put("archive_load_files", this.archiveLoadFiles.toString()); + builder.put("archive_load_files", runContext.render(this.archiveLoadFiles).as(Boolean.class).orElseThrow().toString()); } if (this.archiveLoadFilesS3Prefix != null) { - builder.put("archive_load_files_s3_prefix", runContext.render(this.archiveLoadFilesS3Prefix)); + builder.put("archive_load_files_s3_prefix", runContext.render(this.archiveLoadFilesS3Prefix).as(String.class).orElseThrow()); } if (this.archiveLoadFilesS3Bucket != null) { - builder.put("archive_load_files_s3_bucket", runContext.render(this.archiveLoadFilesS3Bucket)); + builder.put("archive_load_files_s3_bucket", runContext.render(this.archiveLoadFilesS3Bucket).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/targets/SqlServer.java b/src/main/java/io/kestra/plugin/singer/targets/SqlServer.java index c8df58c..df8b4ef 100644 --- a/src/main/java/io/kestra/plugin/singer/targets/SqlServer.java +++ b/src/main/java/io/kestra/plugin/singer/targets/SqlServer.java @@ -48,8 +48,7 @@ public class SqlServer extends AbstractPythonTarget implements RunnableTask port; @NotEmpty @Schema( @@ -68,44 +67,37 @@ public class SqlServer extends AbstractPythonTarget implements RunnableTask defaultTargetSchema; @Schema( title = "Prefix to add to table name. Useful if retrieving data from multiple taps for easier filtering." ) - @PluginProperty(dynamic = true) - private String tablePrefix; + private Property tablePrefix; @Schema( title = "Use float data type for numbers (otherwise number type is used)." ) - @PluginProperty - private Boolean preferFloatOverNumeric; + private Property preferFloatOverNumeric; @Schema( title = "Config object for stream maps capability." ) - @PluginProperty(dynamic = true) - private String streamMaps; + private Property streamMaps; @Schema( title = "User-defined config values to be used within map expressions." ) - @PluginProperty(dynamic = true) - private String streamMapConfig; + private Property streamMapConfig; @Schema( title = "Enable schema flattening and automatically expand nested properties" ) - @PluginProperty - private Boolean flatteningEnabled; + private Property flatteningEnabled; @Schema( title = "The max depth to flatten schemas." ) - @PluginProperty - private Integer flatteningMaxDepth; + private Property flatteningMaxDepth; @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { @@ -113,35 +105,35 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("username", runContext.render(this.username)) .put("password", runContext.render(this.password)) .put("host", runContext.render(this.host)) - .put("port", String.valueOf(this.port)) + .put("port", String.valueOf(runContext.render(this.port).as(Integer.class).orElseThrow())) .put("database", runContext.render(this.database)); if (this.defaultTargetSchema != null) { - builder.put("default_target_schema", runContext.render(this.defaultTargetSchema)); + builder.put("default_target_schema", runContext.render(this.defaultTargetSchema).as(String.class).orElseThrow()); } if (this.tablePrefix != null) { - builder.put("table_prefix", runContext.render(this.tablePrefix)); + builder.put("table_prefix", runContext.render(this.tablePrefix).as(String.class).orElseThrow()); } if (this.preferFloatOverNumeric != null) { - builder.put("prefer_float_over_numeric", StringUtils.capitalize(this.preferFloatOverNumeric.toString())); + builder.put("prefer_float_over_numeric", StringUtils.capitalize(runContext.render(this.preferFloatOverNumeric).as(Boolean.class).orElseThrow().toString())); } if (this.streamMaps != null) { - builder.put("stream_maps", runContext.render(this.streamMaps)); + builder.put("stream_maps", runContext.render(this.streamMaps).as(String.class).orElseThrow()); } if (this.streamMapConfig != null) { - builder.put("stream_map_config", runContext.render(this.streamMapConfig)); + builder.put("stream_map_config", runContext.render(this.streamMapConfig).as(String.class).orElseThrow()); } if (this.flatteningEnabled != null) { - builder.put("flattening_enabled", StringUtils.capitalize(this.flatteningEnabled.toString())); + builder.put("flattening_enabled", StringUtils.capitalize(runContext.render(this.flatteningEnabled).as(Boolean.class).orElseThrow().toString())); } if (this.flatteningMaxDepth != null) { - builder.put("flattening_max_depth", this.flatteningMaxDepth); + builder.put("flattening_max_depth", runContext.render(this.flatteningMaxDepth).as(Integer.class).orElseThrow()); } return builder.build(); diff --git a/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java b/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java index e207faa..d179fe4 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java @@ -81,12 +81,12 @@ void run() throws Exception { .builder() .id(IdUtils.create() + "_bq") .type(io.kestra.plugin.singer.targets.AdswerveBigQuery.class.getName()) - .from(tapOutput.getRaw().toString()) + .from(Property.of(tapOutput.getRaw().toString())) .stateName(Property.of(stateName)) - .serviceAccount(serviceAccount) + .serviceAccount(Property.of(serviceAccount)) .projectId(project) .datasetId(dataset) - .location("EU"); + .location(Property.of("EU")); AdswerveBigQuery task = builder.build(); diff --git a/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java b/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java index d77d9e1..db10479 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java @@ -63,7 +63,7 @@ void run() throws Exception { .builder() .id(IdUtils.create()) .type(DatamillCoPostgres.class.getName()) - .from(tapOutput.getRaw().toString()) + .from(Property.of(tapOutput.getRaw().toString())) .stateName(Property.of(stateName)) .delimiter(";"); Csv target = builder.build(); diff --git a/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java b/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java index b0e02f4..9716a4d 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java @@ -66,13 +66,13 @@ void run() throws Exception { .builder() .id(IdUtils.create()) .type(io.kestra.plugin.singer.targets.DatamillCoPostgres.class.getName()) - .from(tapOutput.getRaw().toString()) + .from(Property.of(tapOutput.getRaw().toString())) .stateName(Property.of(stateName)) .host("172.17.0.1") .username("postgres") - .password("pg_passwd") - .port(65432) - .dbName("sync"); + .password(Property.of("pg_passwd")) + .port(Property.of(65432)) + .dbName(Property.of("sync")); DatamillCoPostgres task = builder.build(); diff --git a/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java b/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java index c561a47..4073b8d 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java @@ -65,7 +65,7 @@ void run() throws Exception { .builder() .id(IdUtils.create()) .type(DatamillCoPostgres.class.getName()) - .from(tapOutput.getRaw().toString()) + .from(Property.of(tapOutput.getRaw().toString())) .stateName(Property.of(stateName)) .build(); diff --git a/src/test/java/io/kestra/plugin/singer/targets/OracleTest.java b/src/test/java/io/kestra/plugin/singer/targets/OracleTest.java index 397fbc2..cbdd136 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/OracleTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/OracleTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.targets; import com.google.common.collect.ImmutableMap; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -56,12 +57,12 @@ void run() throws Exception { Oracle target = Oracle.builder() .id(IdUtils.create()) .type(Oracle.class.getName()) - .from(tapOutput.getRaw().toString()) + .from(Property.of(tapOutput.getRaw().toString())) .host("172.17.0.1") - .database("FREE") + .database(Property.of("FREE")) .username("system") .password("oracle_passwd") - .port(57057) + .port(Property.of(57057)) .build(); runContext = TestsUtils.mockRunContext(runContextFactory, target, ImmutableMap.of()); diff --git a/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java b/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java index d0dd406..84d1cbe 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java @@ -69,13 +69,13 @@ void run() throws Exception { SqlServer target = SqlServer.builder() .id(IdUtils.create()) .type(PipelinewiseSqlServer.class.getName()) - .from(tapOutput.getRaw().toString()) + .from(Property.of(tapOutput.getRaw().toString())) .host("172.17.0.1") .database("msdb") .username("SA") .password("SQLServer_Passwd") - .port(57037) - .defaultTargetSchema("target") + .port(Property.of(57037)) + .defaultTargetSchema(Property.of("target")) .build(); runContext = TestsUtils.mockRunContext(runContextFactory, target, ImmutableMap.of()); From dfd9e8cb4b409108b03da027eccd94bfd3086024 Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle Date: Tue, 17 Dec 2024 16:14:56 +0100 Subject: [PATCH 3/3] refactor: migrate taps tasks --- .../plugin/singer/taps/FacebookAds.java | 5 +- .../kestra/plugin/singer/taps/GenericTap.java | 7 ++- .../io/kestra/plugin/singer/taps/GitHub.java | 5 +- .../io/kestra/plugin/singer/taps/Gitlab.java | 31 +++++----- .../plugin/singer/taps/GoogleAdwords.java | 10 ++- .../plugin/singer/taps/GoogleAnalytics.java | 30 ++++----- .../singer/taps/GoogleSearchConsole.java | 5 +- .../io/kestra/plugin/singer/taps/HubSpot.java | 26 ++++---- .../kestra/plugin/singer/taps/Netsuite.java | 10 ++- .../singer/taps/PipelinewiseMongoDb.java | 50 ++++++--------- .../plugin/singer/taps/PipelinewiseMysql.java | 37 +++++------ .../singer/taps/PipelinewiseOracle.java | 20 +++--- .../singer/taps/PipelinewisePostgres.java | 50 +++++++-------- .../singer/taps/PipelinewiseSqlServer.java | 37 +++++------ .../kestra/plugin/singer/taps/Quickbooks.java | 18 +++--- .../kestra/plugin/singer/taps/Recharge.java | 5 +- .../kestra/plugin/singer/taps/Salesforce.java | 61 +++++++------------ .../io/kestra/plugin/singer/taps/Slack.java | 28 ++++----- .../io/kestra/plugin/singer/taps/Zendesk.java | 15 ++--- .../io/kestra/plugin/singer/taps/Zoom.java | 20 +++--- .../singer/taps/PipelinewiseMongoDbTest.java | 9 +-- .../singer/taps/PipelinewiseMysqlTest.java | 5 +- .../singer/taps/PipelinewisePostgresTest.java | 7 ++- .../taps/PipelinewiseSqlServerTest.java | 4 +- .../singer/targets/AdswerveBigQueryTest.java | 4 +- .../kestra/plugin/singer/targets/CsvTest.java | 4 +- .../targets/DatamillCoPostgresTest.java | 4 +- .../plugin/singer/targets/JsonTest.java | 4 +- .../plugin/singer/targets/OracleTest.java | 4 +- .../plugin/singer/targets/SqlServerTest.java | 6 +- 30 files changed, 223 insertions(+), 298 deletions(-) diff --git a/src/main/java/io/kestra/plugin/singer/taps/FacebookAds.java b/src/main/java/io/kestra/plugin/singer/taps/FacebookAds.java index 9394693..c43984c 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/FacebookAds.java +++ b/src/main/java/io/kestra/plugin/singer/taps/FacebookAds.java @@ -49,9 +49,8 @@ public class FacebookAds extends AbstractPythonTap implements RunnableTask insightsBufferDays = Property.of(0); @NotNull @Schema( @@ -80,7 +79,7 @@ public Map configuration(RunContext runContext) throws IllegalVa ImmutableMap.Builder builder = ImmutableMap.builder() .put("account_id", runContext.render(this.accountId)) .put("access_token", runContext.render(this.accessToken)) - .put("insights_buffer_days", this.insightsBufferDays) + .put("insights_buffer_days", runContext.render(this.insightsBufferDays).as(Integer.class).orElseThrow()) .put("start_date", runContext.render(this.startDate.toString())); if (this.endDate != null) { diff --git a/src/main/java/io/kestra/plugin/singer/taps/GenericTap.java b/src/main/java/io/kestra/plugin/singer/taps/GenericTap.java index 0e74b46..4d28a64 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GenericTap.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GenericTap.java @@ -12,6 +12,7 @@ import lombok.experimental.SuperBuilder; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,8 +53,7 @@ public class GenericTap extends AbstractPythonTap implements RunnableTask configs; + private Property> configs; public List features() { return this.features; @@ -61,7 +61,8 @@ public List features() { @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { - return runContext.render(configs); + var config = runContext.render(configs).asMap(String.class, Object.class); + return config.isEmpty() ? new HashMap<>() : config; } @Override diff --git a/src/main/java/io/kestra/plugin/singer/taps/GitHub.java b/src/main/java/io/kestra/plugin/singer/taps/GitHub.java index 2ac19fc..2293466 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GitHub.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GitHub.java @@ -64,9 +64,8 @@ public class GitHub extends AbstractPythonTap implements RunnableTask requestTimeout = Property.of(300); public List features() { return Arrays.asList( @@ -102,7 +101,7 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("access_token", runContext.render(this.accessToken)) .put("repository", String.join(" ", repositories)) .put("start_date", runContext.render(this.startDate.toString())) - .put("request_timeout", this.requestTimeout); + .put("request_timeout", runContext.render(this.requestTimeout).as(Integer.class).orElseThrow()); return builder.build(); } diff --git a/src/main/java/io/kestra/plugin/singer/taps/Gitlab.java b/src/main/java/io/kestra/plugin/singer/taps/Gitlab.java index 63033b0..6979ea3 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Gitlab.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Gitlab.java @@ -53,38 +53,33 @@ public class Gitlab extends AbstractPythonTap implements RunnableTask groups; + private Property> groups; @Schema( title = "`namespace/project` paths of projects to extract data from.", description = "Leave empty and provide a group name to extract data from all group projects." ) - @PluginProperty(dynamic = true) - private List projects; + private Property> projects; @NotNull @Schema( title = "Enable to pull in extra data (like Epics, Epic Issues and other entities) only available to GitLab Ultimate and GitLab.com Gold accounts." ) - @PluginProperty(dynamic = true) - private final Boolean ultimateLicense = false; + private final Property ultimateLicense = Property.of(false); @NotNull @Schema( title = "For each Merge Request, also fetch the MR's commits and create the join table `merge_request_commits` with the Merge Request and related Commit IDs.", description = "This can slow down extraction considerably because of the many API calls required." ) - @PluginProperty(dynamic = true) - private final Boolean fetchMergeRequestCommits = false; + private final Property fetchMergeRequestCommits = Property.of(false); @NotNull @Schema( title = "For every Pipeline, also fetch extended details of each of these pipelines.", description = "This can slow down extraction considerably because of the many API calls required." ) - @PluginProperty(dynamic = true) - private final Boolean fetchPipelinesExtended = false; + private final Property fetchPipelinesExtended = Property.of(false); @NotNull @Schema( @@ -107,17 +102,19 @@ public Map configuration(RunContext runContext) throws IllegalVa ImmutableMap.Builder builder = ImmutableMap.builder() .put("api_url", runContext.render(this.apiUrl)) .put("private_token", runContext.render(this.private_token)) - .put("ultimate_license", this.ultimateLicense) - .put("fetch_merge_request_commits", this.fetchMergeRequestCommits) - .put("fetch_pipelines_extended", this.fetchPipelinesExtended) + .put("ultimate_license", runContext.render(this.ultimateLicense).as(Boolean.class).orElseThrow()) + .put("fetch_merge_request_commits", runContext.render(this.fetchMergeRequestCommits).as(Boolean.class).orElseThrow()) + .put("fetch_pipelines_extended", runContext.render(this.fetchPipelinesExtended).as(Boolean.class).orElseThrow()) .put("start_date", runContext.render(this.startDate.toString())); - if (groups != null) { - builder.put("groups", String.join(" ", runContext.render(this.groups))); + var renderedGroups = runContext.render(this.groups).asList(String.class); + if (!renderedGroups.isEmpty()) { + builder.put("groups", String.join(" ", renderedGroups)); } - if (projects != null) { - builder.put("projects", String.join(" ", runContext.render(this.projects))); + var renderedProjects = runContext.render(projects).asList(String.class); + if (!renderedProjects.isEmpty()) { + builder.put("projects", String.join(" ", renderedProjects)); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/GoogleAdwords.java b/src/main/java/io/kestra/plugin/singer/taps/GoogleAdwords.java index 11bcede..7682448 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GoogleAdwords.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GoogleAdwords.java @@ -86,9 +86,8 @@ public class GoogleAdwords extends AbstractPythonTap implements RunnableTask conversionWindowDays = Property.of(0); @Schema( title = "Primary Keys for the selected Entities (Streams)." @@ -99,9 +98,8 @@ public class GoogleAdwords extends AbstractPythonTap implements RunnableTask userAgent = Property.of("tap-adwords via Kestra"); public List features() { return Arrays.asList( @@ -120,14 +118,14 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("refresh_token", runContext.render(this.refreshToken)) .put("customer_ids", String.join(",", runContext.render(this.customerIds))) .put("start_date", runContext.render(this.startDate.toString())) - .put("conversion_window_days", this.conversionWindowDays); + .put("conversion_window_days", runContext.render(this.conversionWindowDays).as(Integer.class).orElseThrow()); if (this.endDate != null) { builder.put("end_date", runContext.render(this.endDate.toString())); } if (this.userAgent != null) { - builder.put("user_agent", runContext.render(this.userAgent)); + builder.put("user_agent", runContext.render(this.userAgent).as(String.class).orElseThrow()); } if (this.primaryKeys != null) { diff --git a/src/main/java/io/kestra/plugin/singer/taps/GoogleAnalytics.java b/src/main/java/io/kestra/plugin/singer/taps/GoogleAnalytics.java index 7862791..ff29334 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GoogleAnalytics.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GoogleAnalytics.java @@ -36,38 +36,32 @@ public class GoogleAnalytics extends AbstractPythonTap implements RunnableTask serviceAccount; @Schema( title = "OAuth Client ID" ) - @PluginProperty(dynamic = true) - private String oauthClientId; + private Property oauthClientId; @Schema( title = "OAuth Client Secret." ) - @PluginProperty(dynamic = true) - private String oauthClientSecret; + private Property oauthClientSecret; @Schema( title = "OAuth Access Token." ) - @PluginProperty(dynamic = true) - private String oauthAccessToken; + private Property oauthAccessToken; @Schema( title = "OAuth Refresh Token." ) - @PluginProperty(dynamic = true) - private String oauthRefreshToken; + private Property oauthRefreshToken; @Schema( title = "OAuth Refresh Token." ) - @PluginProperty(dynamic = true) - private String view_id; + private Property view_id; @Schema( title = "Reports." @@ -102,28 +96,28 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("start_date", runContext.render(this.startDate.toString())); if (this.serviceAccount != null) { - this.writeSingerFiles("google-credentials.json", runContext.render(this.serviceAccount)); + this.writeSingerFiles("google-credentials.json", runContext.render(this.serviceAccount).as(String.class).orElseThrow()); builder.put("key_file_location", workingDirectory.toAbsolutePath() + "/google-credentials.json"); } if (this.oauthClientId != null) { - builder.put("oauth_credentials.client_id", runContext.render(this.oauthClientId)); + builder.put("oauth_credentials.client_id", runContext.render(this.oauthClientId).as(String.class).orElseThrow()); } if (this.oauthClientSecret != null) { - builder.put("oauth_credentials.client_secret", runContext.render(this.oauthClientSecret)); + builder.put("oauth_credentials.client_secret", runContext.render(this.oauthClientSecret).as(String.class).orElseThrow()); } if (this.oauthAccessToken != null) { - builder.put("oauth_credentials.access_token", runContext.render(this.oauthAccessToken)); + builder.put("oauth_credentials.access_token", runContext.render(this.oauthAccessToken).as(String.class).orElseThrow()); } if (this.oauthRefreshToken != null) { - builder.put("oauth_credentials.refresh_token", runContext.render(this.oauthRefreshToken)); + builder.put("oauth_credentials.refresh_token", runContext.render(this.oauthRefreshToken).as(String.class).orElseThrow()); } if (this.view_id != null) { - builder.put("view_id", runContext.render(this.view_id)); + builder.put("view_id", runContext.render(this.view_id).as(String.class).orElseThrow()); } if (this.reports != null) { diff --git a/src/main/java/io/kestra/plugin/singer/taps/GoogleSearchConsole.java b/src/main/java/io/kestra/plugin/singer/taps/GoogleSearchConsole.java index 053dd56..f9a2350 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/GoogleSearchConsole.java +++ b/src/main/java/io/kestra/plugin/singer/taps/GoogleSearchConsole.java @@ -73,9 +73,8 @@ public class GoogleSearchConsole extends AbstractPythonTap implements RunnableTa @Schema( title = "User Agent for your OAuth Client." ) - @PluginProperty(dynamic = true) @Builder.Default - private final String userAgent = "tap-google-search-console via Kestra"; + private final Property userAgent = Property.of("tap-google-search-console via Kestra"); public List features() { return Arrays.asList( @@ -95,7 +94,7 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("start_date", runContext.render(this.startDate.toString())); if (this.userAgent != null) { - builder.put("user_agent", runContext.render(this.userAgent)); + builder.put("user_agent", runContext.render(this.userAgent).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/HubSpot.java b/src/main/java/io/kestra/plugin/singer/taps/HubSpot.java index 57b7ec6..88c20c6 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/HubSpot.java +++ b/src/main/java/io/kestra/plugin/singer/taps/HubSpot.java @@ -43,27 +43,23 @@ public class HubSpot extends AbstractPythonTap implements RunnableTask streamMaps; + private Property> streamMaps; @Schema( title = "User-defined config values to be used within map expressions." ) - @PluginProperty(dynamic = true) - private Map streamMapConfig; + private Property> streamMapConfig; @Schema( title = "To enable schema flattening and automatically expand nested properties." ) - @PluginProperty(dynamic = true) @Builder.Default - private Boolean flatteningEnabled = false; + private Property flatteningEnabled = Property.of(false); @Schema( title = "The max depth to flatten schemas." ) - @PluginProperty(dynamic = true) - private Integer flatteningMaxDepth; + private Property flatteningMaxDepth; @NotNull @Schema( @@ -88,23 +84,25 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("start_date", runContext.render(this.startDate.toString())); try { - if (this.streamMaps != null) { - builder.put("stream_maps", JacksonMapper.ofJson().writeValueAsString(runContext.render(this.streamMaps))); + var map = runContext.render(this.streamMaps).asMap(String.class, Object.class); + if (!map.isEmpty()) { + builder.put("stream_maps", JacksonMapper.ofJson().writeValueAsString(map)); } - if (this.streamMapConfig != null) { - builder.put("stream_map_config", JacksonMapper.ofJson().writeValueAsString(runContext.render(this.streamMapConfig))); + var mapConfig = runContext.render(this.streamMapConfig).asMap(String.class, Object.class); + if (!mapConfig.isEmpty()) { + builder.put("stream_map_config", JacksonMapper.ofJson().writeValueAsString(mapConfig)); } } catch (JsonProcessingException e) { throw new IllegalVariableEvaluationException(e); } if (this.flatteningEnabled != null) { - builder.put("flattening_enabled", this.flatteningEnabled); + builder.put("flattening_enabled", runContext.render(this.flatteningEnabled).as(Boolean.class).orElseThrow()); } if (this.flatteningMaxDepth != null) { - builder.put("flattening_max_depth", this.flatteningMaxDepth); + builder.put("flattening_max_depth", runContext.render(this.flatteningMaxDepth).as(Integer.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/Netsuite.java b/src/main/java/io/kestra/plugin/singer/taps/Netsuite.java index 1c5441f..37629f0 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Netsuite.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Netsuite.java @@ -82,16 +82,14 @@ public class Netsuite extends AbstractPythonTap implements RunnableTask selectFieldsByDefault; @NotNull @Schema( title = "Is this sandbox account.", description = "This should always be set to `true` if you are connecting Production account of NetSuite. Set it to `false` if you want to connect to SandBox account." ) - @PluginProperty(dynamic = true) - private Boolean isSandbox; + private Property isSandbox; @NotNull @Schema( @@ -117,8 +115,8 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("ns_consumer_secret", runContext.render(this.consumerSecret)) .put("ns_token_key", runContext.render(this.tokenKey)) .put("ns_token_secret", runContext.render(this.tokenSecret)) - .put("select_fields_by_default", this.selectFieldsByDefault) - .put("is_sandbox", this.isSandbox) + .put("select_fields_by_default", runContext.render(this.selectFieldsByDefault).as(Boolean.class).orElseThrow()) + .put("is_sandbox", runContext.render(this.isSandbox).as(Boolean.class).orElseThrow()) .put("start_date", runContext.render(this.startDate.toString())); return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDb.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDb.java index 466d15c..69e8da9 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDb.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDb.java @@ -47,72 +47,62 @@ public class PipelinewiseMongoDb extends AbstractPythonTap implements RunnableTa @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @NotNull @Schema( title = "The database port." ) - @PluginProperty - private Integer port; + private Property port; @NotNull @Schema( title = "The database name." ) - @PluginProperty(dynamic = true) - private String database; + private Property database; @NotNull @Schema( title = "The database name to authenticate on." ) - @PluginProperty(dynamic = true) - private String authDatabase; + private Property authDatabase; @Schema( title = "If ssl is enabled." ) - @PluginProperty @Builder.Default - private final Boolean ssl = false; + private final Property ssl = Property.of(false); @Schema( title = "Default SSL verify mode." ) - @PluginProperty @Builder.Default - private final Boolean sslVerify = true; + private final Property sslVerify = Property.of(true); @Schema( title = "The name of replica set." ) - @PluginProperty(dynamic = true) - private String replicaSet; + private Property replicaSet; @Schema( title = "Forces the stream names to take the form `_` instead of ``." ) - @PluginProperty @Builder.Default - private final Boolean includeSchemaInStream = false; + private final Property includeSchemaInStream = Property.of(false); @Schema( title = "The size of the buffer that holds detected update operations in memory.", description = "For LOG_BASED only, the buffer is flushed once the size is reached." ) - @PluginProperty @Builder.Default - private final Integer updateBufferSize = 1; + private final Property updateBufferSize = Property.of(1); @Schema( title = "The maximum amount of time in milliseconds waits for new data changes before exiting.", description = "For LOG_BASED only." ) - @PluginProperty @Builder.Default - private final Integer awaitTimeMs = 1000; + private final Property awaitTimeMs = Property.of(1000); public List features() { return Arrays.asList( @@ -136,20 +126,20 @@ protected Property command() { public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("user", runContext.render(this.username)) - .put("password", runContext.render(this.password)) + .put("password", runContext.render(this.password).as(String.class).orElse(null)) .put("host", runContext.render(this.host)) - .put("port", this.port) - .put("ssl", this.ssl.toString()) - .put("verify_mode", this.sslVerify.toString()) - .put("database", runContext.render(this.database)) - .put("auth_database", runContext.render(this.authDatabase)) + .put("port", runContext.render(this.port).as(Integer.class).orElseThrow()) + .put("ssl", runContext.render(this.ssl).as(Boolean.class).orElseThrow().toString()) + .put("verify_mode", runContext.render(this.sslVerify).as(Boolean.class).orElseThrow().toString()) + .put("database", runContext.render(this.database).as(String.class).orElseThrow()) + .put("auth_database", runContext.render(this.authDatabase).as(String.class).orElseThrow()) - .put("include_schema_in_destination_stream_name", this.includeSchemaInStream) - .put("update_buffer_size", this.updateBufferSize) - .put("await_time_ms", this.awaitTimeMs); + .put("include_schema_in_destination_stream_name", runContext.render(this.includeSchemaInStream).as(Boolean.class).orElseThrow()) + .put("update_buffer_size", runContext.render(this.updateBufferSize).as(Integer.class).orElseThrow()) + .put("await_time_ms", runContext.render(this.awaitTimeMs).as(Integer.class).orElseThrow()); if (this.replicaSet != null) { - builder.put("replica_set", runContext.render(this.replicaSet)); + builder.put("replica_set", runContext.render(this.replicaSet).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMysql.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMysql.java index 20c21d7..e57d226 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMysql.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseMysql.java @@ -69,47 +69,41 @@ public class PipelinewiseMysql extends AbstractPythonTap implements RunnableTask @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @NotNull @Schema( title = "The database port." ) - @PluginProperty - private Integer port; + private Property port; @Schema( title = "If ssl is enabled." ) - @PluginProperty @Builder.Default - private final Boolean ssl = false; + private final Property ssl = Property.of(false); @Schema( title = "The list of schemas to extract tables only from particular schemas and to improve data extraction performance." ) - @PluginProperty(dynamic = true) - private List filterDbs; + private Property> filterDbs; @Schema( title = "Number of rows to export from MySQL in one batch." ) - @PluginProperty @Builder.Default - private final Integer exportBatchRows = 50000; + private final Property exportBatchRows = Property.of(50000); @Schema( title = "List of SQL commands to run when a connection made. This allows to set session variables dynamically, like timeouts or charsets." ) - @PluginProperty(dynamic = true) @Builder.Default - private final List sessionSqls = Arrays.asList( + private final Property> sessionSqls = Property.of(Arrays.asList( "SET @@session.time_zone=\"+0:00\"", "SET @@session.wait_timeout=28800", "SET @@session.net_read_timeout=3600", "SET @@session.innodb_lock_wait_timeout=3600" - ); + )); private static final String pipPackage = "pipelinewise-tap-mysql"; private static final String comand = "tap-mysql"; @@ -126,15 +120,16 @@ public List features() { public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("user", runContext.render(this.username)) - .put("password", runContext.render(this.password)) + .put("password", runContext.render(this.password).as(String.class).orElse(null)) .put("host", runContext.render(this.host)) - .put("port", this.port) - .put("ssl", this.ssl) - .put("session_sqls", runContext.render(this.sessionSqls)) - .put("export_batch_rows", this.exportBatchRows); - - if (this.filterDbs != null) { - builder.put("filter_dbs", String.join(",", runContext.render(this.filterDbs))); + .put("port", runContext.render(this.port).as(Integer.class).orElseThrow()) + .put("ssl", runContext.render(this.ssl).as(Boolean.class).orElseThrow()) + .put("session_sqls", runContext.render(this.sessionSqls).asList(String.class)) + .put("export_batch_rows", runContext.render(this.exportBatchRows).as(Integer.class).orElseThrow()); + + var filters = runContext.render(this.filterDbs).asList(String.class); + if (!filters.isEmpty()) { + builder.put("filter_dbs", String.join(",", filters)); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseOracle.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseOracle.java index 3d7bd6a..8f728ff 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseOracle.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseOracle.java @@ -70,28 +70,24 @@ public class PipelinewiseOracle extends AbstractPythonTap implements RunnableTas @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @NotNull @Schema( title = "The database port." ) - @PluginProperty - private Integer port; + private Property port; @NotNull @Schema( title = "The database SID." ) - @PluginProperty(dynamic = true) - private String sid; + private Property sid; @Schema( title = "The schemas to filter." ) - @PluginProperty(dynamic = true) - private String filterSchemas; + private Property filterSchemas; public List features() { return Arrays.asList( @@ -105,13 +101,13 @@ public List features() { public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("user", runContext.render(this.username)) - .put("password", runContext.render(this.password)) + .put("password", runContext.render(this.password).as(String.class).orElse(null)) .put("host", runContext.render(this.host)) - .put("port", this.port) - .put("sid", runContext.render(this.sid)); + .put("port", runContext.render(this.port).as(Integer.class).orElseThrow()) + .put("sid", runContext.render(this.sid).as(String.class).orElseThrow()); if (this.filterSchemas != null) { - builder.put("filter_schemas", runContext.render(this.filterSchemas)); + builder.put("filter_schemas", runContext.render(this.filterSchemas).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewisePostgres.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewisePostgres.java index baf845f..d69ba46 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewisePostgres.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewisePostgres.java @@ -46,62 +46,53 @@ public class PipelinewisePostgres extends AbstractPythonTap implements RunnableT @Schema( title = "The database user's password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @Schema( title = "The database name." ) - @PluginProperty(dynamic = true) - private String dbName; + private Property dbName; @NotNull @Schema( title = "The database port." ) - @PluginProperty - private Integer port; + private Property port; @Schema( title = "If ssl is enabled." ) - @PluginProperty @Builder.Default - private final Boolean ssl = false; + private final Property ssl = Property.of(false); @Schema( title = "Stop running the tap when no data received from wal after certain number of seconds." ) - @PluginProperty @Builder.Default - private final Integer logicalPollSeconds = 10800; + private final Property logicalPollSeconds = Property.of(10800); @Schema( title = "Stop running the tap if the newly received lsn is after the max lsn that was detected when the tap started." ) - @PluginProperty @Builder.Default - private final Boolean breakAtEndLsn = true; + private final Property breakAtEndLsn = Property.of(true); @Schema( title = "Stop running the tap after certain number of seconds." ) - @PluginProperty @Builder.Default - private final Integer maxRunSeconds = 43200; + private final Property maxRunSeconds = Property.of(43200); @Schema( title = "If set to \"true\" then add _sdc_lsn property to the singer messages to debug postgres LSN position in the WAL stream." ) - @PluginProperty @Builder.Default - private final Boolean debugLsn = false; + private final Property debugLsn = Property.of(false); @Schema( title = "The list of schemas to extract tables only from particular schemas and to improve data extraction performance" ) - @PluginProperty(dynamic = true) - private List filterSchemas; + private Property> filterSchemas; public List features() { return Arrays.asList( @@ -115,18 +106,19 @@ public List features() { public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("user", runContext.render(this.username)) - .put("password", runContext.render(this.password)) + .put("password", runContext.render(this.password).as(String.class).orElse(null)) .put("host", runContext.render(this.host)) - .put("port", this.port) - .put("dbname", runContext.render(this.dbName)) - .put("ssl", this.ssl) - .put("logical_poll_seconds", this.logicalPollSeconds) - .put("break_at_end_lsn", this.breakAtEndLsn) - .put("max_run_seconds", this.maxRunSeconds) - .put("debug_lsn", this.debugLsn); - - if (this.filterSchemas != null) { - builder.put("filter_dbs", String.join(",", runContext.render(this.filterSchemas))); + .put("port", runContext.render(this.port).as(Integer.class).orElseThrow()) + .put("dbname", runContext.render(this.dbName).as(String.class).orElseThrow()) + .put("ssl", runContext.render(this.ssl).as(Boolean.class).orElseThrow()) + .put("logical_poll_seconds", runContext.render(this.logicalPollSeconds).as(Integer.class).orElseThrow()) + .put("break_at_end_lsn", runContext.render(this.breakAtEndLsn).as(Boolean.class).orElseThrow()) + .put("max_run_seconds", runContext.render(this.maxRunSeconds).as(Integer.class).orElseThrow()) + .put("debug_lsn", runContext.render(this.debugLsn).as(Boolean.class).orElseThrow()); + + var filters = runContext.render(this.filterSchemas).asList(String.class); + if (!filters.isEmpty()) { + builder.put("filter_dbs", String.join(",", filters)); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServer.java b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServer.java index 9671198..bcc27a9 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServer.java +++ b/src/main/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServer.java @@ -70,7 +70,7 @@ public class PipelinewiseSqlServer extends AbstractPythonTap implements Runnable title = "The database port." ) @PluginProperty - private Integer port; + private Property port; @NotEmpty @Schema( @@ -89,42 +89,36 @@ public class PipelinewiseSqlServer extends AbstractPythonTap implements Runnable @Schema( title = "The list of schemas to extract tables only from particular schemas and to improve data extraction performance." ) - @PluginProperty(dynamic = true) - private List filterDbs; + private Property> filterDbs; @NotNull @Schema( title = "Emit Date datatypes as-is without converting them to datetime." ) - @PluginProperty @Builder.Default - private Boolean useDateDatatype = true; + private Property useDateDatatype = Property.of(true); @Schema( title = "TDS version to use when communicating with SQL Server (default is 7.3)." ) - @PluginProperty(dynamic = true) - private String tdsVersion; + private Property tdsVersion; @Schema( title = "The characterset for the database / source system. The default is utf8, however older databases might use a charactersets like cp1252 for the encoding." ) - @PluginProperty(dynamic = true) - private String characterSet; + private Property characterSet; @Schema( title = "Emit all numeric values as strings and treat floats as string data types for the target (default false).", description = "When true, the resulting SCHEMA message will contain an attribute in additionalProperties containing the scale and precision of the discovered property." ) - @PluginProperty - private Boolean useSingerDecimal; + private Property useSingerDecimal; @Schema( title = "A numeric setting adjusting the internal buffersize for the tap (default 10000).", description = "The common query tuning scenario is for SELECT statements that return a large number of rows over a slow network. Increasing arraysize can improve performance by reducing the number of round-trips to the database. However increasing this value increases the amount of memory required." ) - @PluginProperty - private Integer cursorArraySize; + private Property cursorArraySize; public List features() { return Arrays.asList( @@ -139,30 +133,31 @@ public List features() { public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("host", runContext.render(this.host)) - .put("port", this.port) + .put("port", runContext.render(this.port).as(Integer.class).orElseThrow()) .put("user", runContext.render(this.username)) .put("password", runContext.render(this.password)) .put("database", runContext.render(this.database)) - .put("use_date_datatype", this.useDateDatatype); + .put("use_date_datatype", runContext.render(this.useDateDatatype).as(Boolean.class).orElseThrow()); - if (this.filterDbs != null) { - builder.put("filter_dbs", String.join(",", runContext.render(this.filterDbs))); + var filters = runContext.render(this.filterDbs).asList(String.class); + if (!filters.isEmpty()) { + builder.put("filter_dbs", String.join(",", filters)); } if (this.tdsVersion != null) { - builder.put("tds_version", this.tdsVersion); + builder.put("tds_version", runContext.render(this.tdsVersion).as(String.class).orElseThrow()); } if (this.characterSet != null) { - builder.put("characterset", this.characterSet); + builder.put("characterset", runContext.render(this.characterSet).as(String.class).orElseThrow()); } if (this.useSingerDecimal != null) { - builder.put("use_singer_decimal", this.useSingerDecimal); + builder.put("use_singer_decimal", runContext.render(this.useSingerDecimal).as(Boolean.class).orElseThrow()); } if (this.cursorArraySize != null) { - builder.put("cursor_array_size", this.cursorArraySize); + builder.put("cursor_array_size", runContext.render(this.cursorArraySize).as(Integer.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/Quickbooks.java b/src/main/java/io/kestra/plugin/singer/taps/Quickbooks.java index d33e149..c9865ef 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Quickbooks.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Quickbooks.java @@ -35,7 +35,7 @@ public class Quickbooks extends AbstractPythonTap implements RunnableTask selectFieldsByDefault = Property.of(true); @NotNull @Schema( @@ -43,23 +43,21 @@ public class Quickbooks extends AbstractPythonTap implements RunnableTask isSandbox = Property.of(false); @NotNull @Schema( title = "Generate a STATE message every N records." ) - @PluginProperty @Builder.Default - private final Integer stateMessageThreshold = 1000; + private final Property stateMessageThreshold = Property.of(1000); @NotNull @Schema( title = "Maximum number of threads to use." ) - @PluginProperty @Builder.Default - private final Integer maxWorkers = 8; + private final Property maxWorkers = Property.of(8); @NotNull @NotEmpty @@ -112,10 +110,10 @@ public List features() { @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() - .put("select_fields_by_default", this.selectFieldsByDefault) - .put("is_sandbox", this.isSandbox) - .put("state_message_threshold", this.stateMessageThreshold) - .put("max_workers", this.maxWorkers) + .put("select_fields_by_default", runContext.render(this.selectFieldsByDefault).as(Boolean.class).orElseThrow()) + .put("is_sandbox", runContext.render(this.isSandbox).as(Boolean.class).orElseThrow()) + .put("state_message_threshold", runContext.render(this.stateMessageThreshold).as(Integer.class).orElseThrow()) + .put("max_workers", runContext.render(this.maxWorkers).as(Integer.class).orElseThrow()) .put("start_date", runContext.render(this.startDate.toString())) .put("realmId", runContext.render(this.realmId)) .put("client_id", runContext.render(this.clientId)) diff --git a/src/main/java/io/kestra/plugin/singer/taps/Recharge.java b/src/main/java/io/kestra/plugin/singer/taps/Recharge.java index f88737c..609e0b1 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Recharge.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Recharge.java @@ -45,8 +45,7 @@ public class Recharge extends AbstractPythonTap implements RunnableTask userAgent; @NotNull @Schema( @@ -68,7 +67,7 @@ public List features() { public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() .put("access_token", runContext.render(this.accessToken)) - .put("user_agent", runContext.render(this.userAgent)) + .put("user_agent", runContext.render(this.userAgent).as(String.class).orElseThrow()) .put("start_date", runContext.render(this.startDate.toString())); return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/Salesforce.java b/src/main/java/io/kestra/plugin/singer/taps/Salesforce.java index 43cab16..76e62d4 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Salesforce.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Salesforce.java @@ -33,77 +33,66 @@ public class Salesforce extends AbstractPythonTap implements RunnableTask apiType = Property.of(ApiType.BULK); @NotNull @Schema( title = "Select by default any new fields discovered in Salesforce objects." ) - @PluginProperty @Builder.Default - private final Boolean selectFieldsByDefault = true; + private final Property selectFieldsByDefault = Property.of(true); @NotNull @Schema( title = "Select by default any new fields discovered in Salesforce objects." ) - @PluginProperty @Builder.Default - private final Boolean isSandbox = false; + private final Property isSandbox = Property.of(false); @NotNull @Schema( title = "Generate a STATE message every N records." ) - @PluginProperty @Builder.Default - private final Integer stateMessageThreshold = 1000; + private final Property stateMessageThreshold = Property.of(1000); @NotNull @Schema( title = "Maximum number of threads to use." ) - @PluginProperty @Builder.Default - private final Integer maxWorkers = 8; + private final Property maxWorkers = Property.of(8); @Schema( title = "Salesforce username." ) - @PluginProperty(dynamic = true) - private String username; + private Property username; @Schema( title = "Salesforce password." ) - @PluginProperty(dynamic = true) - private String password; + private Property password; @Schema( title = "Your Salesforce Account access token." ) - @PluginProperty(dynamic = true) - private String securityToken; + private Property securityToken; @Schema( title = "Salesforce client ID." ) - @PluginProperty(dynamic = true) - private String clientId; + private Property clientId; @Schema( title = "Salesforce client secret." ) - @PluginProperty(dynamic = true) - private String clientSecret; + private Property clientSecret; @Schema( title = "Salesforce refresh token." ) - @PluginProperty(dynamic = true) - private String refreshToken; + private Property refreshToken; @NotNull @Schema( @@ -124,43 +113,39 @@ public List features() { @Override public Map configuration(RunContext runContext) throws IllegalVariableEvaluationException { ImmutableMap.Builder builder = ImmutableMap.builder() - .put("api_type", this.apiType) - .put("select_fields_by_default", this.selectFieldsByDefault) - .put("is_sandbox", this.isSandbox) - .put("state_message_threshold", this.stateMessageThreshold) - .put("max_workers", this.maxWorkers) + .put("api_type", runContext.render(this.apiType).as(ApiType.class).orElseThrow()) + .put("select_fields_by_default", runContext.render(this.selectFieldsByDefault).as(Boolean.class).orElseThrow()) + .put("is_sandbox", runContext.render(this.isSandbox).as(Boolean.class).orElseThrow()) + .put("state_message_threshold", runContext.render(this.stateMessageThreshold).as(Integer.class).orElseThrow()) + .put("max_workers", runContext.render(this.maxWorkers).as(Integer.class).orElseThrow()) .put("start_date", runContext.render(this.startDate.toString())); if (this.username != null) { - builder.put("username", runContext.render(this.username)); + builder.put("username", runContext.render(this.username).as(String.class).orElseThrow()); } if (this.password != null) { - builder.put("password", runContext.render(this.password)); + builder.put("password", runContext.render(this.password).as(String.class).orElseThrow()); } if (this.securityToken != null) { - builder.put("security_token", runContext.render(this.securityToken)); + builder.put("security_token", runContext.render(this.securityToken).as(String.class).orElseThrow()); } if (this.clientId != null) { - builder.put("client_id", runContext.render(this.clientId)); + builder.put("client_id", runContext.render(this.clientId).as(String.class).orElseThrow()); } if (this.clientSecret != null) { - builder.put("client_secret", runContext.render(this.clientSecret)); + builder.put("client_secret", runContext.render(this.clientSecret).as(String.class).orElseThrow()); } if (this.refreshToken != null) { - builder.put("refresh_token", runContext.render(this.refreshToken)); - } - - if (this.username != null) { - builder.put("username", runContext.render(this.username)); + builder.put("refresh_token", runContext.render(this.refreshToken).as(String.class).orElseThrow()); } if (this.username != null) { - builder.put("username", runContext.render(this.username)); + builder.put("username", runContext.render(this.username).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/Slack.java b/src/main/java/io/kestra/plugin/singer/taps/Slack.java index da7bbd9..fe5fd4c 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Slack.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Slack.java @@ -50,17 +50,15 @@ public class Slack extends AbstractPythonTap implements RunnableTask privateChannels = Property.of(true); @Schema( title = "Join Public Channels.", description = "Specifies whether to have the tap auto-join all public channels in your ogranziation." ) - @PluginProperty @Builder.Default - private final Boolean publicChannels = false; + private final Property publicChannels = Property.of(false); @Schema( title = "Sync Archived Channels.", @@ -68,9 +66,8 @@ public class Slack extends AbstractPythonTap implements RunnableTask archivedChannels = Property.of(false); @Schema( title = "Channels to Sync.", @@ -78,8 +75,7 @@ public class Slack extends AbstractPythonTap implements RunnableTask channels; + private Property> channels; @Schema( title = "Channels to Sync.", @@ -87,9 +83,8 @@ public class Slack extends AbstractPythonTap implements RunnableTask dateWindowSize = Property.of(7); public List features() { return Arrays.asList( @@ -104,16 +99,17 @@ public Map configuration(RunContext runContext) throws IllegalVa ImmutableMap.Builder builder = ImmutableMap.builder() .put("token", runContext.render(this.apiToken)) .put("start_date", runContext.render(this.startDate.toString())) - .put("private_channels", this.privateChannels) - .put("join_public_channels", this.publicChannels) - .put("archived_channels", this.archivedChannels); + .put("private_channels", runContext.render(this.privateChannels).as(Boolean.class).orElseThrow()) + .put("join_public_channels", runContext.render(this.publicChannels).as(Boolean.class).orElseThrow()) + .put("archived_channels", runContext.render(this.archivedChannels).as(Boolean.class).orElseThrow()); - if (channels != null) { - builder.put("channels", runContext.render(this.channels)); + var renderedChannels = runContext.render(this.channels).asList(String.class); + if (!renderedChannels.isEmpty()) { + builder.put("channels", renderedChannels); } if (dateWindowSize != null) { - builder.put("date_window_size", this.dateWindowSize); + builder.put("date_window_size", runContext.render(this.dateWindowSize).as(Integer.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/Zendesk.java b/src/main/java/io/kestra/plugin/singer/taps/Zendesk.java index 616a7a1..90d4a60 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Zendesk.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Zendesk.java @@ -44,21 +44,18 @@ public class Zendesk extends AbstractPythonTap implements RunnableTask email; @Schema( title = "Zendesk API token." ) - @PluginProperty(dynamic = true) - private String apiToken; + private Property apiToken; @Schema( title = "Zendesk access token.", description = "See [Zendesk Documentation](https://support.zendesk.com/hc/en-us/articles/203663836)" ) - @PluginProperty(dynamic = true) - private String accessToken; + private Property accessToken; @NotNull @Schema( @@ -83,15 +80,15 @@ public Map configuration(RunContext runContext) throws IllegalVa .put("start_date", runContext.render(this.startDate.toString())); if (this.email != null) { - builder.put("email", runContext.render(this.email)); + builder.put("email", runContext.render(this.email).as(String.class).orElseThrow()); } if (this.apiToken != null) { - builder.put("api_token", runContext.render(this.apiToken)); + builder.put("api_token", runContext.render(this.apiToken).as(String.class).orElseThrow()); } if (this.accessToken != null) { - builder.put("access_token", runContext.render(this.accessToken)); + builder.put("access_token", runContext.render(this.accessToken).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/main/java/io/kestra/plugin/singer/taps/Zoom.java b/src/main/java/io/kestra/plugin/singer/taps/Zoom.java index d207117..ebeb202 100644 --- a/src/main/java/io/kestra/plugin/singer/taps/Zoom.java +++ b/src/main/java/io/kestra/plugin/singer/taps/Zoom.java @@ -32,26 +32,22 @@ public class Zoom extends AbstractPythonTap implements RunnableTask jwt; @Schema( title = "Zoom client id." ) - @PluginProperty(dynamic = true) - private String clientId; + private Property clientId; @Schema( title = "Zoom Client secret." ) - @PluginProperty(dynamic = true) - private String clientSecret; + private Property clientSecret; @Schema( title = "Zoom refresh token." ) - @PluginProperty(dynamic = true) - private String refreshToken; + private Property refreshToken; public List features() { return Arrays.asList( @@ -65,19 +61,19 @@ public Map configuration(RunContext runContext) throws IllegalVa ImmutableMap.Builder builder = ImmutableMap.builder(); if (this.jwt != null) { - builder.put("jwt", runContext.render(this.jwt)); + builder.put("jwt", runContext.render(this.jwt).as(String.class).orElseThrow()); } if (this.clientId != null) { - builder.put("client_id", runContext.render(this.clientId)); + builder.put("client_id", runContext.render(this.clientId).as(String.class).orElseThrow()); } if (this.clientSecret != null) { - builder.put("client_secret", runContext.render(this.clientSecret)); + builder.put("client_secret", runContext.render(this.clientSecret).as(String.class).orElseThrow()); } if (this.refreshToken != null) { - builder.put("refresh_token", runContext.render(this.refreshToken)); + builder.put("refresh_token", runContext.render(this.refreshToken).as(String.class).orElseThrow()); } return builder.build(); diff --git a/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDbTest.java b/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDbTest.java index 3941583..e0fac4d 100644 --- a/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDbTest.java +++ b/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseMongoDbTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.taps; import com.google.common.collect.ImmutableMap; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -29,10 +30,10 @@ void run() throws Exception { .type(PipelinewiseMongoDb.class.getName()) .host("172.17.0.1") .username("root") - .password("example") - .port(57017) - .database("samples") - .authDatabase("admin") + .password(Property.of("example")) + .port(Property.of(57017)) + .database(Property.of("samples")) + .authDatabase(Property.of("admin")) .streamsConfigurations(Collections.singletonList( StreamsConfiguration.builder() .stream("books") diff --git a/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseMysqlTest.java b/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseMysqlTest.java index 8bb4f0a..4e7c474 100644 --- a/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseMysqlTest.java +++ b/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseMysqlTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.taps; import com.google.common.collect.ImmutableMap; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -30,8 +31,8 @@ void run() throws Exception { .type(PipelinewiseMysql.class.getName()) .host("172.17.0.1") .username("root") - .password("mysql_passwd") - .port(63306) + .password(Property.of("mysql_passwd")) + .port(Property.of(63306)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("Category") diff --git a/src/test/java/io/kestra/plugin/singer/taps/PipelinewisePostgresTest.java b/src/test/java/io/kestra/plugin/singer/taps/PipelinewisePostgresTest.java index aadaf95..3e36179 100644 --- a/src/test/java/io/kestra/plugin/singer/taps/PipelinewisePostgresTest.java +++ b/src/test/java/io/kestra/plugin/singer/taps/PipelinewisePostgresTest.java @@ -1,6 +1,7 @@ package io.kestra.plugin.singer.taps; import com.google.common.collect.ImmutableMap; +import io.kestra.core.models.property.Property; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.utils.IdUtils; @@ -30,9 +31,9 @@ void run() throws Exception { .type(PipelinewisePostgres.class.getName()) .host("172.17.0.1") .username("postgres") - .password("pg_passwd") - .port(65432) - .dbName("postgres") + .password(Property.of("pg_passwd")) + .port(Property.of(65432)) + .dbName(Property.of("postgres")) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("category") diff --git a/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServerTest.java b/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServerTest.java index 1d28301..8fc9cdc 100644 --- a/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServerTest.java +++ b/src/test/java/io/kestra/plugin/singer/taps/PipelinewiseSqlServerTest.java @@ -33,8 +33,8 @@ void run() throws Exception { .database("msdb") .username("SA") .password("SQLServer_Passwd") - .port(57037) - .filterDbs(Collections.singletonList("dbo")) + .port(Property.of(57037)) + .filterDbs(Property.of(Collections.singletonList("dbo"))) .stateName(Property.of("tap-test")) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() diff --git a/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java b/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java index d179fe4..90a9893 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/AdswerveBigQueryTest.java @@ -51,8 +51,8 @@ void run() throws Exception { .type(PipelinewiseMysql.class.getName()) .host("172.17.0.1") .username("root") - .password("mysql_passwd") - .port(63306) + .password(Property.of("mysql_passwd")) + .port(Property.of(63306)) .stateName(Property.of(stateName)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() diff --git a/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java b/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java index db10479..b9203b5 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/CsvTest.java @@ -35,8 +35,8 @@ void run() throws Exception { .type(PipelinewiseMysql.class.getName()) .host("172.17.0.1") .username("root") - .password("mysql_passwd") - .port(63306) + .password(Property.of("mysql_passwd")) + .port(Property.of(63306)) .stateName(Property.of(stateName)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() diff --git a/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java b/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java index 9716a4d..55c5db8 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/DatamillCoPostgresTest.java @@ -36,8 +36,8 @@ void run() throws Exception { .type(PipelinewiseMysql.class.getName()) .host("172.17.0.1") .username("root") - .password("mysql_passwd") - .port(63306) + .password(Property.of("mysql_passwd")) + .port(Property.of(63306)) .stateName(Property.of(stateName)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() diff --git a/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java b/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java index 4073b8d..5d75064 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/JsonTest.java @@ -35,8 +35,8 @@ void run() throws Exception { .type(PipelinewiseMysql.class.getName()) .host("172.17.0.1") .username("root") - .password("mysql_passwd") - .port(63306) + .password(Property.of("mysql_passwd")) + .port(Property.of(63306)) .stateName(Property.of(stateName)) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() diff --git a/src/test/java/io/kestra/plugin/singer/targets/OracleTest.java b/src/test/java/io/kestra/plugin/singer/targets/OracleTest.java index cbdd136..efbf986 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/OracleTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/OracleTest.java @@ -35,8 +35,8 @@ void run() throws Exception { .database("msdb") .username("SA") .password("SQLServer_Passwd") - .port(57037) - .filterDbs(Collections.singletonList("dbo")) + .port(Property.of(57037)) + .filterDbs(Property.of(Collections.singletonList("dbo"))) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() .stream("Categories") diff --git a/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java b/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java index 84d1cbe..a6512ac 100644 --- a/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java +++ b/src/test/java/io/kestra/plugin/singer/targets/SqlServerTest.java @@ -37,8 +37,8 @@ void run() throws Exception { .database("msdb") .username("SA") .password("SQLServer_Passwd") - .port(57037) - .filterDbs(Collections.singletonList("dbo")) + .port(Property.of(57037)) + .filterDbs(Property.of(Collections.singletonList("dbo"))) .stateName(Property.of("before-target-test")) .streamsConfigurations(Arrays.asList( StreamsConfiguration.builder() @@ -84,7 +84,7 @@ void run() throws Exception { assertThat(output.getStateKey(), not((nullValue()))); tap = tapBuilder - .filterDbs(Collections.singletonList("target")) + .filterDbs(Property.of(Collections.singletonList("target"))) .stateName(Property.of("after-target-test")) .streamsConfigurations(Collections.singletonList( StreamsConfiguration.builder()