From 9ab8fdb57bc4bc3152d461d0402faa72b2158e2b Mon Sep 17 00:00:00 2001 From: Jurrie Overgoor <1213142+Jurrie@users.noreply.github.com> Date: Fri, 16 Feb 2024 15:45:29 +0100 Subject: [PATCH] Better way to escape Helm (also unescape afterwards) Signed-off-by: Jurrie Overgoor --- .../jkube/kit/common/util/Serialization.java | 51 ++++-- .../jkube/kit/common/util/TemplateUtil.java | 171 ++++++++++++++---- .../kit/common/util/TemplateUtilTest.java | 88 +++++++-- .../jkube/kit/resource/helm/HelmService.java | 107 ++++++----- .../kit/resource/helm/HelmServiceIT.java | 14 +- .../kubernetes/templates/kubernetes.yaml | 15 +- .../it/expected/kubernetes/values.yaml | 2 +- .../openshift/templates/openshift.yaml | 10 +- .../it/expected/openshift/values.yaml | 2 +- .../it/sources/kubernetes/kubernetes.yml | 15 +- .../it/sources/openshift/openshift.yml | 10 +- 11 files changed, 338 insertions(+), 147 deletions(-) diff --git a/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/Serialization.java b/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/Serialization.java index dea97ca82e..f2ca476667 100644 --- a/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/Serialization.java +++ b/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/Serialization.java @@ -13,6 +13,18 @@ */ package org.eclipse.jkube.kit.common.util; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Collectors; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; @@ -24,15 +36,6 @@ import io.fabric8.kubernetes.api.model.KubernetesResource; import io.fabric8.kubernetes.client.utils.KubernetesSerialization; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; - public class Serialization { private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); @@ -97,20 +100,22 @@ public static T unmarshal(URL url, TypeReference type) throws IOException } } - public static T unmarshal(InputStream is, Class clazz) { - return KUBERNETES_SERIALIZATION.unmarshal(is, clazz); + public static T unmarshal(final InputStream is, final Class clazz) { + return unmarshal(inputStreamToString(is), clazz); } - public static T unmarshal(InputStream is, TypeReference type) { - return KUBERNETES_SERIALIZATION.unmarshal(is, type); + public static T unmarshal(final InputStream is, final TypeReference type) { + return unmarshal(inputStreamToString(is), type); } public static T unmarshal(String string, Class clazz) { - return KUBERNETES_SERIALIZATION.unmarshal(string, clazz); + final byte[] yaml = TemplateUtil.escapeYamlTemplate(string).getBytes(StandardCharsets.UTF_8); + return KUBERNETES_SERIALIZATION.unmarshal(new ByteArrayInputStream(yaml), clazz); } - public static T unmarshal(String string, TypeReference type) { - return unmarshal(new ByteArrayInputStream(string.getBytes(StandardCharsets.UTF_8)), type); + public static T unmarshal(final String string, final TypeReference type) { + final byte[] yaml = TemplateUtil.escapeYamlTemplate(string).getBytes(StandardCharsets.UTF_8); + return KUBERNETES_SERIALIZATION.unmarshal(new ByteArrayInputStream(yaml), type); } public static T merge(T original, T overrides) throws IOException { @@ -131,7 +136,8 @@ public static ObjectWriter jsonWriter() { } public static String asYaml(Object object) { - return KUBERNETES_SERIALIZATION.asYaml(object); + final String yamlString = KUBERNETES_SERIALIZATION.asYaml(object); + return TemplateUtil.unescapeYamlTemplate(yamlString); } public static void saveJson(File resultFile, Object value) throws IOException { @@ -139,6 +145,15 @@ public static void saveJson(File resultFile, Object value) throws IOException { } public static void saveYaml(File resultFile, Object value) throws IOException { - YAML_MAPPER.writeValue(resultFile, value); + String yamlString = YAML_MAPPER.writeValueAsString(value); + yamlString = TemplateUtil.unescapeYamlTemplate(yamlString); + Files.write(resultFile.toPath(), yamlString.getBytes(StandardCharsets.UTF_8)); + } + + private static String inputStreamToString(final InputStream is) { + return new BufferedReader( + new InputStreamReader(is, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); } } diff --git a/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/TemplateUtil.java b/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/TemplateUtil.java index 1aba31c50d..138ba51fae 100644 --- a/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/TemplateUtil.java +++ b/jkube-kit/common/src/main/java/org/eclipse/jkube/kit/common/util/TemplateUtil.java @@ -13,49 +13,152 @@ */ package org.eclipse.jkube.kit.common.util; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class TemplateUtil { + private static final String HELM_DIRECTIVE_REGEX = "\\{\\{.*\\}\\}"; private TemplateUtil() { } /** - * Ported from https://github.com/fabric8io/fabric8-maven-plugin/commit/d6bdaa37e06863677bc01cefa31f7d23c6d5f0f9 + * This function will replace all Helm directives with a valid Yaml line containing the base64 encoded Helm directive. + * + * Helm lines that are by themselves will be converted like so: + *
+ * Input: + * + * 
+   * {{- $myDate := .Value.date }}
+   * {{ include "something" . }}
+   * someKey: {{ a bit of Helm }}
+   * someOtherKey: {{ another bit of Helm }}
+   *
+ * + * Output: + * + * 
+   * escapedHelm0: BASE64STRINGOFCHARACTERS=
+   * escapedHelm1: ANOTHERBASE64STRING=
+   * someKey: escapedHelmValueBASE64STRING==
+   * someOtherKey: escapedHelmValueBASE64STRING
+   *
+ * + * The escapedHelm and escapedHelmValue flags are needed for unescaping. + * + * @param yaml the input Yaml with Helm directives to be escaped + * @return the same Yaml, only with Helm directives converted to valid Yaml + * @see #unescapeYamlTemplate(String) + */ + public static String escapeYamlTemplate(final String yaml) { + return escapeYamlTemplateLines(escapeYamlTemplateValues(yaml)); + } + + /** + * This function will replace all escaped Helm directives by {@link #escapeYamlTemplate(String)} back to actual Helm. + *
+ * This function promises to be the opposite of {@link #escapeYamlTemplate(String)}. + * + * @param template the input Yaml that was returned by a call to {@link #escapeYamlTemplate(String)} + * @return the Yaml that was originally provided to {@link #escapeYamlTemplate(String)} + * @see #escapeYamlTemplate(String) + */ + public static String unescapeYamlTemplate(final String template) { + return unescapeYamlTemplateLines(unescapeYamlTemplateValues(template)); + } + + /** + * This function is responsible for escaping the Helm directives that are stand-alone. + * For example: + * + * 
+   * {{ include "something" . }}
+   *
* - * @param template String to escape - * @return the escaped Yaml template + * @see #unescapeYamlTemplateLines(String) */ - public static String escapeYamlTemplate(String template) { - StringBuilder answer = new StringBuilder(); - int count = 0; - char last = 0; - for (int i = 0, size = template.length(); i < size; i++) { - char ch = template.charAt(i); - if (ch == '{' || ch == '}') { - if (count == 0) { - last = ch; - count = 1; - } else { - if (ch == last) { - answer.append(ch == '{' ? "{{\"{{\"}}" : "{{\"}}\"}}"); - } else { - answer.append(last); - answer.append(ch); - } - count = 0; - last = 0; - } - } else { - if (count > 0) { - answer.append(last); - } - answer.append(ch); - count = 0; - last = 0; - } + private static String escapeYamlTemplateLines(String template) { + long escapedHelmIndex = 0; + final Pattern compile = Pattern.compile("^( *-? *)(" + HELM_DIRECTIVE_REGEX + ".*)$", Pattern.MULTILINE); + Matcher matcher = compile.matcher(template); + while (matcher.find()) { + final String indentation = matcher.group(1); + final String base64Line = Base64Util.encodeToString(matcher.group(2)); + template = matcher.replaceFirst(indentation + "escapedHelm" + escapedHelmIndex + ": " + base64Line); + matcher = compile.matcher(template); + escapedHelmIndex++; } - if (count > 0) { - answer.append(last); + return template; + } + + /** + * This function is responsible for reinstating the stand-alone Helm directives. + * For example: + * + * 
+   * BASE64STRINGOFCHARACTERS=
+   *
+ * + * It is the opposite of {@link #escapeYamlTemplateLines(String)}. + * + * @see #escapeYamlTemplateLines(String) + */ + private static String unescapeYamlTemplateLines(String template) { + final Pattern compile = Pattern.compile("^( *-? *)escapedHelm[\\d]+: \"?(.*?)\"?$", Pattern.MULTILINE); + Matcher matcher = compile.matcher(template); + while (matcher.find()) { + final String indentation = matcher.group(1); + final String helmLine = Base64Util.decodeToString(matcher.group(2)); + template = matcher.replaceFirst(indentation + helmLine.replace("$", "\\$")); + matcher = compile.matcher(template); + } + return template; + } + + /** + * This function is responsible for escaping the Helm directives that are Yaml values. + * For example: + * + * 
+   * someKey: {{ a bit of Helm }}
+   *
+ * + * @see #unescapeYamlTemplateValues(String) + */ + private static String escapeYamlTemplateValues(String template) { + final Pattern compile = Pattern.compile("^( *[^ ]+ *): *(" + HELM_DIRECTIVE_REGEX + ".*)$", Pattern.MULTILINE); + Matcher matcher = compile.matcher(template); + while (matcher.find()) { + final String indentation = matcher.group(1); + final String base64Value = Base64Util.encodeToString(matcher.group(2)); + template = matcher.replaceFirst(indentation + ": escapedHelmValue" + base64Value); + matcher = compile.matcher(template); + } + return template; + } + + /** + * This function is responsible for reinstating the Helm directives that were Yaml values. + * For example: + * + * 
+   * someKey: escapedHelmValueBASE64STRING==
+   *
+ * + * It is the opposite of {@link #escapeYamlTemplateValues(String)}. + * + * @see #escapeYamlTemplateValues(String) + */ + private static String unescapeYamlTemplateValues(String template) { + final Pattern compile = Pattern.compile("^( *[^ ]+ *): *\"?escapedHelmValue(.*?)\"?$", Pattern.MULTILINE); + Matcher matcher = compile.matcher(template); + while (matcher.find()) { + final String indentation = matcher.group(1); + final String helmValue = Base64Util.decodeToString(matcher.group(2)); + template = matcher.replaceFirst(indentation + ": " + helmValue.replace("$", "\\$")); + matcher = compile.matcher(template); } - return answer.toString(); + return template; } } diff --git a/jkube-kit/common/src/test/java/org/eclipse/jkube/kit/common/util/TemplateUtilTest.java b/jkube-kit/common/src/test/java/org/eclipse/jkube/kit/common/util/TemplateUtilTest.java index 8ee952061a..2280cfde60 100644 --- a/jkube-kit/common/src/test/java/org/eclipse/jkube/kit/common/util/TemplateUtilTest.java +++ b/jkube-kit/common/src/test/java/org/eclipse/jkube/kit/common/util/TemplateUtilTest.java @@ -13,28 +13,80 @@ */ package org.eclipse.jkube.kit.common.util; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.jkube.kit.common.util.TemplateUtil.escapeYamlTemplate; +import static org.eclipse.jkube.kit.common.util.TemplateUtil.unescapeYamlTemplate; import java.util.stream.Stream; -import static org.assertj.core.api.Assertions.assertThat; -import static org.eclipse.jkube.kit.common.util.TemplateUtil.escapeYamlTemplate; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; class TemplateUtilTest { - public static Stream data() { - return Stream.of(new Object[][]{ - {"abcd", "abcd"}, - {"abc{de}f}", "abc{de}f}"}, - {"abc{{de}f", "abc{{\"{{\"}}de}f"}, - {"abc{{de}f}}", "abc{{\"{{\"}}de}f{{\"}}\"}}"} - }); - } - - @ParameterizedTest(name = "{0} is {1}") - @MethodSource("data") - void escapeYamlTemplateTest(String input, String expected) { - assertThat(escapeYamlTemplate(input)).isEqualTo(expected); - } + public static Stream data() { + return Stream.of(new Object[][] { + // No Helm directive + { "abcd", "abcd" }, + + // When the Helm directive is not the first on the line + { "abc{de}f}", "abc{de}f}" }, + { "abc{{de}f", "abc{{de}f" }, + { "abc{{$def}}", "abc{{$def}}" }, + { "abc{{de}}f", "abc{{de}}f" }, + { "abc{{de}f}}", "abc{{de}f}}" }, + { "abc{{def}}ghi{{jkl}}mno", "abc{{def}}ghi{{jkl}}mno" }, + + // When the Helm directive is the first on the line + { "{de}f}", "{de}f}" }, + { "{{de}f", "{{de}f" }, + { "{{$def}}", "escapedHelm0: " + Base64Util.encodeToString("{{$def}}") }, + { "{{de}}f", "escapedHelm0: " + Base64Util.encodeToString("{{de}}f") }, + { "{{de}f}}", "escapedHelm0: " + Base64Util.encodeToString("{{de}f}}") }, + { "{{def}}ghi{{jkl}}mno", "escapedHelm0: " + Base64Util.encodeToString("{{def}}ghi{{jkl}}mno") }, + { "hello\n{{def}}\nworld", "hello\nescapedHelm0: " + Base64Util.encodeToString("{{def}}") + "\nworld" }, + { "- hello\n- {{def}}\n- world", "- hello\n- escapedHelm0: " + Base64Util.encodeToString("{{def}}") + "\n- world" }, + { "{{multiple}}\n{{helm}}\n{{lines}}", + "escapedHelm0: " + Base64Util.encodeToString("{{multiple}}") + "\n" + + "escapedHelm1: " + Base64Util.encodeToString("{{helm}}") + "\n" + + "escapedHelm2: " + Base64Util.encodeToString("{{lines}}") }, + + // When the Helm directive is the first on the line, but indented + { " {de}f}", " {de}f}" }, + { " {{de}f", " {{de}f" }, + { " {{$def}}", " escapedHelm0: " + Base64Util.encodeToString("{{$def}}") }, + { " {{de}}f", " escapedHelm0: " + Base64Util.encodeToString("{{de}}f") }, + { " {{de}f}}", " escapedHelm0: " + Base64Util.encodeToString("{{de}f}}") }, + { " {{def}}ghi{{jkl}}mno", " escapedHelm0: " + Base64Util.encodeToString("{{def}}ghi{{jkl}}mno") }, + { "hello:\n {{def}}\n world", "hello:\n escapedHelm0: " + Base64Util.encodeToString("{{def}}") + "\n world" }, + { "hello:\n - {{def}}\n - world", + "hello:\n - escapedHelm0: " + Base64Util.encodeToString("{{def}}") + "\n - world" }, + + // When the Helm directive is a value + { "key: {de}f}", "key: {de}f}" }, + { "key: {{de}f", "key: {{de}f" }, + { "key: {{$def}}", "key: escapedHelmValue" + Base64Util.encodeToString("{{$def}}") }, + { "key: {{de}}f", "key: escapedHelmValue" + Base64Util.encodeToString("{{de}}f") }, + { "key: {{de}f}}", "key: escapedHelmValue" + Base64Util.encodeToString("{{de}f}}") }, + { "key: {{def}}ghi{{jkl}}mno", "key: escapedHelmValue" + Base64Util.encodeToString("{{def}}ghi{{jkl}}mno") }, + + // When the Helm directive is a value, but indented + { " key: {de}f}", " key: {de}f}" }, + { " key: {{de}f", " key: {{de}f" }, + { " key: {{$def}}", " key: escapedHelmValue" + Base64Util.encodeToString("{{$def}}") }, + { " key: {{de}}f", " key: escapedHelmValue" + Base64Util.encodeToString("{{de}}f") }, + { " key: {{de}f}}", " key: escapedHelmValue" + Base64Util.encodeToString("{{de}f}}") }, + { " key: {{def}}ghi{{jkl}}mno", " key: escapedHelmValue" + Base64Util.encodeToString("{{def}}ghi{{jkl}}mno") }, + }); + } + + @ParameterizedTest(name = "{0} → {1}") + @MethodSource("data") + void escapeYamlTemplateTest(final String input, final String expected) { + final String escapedYaml = escapeYamlTemplate(input); + assertThat(escapedYaml).isEqualTo(expected); + + final String unescapedYaml = unescapeYamlTemplate(escapedYaml); + assertThat(input).isEqualTo(unescapedYaml); + } } \ No newline at end of file diff --git a/jkube-kit/helm/src/main/java/org/eclipse/jkube/kit/resource/helm/HelmService.java b/jkube-kit/helm/src/main/java/org/eclipse/jkube/kit/resource/helm/HelmService.java index 634d4ab1c6..59ca869eee 100644 --- a/jkube-kit/helm/src/main/java/org/eclipse/jkube/kit/resource/helm/HelmService.java +++ b/jkube-kit/helm/src/main/java/org/eclipse/jkube/kit/resource/helm/HelmService.java @@ -13,10 +13,18 @@ */ package org.eclipse.jkube.kit.resource.helm; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.eclipse.jkube.kit.common.JKubeFileInterpolator.DEFAULT_FILTER; +import static org.eclipse.jkube.kit.common.JKubeFileInterpolator.interpolate; +import static org.eclipse.jkube.kit.common.util.MapUtil.getNestedMap; +import static org.eclipse.jkube.kit.common.util.YamlUtil.listYamls; +import static org.eclipse.jkube.kit.resource.helm.HelmServiceUtil.isRepositoryValid; +import static org.eclipse.jkube.kit.resource.helm.HelmServiceUtil.selectHelmRepository; +import static org.eclipse.jkube.kit.resource.helm.HelmServiceUtil.setAuthentication; + import java.io.File; import java.io.IOException; import java.nio.charset.Charset; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -36,6 +44,12 @@ import com.marcnuri.helm.Helm; import com.marcnuri.helm.LintCommand; import com.marcnuri.helm.LintResult; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.fabric8.kubernetes.api.model.KubernetesResource; +import io.fabric8.openshift.api.model.Template; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.eclipse.jkube.kit.common.JKubeConfiguration; import org.eclipse.jkube.kit.common.JKubeException; import org.eclipse.jkube.kit.common.KitLogger; @@ -51,24 +65,6 @@ import org.eclipse.jkube.kit.config.resource.ResourceServiceConfig; import org.eclipse.jkube.kit.enricher.api.util.KubernetesResourceFragments; -import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.api.model.KubernetesResource; -import io.fabric8.openshift.api.model.Template; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; - - -import static org.apache.commons.lang3.StringUtils.EMPTY; -import static org.eclipse.jkube.kit.common.JKubeFileInterpolator.DEFAULT_FILTER; -import static org.eclipse.jkube.kit.common.JKubeFileInterpolator.interpolate; -import static org.eclipse.jkube.kit.common.util.MapUtil.getNestedMap; -import static org.eclipse.jkube.kit.common.util.TemplateUtil.escapeYamlTemplate; -import static org.eclipse.jkube.kit.common.util.YamlUtil.listYamls; -import static org.eclipse.jkube.kit.resource.helm.HelmServiceUtil.isRepositoryValid; -import static org.eclipse.jkube.kit.resource.helm.HelmServiceUtil.selectHelmRepository; -import static org.eclipse.jkube.kit.resource.helm.HelmServiceUtil.setAuthentication; - public class HelmService { private static final String YAML_EXTENSION = ".yaml"; @@ -125,8 +121,8 @@ public void generateHelmCharts(HelmConfig helmConfig) throws IOException { final File tarballFile = new File(tarballOutputDir, String.format("%s-%s%s.%s", helmConfig.getChart(), helmConfig.getVersion(), resolveHelmClassifier(helmConfig), helmConfig.getChartExtension())); logger.debug("Creating Helm configuration Tarball: '%s'", tarballFile.getAbsolutePath()); - final Consumer prependNameAsDirectory = tae -> - tae.setName(String.format("%s/%s", helmConfig.getChart(), tae.getName())); + final Consumer prependNameAsDirectory = tae -> tae + .setName(String.format("%s/%s", helmConfig.getChart(), tae.getName())); // outputDir might contain tarball already from previous run, filter out tarball file outputDir from recursive listing List helmTarballContents = FileUtil.listFilesAndDirsRecursivelyInDirectory(outputDir).stream() .filter(f -> !f.equals(tarballFile)) @@ -142,7 +138,8 @@ public void generateHelmCharts(HelmConfig helmConfig) throws IOException { /** * Uploads the charts defined in the provided {@link HelmConfig} to the applicable configured repository. * - *

For Charts with versions ending in "-SNAPSHOT" the {@link HelmConfig#getSnapshotRepository()} is used. + *

+ * For Charts with versions ending in "-SNAPSHOT" the {@link HelmConfig#getSnapshotRepository()} is used. * {@link HelmConfig#getStableRepository()} is used for other versions. * * @@ -204,30 +201,31 @@ private void uploadHelmChart(HelmConfig helmConfig, HelmRepository helmRepositor logger.info("Uploading Helm Chart \"%s\" to %s", helmConfig.getChart(), helmRepository.getName()); logger.debug("OutputDir: %s", helmConfig.getOutputDir()); helmUploaderManager.getHelmUploader(helmRepository.getType()) - .uploadSingle(resolveTarballFile(helmConfig, helmType).toFile(), helmRepository); + .uploadSingle(resolveTarballFile(helmConfig, helmType).toFile(), helmRepository); logger.info("Upload Successful"); } } private static Path resolveTarballFile(HelmConfig helmConfig, HelmConfig.HelmType helmType) { return Paths.get(Objects.requireNonNull(helmConfig.getTarballOutputDir(), "Tarball output directory is required")) - .resolve(helmType.getOutputDir()) - .resolve(String.format("%s-%s%s.%s", helmConfig.getChart(), helmConfig.getVersion(), resolveHelmClassifier(helmConfig), helmConfig.getChartExtension())); + .resolve(helmType.getOutputDir()) + .resolve(String.format("%s-%s%s.%s", helmConfig.getChart(), helmConfig.getVersion(), resolveHelmClassifier(helmConfig), + helmConfig.getChartExtension())); } static File prepareSourceDir(HelmConfig helmConfig, HelmConfig.HelmType type) throws IOException { final File sourceDir = new File(helmConfig.getSourceDir(), type.getSourceDir()); if (!sourceDir.isDirectory()) { throw new IOException(String.format( - "Chart source directory %s does not exist so cannot make chart \"%s\". " + - "Probably you need run 'mvn kubernetes:resource' before.", - sourceDir.getAbsolutePath(), helmConfig.getChart())); + "Chart source directory %s does not exist so cannot make chart \"%s\". " + + "Probably you need run 'mvn kubernetes:resource' before.", + sourceDir.getAbsolutePath(), helmConfig.getChart())); } if (!containsYamlFiles(sourceDir)) { throw new IOException(String.format( - "Chart source directory %s does not contain any YAML manifest to make chart \"%s\". " + - "Probably you need run 'mvn kubernetes:resource' before.", - sourceDir.getAbsolutePath(), helmConfig.getChart())); + "Chart source directory %s does not contain any YAML manifest to make chart \"%s\". " + + "Probably you need run 'mvn kubernetes:resource' before.", + sourceDir.getAbsolutePath(), helmConfig.getChart())); } return sourceDir; } @@ -241,7 +239,6 @@ private static File prepareOutputDir(HelmConfig helmConfig, HelmConfig.HelmType return outputDir; } - public static boolean containsYamlFiles(File directory) { return !listYamls(directory).isEmpty(); } @@ -252,12 +249,9 @@ private static void processSourceFiles(File sourceDir, File templatesDir) throws if (dto instanceof Template) { splitAndSaveTemplate((Template) dto, templatesDir); } else { - final String fileName = FileUtil.stripPostfix(FileUtil.stripPostfix(file.getName(), ".yml"), YAML_EXTENSION) + YAML_EXTENSION; - File targetFile = new File(templatesDir, fileName); - // lets escape any {{ or }} characters to avoid creating invalid templates - String text = FileUtils.readFileToString(file, Charset.defaultCharset()); - text = escapeYamlTemplate(text); - FileUtils.write(targetFile, text, Charset.defaultCharset()); + final String fileName = FileUtil.stripPostfix(FileUtil.stripPostfix(file.getName(), ".yml"), YAML_EXTENSION) + + YAML_EXTENSION; + FileUtil.copy(file, new File(templatesDir, fileName)); } } } @@ -280,19 +274,19 @@ private void createChartYaml(HelmConfig helmConfig, File outputDir) throws IOExc private static Chart chartFromHelmConfig(HelmConfig helmConfig) { return Chart.builder() - .apiVersion(helmConfig.getApiVersion()) - .name(helmConfig.getChart()) - .version(helmConfig.getVersion()) - .description(helmConfig.getDescription()) - .home(helmConfig.getHome()) - .sources(helmConfig.getSources()) - .maintainers(helmConfig.getMaintainers()) - .icon(helmConfig.getIcon()) - .appVersion(helmConfig.getAppVersion()) - .keywords(helmConfig.getKeywords()) - .engine(helmConfig.getEngine()) - .dependencies(helmConfig.getDependencies()) - .build(); + .apiVersion(helmConfig.getApiVersion()) + .name(helmConfig.getChart()) + .version(helmConfig.getVersion()) + .description(helmConfig.getDescription()) + .home(helmConfig.getHome()) + .sources(helmConfig.getSources()) + .maintainers(helmConfig.getMaintainers()) + .icon(helmConfig.getIcon()) + .appVersion(helmConfig.getAppVersion()) + .keywords(helmConfig.getKeywords()) + .engine(helmConfig.getEngine()) + .dependencies(helmConfig.getDependencies()) + .build(); } private T readFragment(Pattern filePattern, Class type) { @@ -300,9 +294,10 @@ private T readFragment(Pattern filePattern, Class type) { if (helmChartFragment != null) { try { return Serialization.unmarshal( - interpolate(helmChartFragment, jKubeConfiguration.getProperties(), DEFAULT_FILTER), type); + interpolate(helmChartFragment, jKubeConfiguration.getProperties(), DEFAULT_FILTER), type); } catch (Exception e) { - throw new IllegalArgumentException("Failure in parsing Helm fragment (" + helmChartFragment.getName() + "): " + e.getMessage(), e); + throw new IllegalArgumentException( + "Failure in parsing Helm fragment (" + helmChartFragment.getName() + "): " + e.getMessage(), e); } } return null; @@ -342,8 +337,8 @@ private static String interpolateTemplateWithHelmParameter(String template, Helm return answer; } - - private static void interpolateTemplateParameterExpressionsWithHelmExpressions(File file, List helmParameters) throws IOException { + private static void interpolateTemplateParameterExpressionsWithHelmExpressions(File file, List helmParameters) + throws IOException { final String originalTemplate = FileUtils.readFileToString(file, Charset.defaultCharset()); String interpolatedTemplate = originalTemplate; @@ -395,7 +390,7 @@ private static List collectParameters(HelmConfig helmConfig) { helmConfig.getParameterTemplates().stream() .map(Template::getParameters).flatMap(List::stream) .map(p -> HelmParameter.builder() - .name(p.getName()).required(Boolean.TRUE.equals(p.getRequired())).value(p.getValue()).build()) + .name(p.getName()).required(Boolean.TRUE.equals(p.getRequired())).value(p.getValue()).build()) .forEach(parameters::add); } if (helmConfig.getParameters() != null && !helmConfig.getParameters().isEmpty()) { diff --git a/jkube-kit/helm/src/test/java/org/eclipse/jkube/kit/resource/helm/HelmServiceIT.java b/jkube-kit/helm/src/test/java/org/eclipse/jkube/kit/resource/helm/HelmServiceIT.java index 580cd6473e..592aa30770 100644 --- a/jkube-kit/helm/src/test/java/org/eclipse/jkube/kit/resource/helm/HelmServiceIT.java +++ b/jkube-kit/helm/src/test/java/org/eclipse/jkube/kit/resource/helm/HelmServiceIT.java @@ -15,7 +15,6 @@ import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -32,7 +31,6 @@ import org.eclipse.jkube.kit.common.assertj.ArchiveAssertions; import io.fabric8.openshift.api.model.Template; -import org.apache.commons.io.FileUtils; import org.eclipse.jkube.kit.common.util.Serialization; import org.eclipse.jkube.kit.config.resource.ResourceServiceConfig; import org.junit.jupiter.api.BeforeEach; @@ -174,16 +172,10 @@ private void assertYamls() throws Exception { final Path expectations = new File(HelmServiceIT.class.getResource("/it/expected").toURI()).toPath(); final Path generatedYamls = helmOutputDir.toPath(); for (Path expected : Files.walk(expectations).filter(Files::isRegularFile).collect(Collectors.toList())) { - final Map expectedContent = Serialization.unmarshal(replacePlaceholders(expected), Map.class); - final Map actualContent = Serialization.unmarshal( - replacePlaceholders(generatedYamls.resolve(expectations.relativize(expected))), Map.class); + + final Map expectedContent = Serialization.unmarshal(expected, Map.class); + final Map actualContent = Serialization.unmarshal(generatedYamls.resolve(expectations.relativize(expected)), Map.class); assertThat(actualContent).isEqualTo(expectedContent); } } - - private static String replacePlaceholders(final Path yamlWithJsonPlaceholders) throws IOException { - return FileUtils.readFileToString(yamlWithJsonPlaceholders.toFile(), StandardCharsets.UTF_8) - .replace("\"{{\"{{\"}}", "").replace("{{\"}}\"}}", "") - .replace("{", "(").replace("}", ")"); - } } diff --git a/jkube-kit/helm/src/test/resources/it/expected/kubernetes/templates/kubernetes.yaml b/jkube-kit/helm/src/test/resources/it/expected/kubernetes/templates/kubernetes.yaml index e110adca10..a0a8823df0 100644 --- a/jkube-kit/helm/src/test/resources/it/expected/kubernetes/templates/kubernetes.yaml +++ b/jkube-kit/helm/src/test/resources/it/expected/kubernetes/templates/kubernetes.yaml @@ -27,11 +27,14 @@ items: - apiVersion: v1 kind: Service metadata: + {{ if .Values.addAnnotations }} annotations: jkube/it: This is a test helm-variable: {{ required "A valid .Values.GLOBAL_TEMPLATE_ENV_VAR entry required!" .Values.GLOBAL_TEMPLATE_ENV_VAR }} - escape-test: "{{"{{"}} {{"}}"}} should be escaped to prevent helm errors" + value-that-looks-like-helm-test: "{{ $The term 'helm' finds its roots in Old English }}" + value-that-is-a-helm-expression: {{ .Values.abandonShip }} helm-golang-expression: {{ .Values.unused_value | upper | quote }} + {{ end }} labels: app: test provider: jkube @@ -51,12 +54,15 @@ items: - apiVersion: apps/v1 kind: Deployment metadata: + {{ if .Values.addAnnotations }} annotations: jkube/it: This is a test helm-variable: {{ required "A valid .Values.GLOBAL_TEMPLATE_ENV_VAR entry required!" .Values.GLOBAL_TEMPLATE_ENV_VAR }} - escape-test: "{{"{{"}} {{"}}"}} should be escaped to prevent helm errors" + value-that-looks-like-helm-test: "{{ $The term 'helm' finds its roots in Old English }}" + value-that-is-a-helm-expression: {{ .Values.abandonShip }} annotation-from-config: {{ .Chart.Name | upper }} annotation-from-config/dotted: {{ .Chart.Name }} + {{ end }} labels: app: test provider: jkube @@ -72,10 +78,13 @@ items: group: org.eclipse.jkube.quickstarts.maven template: metadata: + {{ if .Values.addAnnotations }} annotations: jkube/it: This is a test helm-variable: {{ required "A valid .Values.GLOBAL_TEMPLATE_ENV_VAR entry required!" .Values.GLOBAL_TEMPLATE_ENV_VAR }} - escape-test: "{{"{{"}} {{"}}"}} should be escaped to prevent helm errors" + value-that-looks-like-helm-test: "{{ $The term 'helm' finds its roots in Old English }}" + value-that-is-a-helm-expression: {{ .Values.abandonShip }} + {{ end }} labels: app: test provider: jkube diff --git a/jkube-kit/helm/src/test/resources/it/expected/kubernetes/values.yaml b/jkube-kit/helm/src/test/resources/it/expected/kubernetes/values.yaml index 95318ff9f8..b979c5b831 100644 --- a/jkube-kit/helm/src/test/resources/it/expected/kubernetes/values.yaml +++ b/jkube-kit/helm/src/test/resources/it/expected/kubernetes/values.yaml @@ -13,9 +13,9 @@ # deployment: - replicas: 1 limits: memory: 512Mi + replicas: 1 GLOBAL_TEMPLATE_ENV_VAR: This is a sample unused_value: the-value requests_memory: 256Mi diff --git a/jkube-kit/helm/src/test/resources/it/expected/openshift/templates/openshift.yaml b/jkube-kit/helm/src/test/resources/it/expected/openshift/templates/openshift.yaml index ea9641ca3a..a471f1748a 100644 --- a/jkube-kit/helm/src/test/resources/it/expected/openshift/templates/openshift.yaml +++ b/jkube-kit/helm/src/test/resources/it/expected/openshift/templates/openshift.yaml @@ -13,6 +13,8 @@ # --- +{{- $shouldDeploy := .Values.shouldSetSail -}} +{{ if eq $shouldDeploy true }} apiVersion: v1 kind: List items: @@ -32,5 +34,11 @@ items: provider: jkube version: 1337 helm-variable: {{ required "A valid .Values.GLOBAL_TEMPLATE_ENV_VAR entry required!" .Values.GLOBAL_TEMPLATE_ENV_VAR }} - escape-test: "{{"{{"}} {{"}}"}} should be escaped to prevent helm errors" + value-that-looks-like-helm-test: "{{ $The term 'helm' finds its roots in Old English }}" + value-that-is-a-helm-expression: {{ .Values.abandonShip }} + toppings: |- + {{- range $index, $topping := .Values.pizzaToppings }} + {{ $index }}: {{ $topping }} + {{- end }} name: test +{{ end }} \ No newline at end of file diff --git a/jkube-kit/helm/src/test/resources/it/expected/openshift/values.yaml b/jkube-kit/helm/src/test/resources/it/expected/openshift/values.yaml index 81a42482c0..6dbac326ee 100644 --- a/jkube-kit/helm/src/test/resources/it/expected/openshift/values.yaml +++ b/jkube-kit/helm/src/test/resources/it/expected/openshift/values.yaml @@ -13,9 +13,9 @@ # deployment: - replicas: 1 limits: memory: "512Mi" + replicas: 1 GLOBAL_TEMPLATE_ENV_VAR: This is a sample unused_value: the-value requests_memory: 256Mi diff --git a/jkube-kit/helm/src/test/resources/it/sources/kubernetes/kubernetes.yml b/jkube-kit/helm/src/test/resources/it/sources/kubernetes/kubernetes.yml index 54d8b1a139..1e43391e9e 100644 --- a/jkube-kit/helm/src/test/resources/it/sources/kubernetes/kubernetes.yml +++ b/jkube-kit/helm/src/test/resources/it/sources/kubernetes/kubernetes.yml @@ -27,11 +27,14 @@ items: - apiVersion: v1 kind: Service metadata: + {{ if .Values.addAnnotations }} annotations: jkube/it: This is a test helm-variable: ${GLOBAL_TEMPLATE_ENV_VAR} - escape-test: "{{ }} should be escaped to prevent helm errors" + value-that-looks-like-helm-test: "{{ $The term 'helm' finds its roots in Old English }}" + value-that-is-a-helm-expression: {{ .Values.abandonShip }} helm-golang-expression: ${golang_expression} + {{ end }} labels: app: test provider: jkube @@ -51,12 +54,15 @@ items: - apiVersion: apps/v1 kind: Deployment metadata: + {{ if .Values.addAnnotations }} annotations: jkube/it: This is a test helm-variable: ${GLOBAL_TEMPLATE_ENV_VAR} - escape-test: "{{ }} should be escaped to prevent helm errors" + value-that-looks-like-helm-test: "{{ $The term 'helm' finds its roots in Old English }}" + value-that-is-a-helm-expression: {{ .Values.abandonShip }} annotation-from-config: ${annotation_from_config} annotation-from-config/dotted: ${annotation.from.config.dotted} + {{ end }} labels: app: test provider: jkube @@ -72,10 +78,13 @@ items: group: org.eclipse.jkube.quickstarts.maven template: metadata: + {{ if .Values.addAnnotations }} annotations: jkube/it: This is a test helm-variable: ${GLOBAL_TEMPLATE_ENV_VAR} - escape-test: "{{ }} should be escaped to prevent helm errors" + value-that-looks-like-helm-test: "{{ $The term 'helm' finds its roots in Old English }}" + value-that-is-a-helm-expression: {{ .Values.abandonShip }} + {{ end }} labels: app: test provider: jkube diff --git a/jkube-kit/helm/src/test/resources/it/sources/openshift/openshift.yml b/jkube-kit/helm/src/test/resources/it/sources/openshift/openshift.yml index cd9f8955e8..bd0be30ed8 100644 --- a/jkube-kit/helm/src/test/resources/it/sources/openshift/openshift.yml +++ b/jkube-kit/helm/src/test/resources/it/sources/openshift/openshift.yml @@ -13,6 +13,8 @@ # --- +{{- $shouldDeploy := .Values.shouldSetSail -}} +{{ if eq $shouldDeploy true }} apiVersion: v1 kind: List items: @@ -32,5 +34,11 @@ items: provider: jkube version: 1337 helm-variable: ${GLOBAL_TEMPLATE_ENV_VAR} - escape-test: "{{ }} should be escaped to prevent helm errors" + value-that-looks-like-helm-test: "{{ $The term 'helm' finds its roots in Old English }}" + value-that-is-a-helm-expression: {{ .Values.abandonShip }} + toppings: |- + {{- range $index, $topping := .Values.pizzaToppings }} + {{ $index }}: {{ $topping }} + {{- end }} name: test +{{ end }} \ No newline at end of file