From b2c66af404de473e89a6e23ebba159dfc09976e0 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 14 Jun 2023 22:08:29 +0200 Subject: [PATCH] Add support for binary cache for Spack builds (#249) Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- .../wave/configuration/SpackConfig.groovy | 93 ++++++++ .../ContainerTokenController.groovy | 5 +- .../wave/service/builder/BuildRequest.groovy | 3 + .../builder/ContainerBuildServiceImpl.groovy | 24 ++- .../builder/DockerBuildStrategy.groovy | 24 ++- .../service/builder/KubeBuildStrategy.groovy | 7 +- .../seqera/wave/service/k8s/K8sService.groovy | 3 +- .../wave/service/k8s/K8sServiceImpl.groovy | 37 +++- .../io/seqera/wave/util/SpackHelper.groovy | 33 +++ src/main/resources/application-dev.yml | 3 + .../spack/spack-builder-containerfile.txt | 40 ++++ .../ContainerTokenControllerTest.groovy | 7 +- .../builder/ContainerBuildServiceTest.groovy | 60 +++++- .../builder/DockerBuilderStrategyTest.groovy | 28 ++- .../builder/KubeBuildStrategyTest.groovy | 4 +- .../wave/service/k8s/K8sClientTest.groovy | 6 +- .../service/k8s/K8sServiceImplTest.groovy | 89 +++++++- .../seqera/wave/util/SpackHelperTest.groovy | 38 ++++ src/test/resources/application-test.yml | 3 + .../io/seqera/wave/util/TemplateRenderer.java | 150 +++++++++++++ .../wave/util/TemplateRendererTest.groovy | 202 ++++++++++++++++++ 22 files changed, 829 insertions(+), 32 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/configuration/SpackConfig.groovy create mode 100644 src/main/groovy/io/seqera/wave/util/SpackHelper.groovy create mode 100644 src/main/resources/io/seqera/wave/spack/spack-builder-containerfile.txt create mode 100644 src/test/groovy/io/seqera/wave/util/SpackHelperTest.groovy create mode 100644 wave-utils/src/main/java/io/seqera/wave/util/TemplateRenderer.java create mode 100644 wave-utils/src/test/groovy/io/seqera/wave/util/TemplateRendererTest.groovy diff --git a/VERSION b/VERSION index dddc2e9ca..0dd517953 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.33.4 +0.34.0-RC6 diff --git a/src/main/groovy/io/seqera/wave/configuration/SpackConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/SpackConfig.groovy new file mode 100644 index 000000000..bcc11ded7 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/configuration/SpackConfig.groovy @@ -0,0 +1,93 @@ +package io.seqera.wave.configuration + +import java.nio.file.Path + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import io.micronaut.context.annotation.Value +import io.micronaut.core.annotation.Nullable +import jakarta.inject.Singleton +/** + * Model Spack configuration + * + * @author Paolo Di Tommaso + */ +@ToString +@EqualsAndHashCode +@Singleton +@CompileStatic +class SpackConfig { + + /** + * The host path where where Spack cached binaries are stored + */ + @Nullable + @Value('${wave.build.spack.cacheDirectory}') + private String cacheDirectory + + /** + * The container mount path where Spack cached binaries are stored + */ + @Nullable + @Value('${wave.build.spack.cacheMountPath}') + private String cacheMountPath + + /** + * The host path where the GPG key required by the Spack "buildcache" is located + */ + @Nullable + @Value('${wave.build.spack.secretKeyFile}') + private String secretKeyFile + + /** + * The container path where the GPG key required by the Spack "buildcache" is located + */ + @Nullable + @Value('${wave.build.spack.secretMountPath}') + private String secretMountPath + + /** + * The container image used for Spack builds + */ + @Value('${wave.build.spack.builderImage:`spack/ubuntu-jammy:v0.20.0`}') + private String builderImage + + /** + * The container image used for Spack container + */ + @Value('${wave.build.spack.runnerImage:`ubuntu:22.04`}') + private String runnerImage + + Path getCacheDirectory() { + if( !cacheDirectory ) + throw new IllegalStateException("Missing Spack cacheDirectory configuration setting") + return Path.of(cacheDirectory).toAbsolutePath().normalize() + } + + String getCacheMountPath() { + if( !cacheMountPath ) + throw new IllegalStateException("Missing Spack cacheMountPath configuration setting") + return cacheMountPath + } + + Path getSecretKeyFile() { + if( !secretKeyFile ) + throw new IllegalStateException("Missing Spack secretKeyFile configuration setting") + return Path.of(secretKeyFile).toAbsolutePath().normalize() + } + + String getSecretMountPath() { + if( !secretMountPath ) + throw new IllegalStateException("Missing Spack secretMountPath configuration setting") + return secretMountPath + } + + String getBuilderImage() { + return builderImage + } + + String getRunnerImage() { + return runnerImage + } +} diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerTokenController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerTokenController.groovy index 87173f1c4..3202499da 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerTokenController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerTokenController.groovy @@ -37,9 +37,8 @@ import io.seqera.wave.tower.User import io.seqera.wave.tower.auth.JwtAuthStore import io.seqera.wave.util.DataTimeUtils import jakarta.inject.Inject - import static io.seqera.wave.WaveDefault.TOWER - +import static io.seqera.wave.util.SpackHelper.prependBuilderTemplate /** * Implement a controller to receive container token requests * @@ -177,7 +176,7 @@ class ContainerTokenController { final offset = DataTimeUtils.offsetId(req.timestamp) // create a unique digest to identify the request return new BuildRequest( - dockerContent, + (spackContent ? prependBuilderTemplate(dockerContent) : dockerContent), Path.of(workspace), build, condaContent, diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 1badce57f..1299dac11 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -91,6 +91,8 @@ class BuildRequest { */ final String offsetId + final boolean isSpackBuild + BuildRequest(String containerFile, Path workspace, String repo, String condaFile, String spackFile, User user, ContainerPlatform platform, String configJson, String cacheRepo, String ip, String offsetId = null) { this.id = computeDigest(containerFile, condaFile, spackFile, platform, repo) this.dockerFile = containerFile @@ -106,6 +108,7 @@ class BuildRequest { this.startTime = Instant.now() this.job = "${id}-${startTime.toEpochMilli().toString().md5()[-5..-1]}" this.ip = ip + this.isSpackBuild = spackFile } static private String computeDigest(String containerFile, String condaFile, String spackFile, ContainerPlatform platform, String repository) { diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy index 75ab5e3bf..306e3f4f9 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy @@ -14,8 +14,11 @@ import io.micronaut.context.annotation.Value import io.micronaut.context.event.ApplicationEventPublisher import io.seqera.wave.auth.RegistryCredentialsProvider import io.seqera.wave.auth.RegistryLookupService +import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.ratelimit.AcquireRequest import io.seqera.wave.ratelimit.RateLimiterService +import io.seqera.wave.util.SpackHelper +import io.seqera.wave.util.TemplateRenderer import io.seqera.wave.util.ThreadPoolBuilder import jakarta.inject.Inject import jakarta.inject.Singleton @@ -77,6 +80,9 @@ class ContainerBuildServiceImpl implements ContainerBuildService { @Inject private MeterRegistry meterRegistry + @Inject + private SpackConfig spackConfig + @PostConstruct void init() { executor = ThreadPoolBuilder.io(10, 10, 100, 'wave-builder') @@ -111,6 +117,20 @@ class ContainerBuildServiceImpl implements ContainerBuildService { .awaitBuild(targetImage) } + protected String dockerFile0(BuildRequest req, SpackConfig config) { + if( req.isSpackBuild ) { + final binding = new HashMap(2) + binding.spack_builder_image = config.builderImage + binding.spack_runner_image = config.runnerImage + binding.spack_arch = SpackHelper.toSpackArch(req.getPlatform()) + binding.spack_cache_dir = config.cacheMountPath + binding.spack_key_file = config.secretMountPath + return new TemplateRenderer().render(req.dockerFile, binding) + } + else + return req.dockerFile + } + protected BuildResult launch(BuildRequest req) { // launch an external process to build the container BuildResult resp=null @@ -118,8 +138,8 @@ class ContainerBuildServiceImpl implements ContainerBuildService { // create the workdir path Files.createDirectories(req.workDir) // save the dockerfile - final dockerfile = req.workDir.resolve('Dockerfile') - Files.write(dockerfile, req.dockerFile.bytes, CREATE, WRITE, TRUNCATE_EXISTING) + final dockerFile = req.workDir.resolve('Dockerfile') + Files.write(dockerFile, dockerFile0(req,spackConfig).bytes, CREATE, WRITE, TRUNCATE_EXISTING) // save the conda file if( req.condaFile ) { final condaFile = req.workDir.resolve('conda.yml') diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 2691506ec..7d52cd72a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -9,7 +9,9 @@ import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value +import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform +import jakarta.inject.Inject import jakarta.inject.Singleton import static java.nio.file.StandardOpenOption.CREATE @@ -35,6 +37,9 @@ class DockerBuildStrategy extends BuildStrategy { @Value('${wave.debug}') Boolean debug + @Inject + SpackConfig spackConfig + @Override BuildResult build(BuildRequest req) { @@ -44,7 +49,7 @@ class DockerBuildStrategy extends BuildStrategy { Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) } - // comand the docker build command + // command the docker build command final buildCmd= buildCmd(req, configFile) log.debug "Build run command: ${buildCmd.join(' ')}" // save docker cli for debugging purpose @@ -66,11 +71,15 @@ class DockerBuildStrategy extends BuildStrategy { } protected List buildCmd(BuildRequest req, Path credsFile) { - final dockerCmd = dockerWrapper(req.workDir, credsFile, req.platform) + final dockerCmd = dockerWrapper( + req.workDir, + credsFile, + req.isSpackBuild ? spackConfig : null, + req.platform) return dockerCmd + launchCmd(req) } - protected List dockerWrapper(Path workDir, Path credsFile, ContainerPlatform platform) { + protected List dockerWrapper(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { final wrapper = ['docker', 'run', '--rm', @@ -82,6 +91,15 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add("$credsFile:/kaniko/.docker/config.json:ro".toString()) } + if( spackConfig ) { + // secret file + wrapper.add('-v') + wrapper.add("${spackConfig.secretKeyFile}:${spackConfig.secretMountPath}:ro".toString()) + // cache directory + wrapper.add('-v') + wrapper.add("${spackConfig.cacheDirectory}:${spackConfig.cacheMountPath}:rw".toString()) + } + if( platform ) { wrapper.add('--platform') wrapper.add(platform.toString()) diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index e8009c061..79bf4c275 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -12,6 +12,7 @@ import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Property import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value +import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.k8s.K8sService @@ -46,6 +47,9 @@ class KubeBuildStrategy extends BuildStrategy { @Nullable private Map nodeSelectorMap + @Inject + private SpackConfig spackConfig + private String podName(BuildRequest req) { return "build-${req.job}" } @@ -62,7 +66,8 @@ class KubeBuildStrategy extends BuildStrategy { final buildCmd = launchCmd(req) final name = podName(req) final selector= getSelectorLabel(req.platform, nodeSelectorMap) - final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, selector) + final spackCfg0 = req.isSpackBuild ? spackConfig : null + final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, spackCfg0, selector) final terminated = k8sService.waitPod(pod, buildTimeout.toMillis()) final stdout = k8sService.logsPod(name) if( terminated ) { diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index fb38fca9b..dd684402b 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -5,6 +5,7 @@ import java.nio.file.Path import io.kubernetes.client.openapi.models.V1ContainerStateTerminated import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod +import io.seqera.wave.configuration.SpackConfig /** * Defines Kubernetes operations @@ -27,7 +28,7 @@ interface K8sService { void deletePod(String name) - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Map nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, SpackConfig spackConfig, Map nodeSelector) V1ContainerStateTerminated waitPod(V1Pod pod, long timeout) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 4bee8f6f4..9e9d3108f 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -23,6 +23,7 @@ import io.kubernetes.client.openapi.models.V1VolumeMount import io.micronaut.context.annotation.Property import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value +import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import jakarta.inject.Inject import jakarta.inject.Singleton @@ -78,6 +79,9 @@ class K8sServiceImpl implements K8sService { @Nullable private String requestsMemory + @Inject + private SpackConfig spackConfig + @Inject private K8sClient k8sClient @@ -251,6 +255,27 @@ class K8sServiceImpl implements K8sService { .readOnly(true) } + protected V1VolumeMount mountSpackCacheDir(Path spackCacheDir, @Nullable String storageMountPath, String containerPath) { + final rel = Path.of(storageMountPath).relativize(spackCacheDir).toString() + if( !rel || rel.startsWith('../') ) + throw new IllegalArgumentException("Spack cacheDirectory '$spackCacheDir' must be a sub-directory of storage path '$storageMountPath'") + return new V1VolumeMount() + .name('build-data') + .mountPath(containerPath) + .subPath(rel) + } + + protected V1VolumeMount mountSpackSecretFile(Path secretFile, @Nullable String storageMountPath, String containerPath) { + final rel = Path.of(storageMountPath).relativize(secretFile).toString() + if( !rel || rel.startsWith('../') ) + throw new IllegalArgumentException("Spack secretKeyFile '$secretFile' must be a sub-directory of storage path '$storageMountPath'") + return new V1VolumeMount() + .name('build-data') + .readOnly(true) + .mountPath(containerPath) + .subPath(rel) + } + /** * Create a container for container image building via Kaniko * @@ -269,15 +294,14 @@ class K8sServiceImpl implements K8sService { */ @Override @CompileDynamic - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Map nodeSelector) { - final spec = buildSpec(name, containerImage, args, workDir, creds, nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, SpackConfig spackConfig, Map nodeSelector) { + final spec = buildSpec(name, containerImage, args, workDir, creds, spackConfig, nodeSelector) return k8sClient .coreV1Api() .createNamespacedPod(namespace, spec, null, null, null) } - @CompileDynamic - V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path creds, Map nodeSelector) { + V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path creds, SpackConfig spackConfig, Map nodeSelector) { // required volumes final mounts = new ArrayList(5) @@ -290,6 +314,11 @@ class K8sServiceImpl implements K8sService { mounts.add(0, mountDockerConfig(workDir, storageMountPath)) } + if( spackConfig ) { + mounts.add(mountSpackCacheDir(spackConfig.cacheDirectory, storageMountPath, spackConfig.cacheMountPath)) + mounts.add(mountSpackSecretFile(spackConfig.secretKeyFile, storageMountPath, spackConfig.secretMountPath)) + } + V1PodBuilder builder = new V1PodBuilder() //metadata section diff --git a/src/main/groovy/io/seqera/wave/util/SpackHelper.groovy b/src/main/groovy/io/seqera/wave/util/SpackHelper.groovy new file mode 100644 index 000000000..0e0150094 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/util/SpackHelper.groovy @@ -0,0 +1,33 @@ +package io.seqera.wave.util + +import io.seqera.wave.core.ContainerPlatform + +/** + * Helper class for Spack package manager + * + * @author Paolo Di Tommaso + */ +class SpackHelper { + + static String builderTemplate() { + SpackHelper.class + .getResourceAsStream('/io/seqera/wave/spack/spack-builder-containerfile.txt') + .getText() + } + + static String prependBuilderTemplate(String dockerContent) { + return builderTemplate() + dockerContent + } + + static String toSpackArch(ContainerPlatform platform) { + if( !platform ) + throw new IllegalArgumentException("Missing container platform argument") + final value = platform.toString() + if( value=='linux/amd64' ) + return 'x86_64' + if( value=='linux/arm64' ) + return 'aarch64' + throw new IllegalArgumentException("Unable to map container platform '${platform}' to Spack architecture") + } + +} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 21f333e97..6cc176e9a 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -1,5 +1,8 @@ wave: debug: true + build: + spack: + cacheDirectory: 'spack-cache' --- # uses TOWER_xxx variable because the dev environment is expected to be # the same as of tower diff --git a/src/main/resources/io/seqera/wave/spack/spack-builder-containerfile.txt b/src/main/resources/io/seqera/wave/spack/spack-builder-containerfile.txt new file mode 100644 index 000000000..6f60aee61 --- /dev/null +++ b/src/main/resources/io/seqera/wave/spack/spack-builder-containerfile.txt @@ -0,0 +1,40 @@ +# Builder image +FROM {{spack_builder_image}} as builder +COPY spack.yaml /opt/spack-env/spack.yaml + +RUN mkdir -p /opt/spack-env \ +&& sed -i -e 's;compilers:;compilers::;' \ + -e 's;^ *flags: *{}; flags:\n cflags: -O3\n cxxflags: -O3\n fflags: -O3;' \ + /root/.spack/linux/compilers.yaml \ +&& cd /opt/spack-env && spack env activate . \ +&& spack config add config:install_tree:/opt/software \ +&& spack config add concretizer:unify:true \ +&& spack config add concretizer:reuse:false \ +&& spack config add packages:all:target:[{{spack_arch}}] \ +&& echo -e "\ + view: /opt/view \n\ +" >> /opt/spack-env/spack.yaml + +# Install packages, clean afterwards, finally strip binaries +RUN cd /opt/spack-env \ +&& spack gpg trust {{spack_key_file}} \ +&& spack mirror add seqera-spack {{spack_cache_dir}} \ +&& spack mirror add binary_mirror https://binaries.spack.io/releases/v0.20 \ +&& spack buildcache keys --install --trust \ +&& spack env activate . \ +&& spack concretize -f \ +&& spack install --fail-fast \ +&& spack gc -y \ +&& find -L /opt/._view/* -type f -exec readlink -f '{}' \; | \ + xargs file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs strip -s + +RUN cd /opt/spack-env && \ + spack env activate --sh -d . >> /opt/spack-env/z10_spack_environment.sh && \ + original_view=$( cd /opt ; ls -1d ._view/* ) && \ + sed -i "s;/view/;/$original_view/;" /opt/spack-env/z10_spack_environment.sh && \ + echo "# Needed for Perl applications" >>/opt/spack-env/z10_spack_environment.sh && \ + echo "export PERL5LIB=$(eval ls -d /opt/._view/*/lib/5.*):$PERL5LIB" >>/opt/spack-env/z10_spack_environment.sh && \ + rm -rf /opt/view diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerTokenControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerTokenControllerTest.groovy index 0de38c6d3..01bf6946a 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerTokenControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerTokenControllerTest.groovy @@ -186,11 +186,12 @@ class ContainerTokenControllerTest extends Specification { submit = new SubmitContainerTokenRequest(containerFile: encode('FROM foo'), spackFile: encode('some::spack-recipe'), containerPlatform: 'arm64') build = controller.makeBuildRequest(submit, null, "") then: - build.id == '8a24dd0ea739ad970f2653ebc18618db' - build.dockerFile == 'FROM foo' + build.id == '23ef4010a60670510393f5ae7414eb84' + build.dockerFile.endsWith('\nFROM foo') + build.dockerFile.startsWith('# Builder image\n') build.condaFile == null build.spackFile == 'some::spack-recipe' - build.targetImage == 'wave/build:8a24dd0ea739ad970f2653ebc18618db' + build.targetImage == 'wave/build:23ef4010a60670510393f5ae7414eb84' build.workDir == Path.of('/some/wsp').resolve(build.id) build.platform == ContainerPlatform.of('arm64') } diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index 2b3788f11..95fa9450e 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -12,8 +12,11 @@ import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.auth.DockerAuthService import io.seqera.wave.auth.RegistryCredentialsProvider import io.seqera.wave.auth.RegistryLookupService +import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.User +import io.seqera.wave.util.SpackHelper +import io.seqera.wave.util.TemplateRenderer import jakarta.inject.Inject /** * @@ -153,6 +156,7 @@ class ContainerBuildServiceTest extends Specification { def dockerFile = ''' FROM busybox RUN echo Hello > hello.txt + RUN {{spack_cache_dir}} {{spack_key_file}} '''.stripIndent() and: def condaFile = ''' @@ -167,11 +171,12 @@ class ContainerBuildServiceTest extends Specification { build file ''' and: + def spackConfig = new SpackConfig(cacheMountPath: '/mnt/cache', secretMountPath: '/mnt/secret') def REQ = new BuildRequest(dockerFile, folder, 'box:latest', condaFile, spackFile, Mock(User), ContainerPlatform.of('amd64'), cfg, null, "") and: def store = Mock(BuildStore) def strategy = Mock(BuildStrategy) - def builder = new ContainerBuildServiceImpl(buildStrategy: strategy, buildStore: store, statusDuration: DURATION) + def builder = new ContainerBuildServiceImpl(buildStrategy: strategy, buildStore: store, statusDuration: DURATION, spackConfig:spackConfig) def RESPONSE = Mock(BuildResult) when: @@ -180,7 +185,7 @@ class ContainerBuildServiceTest extends Specification { 1 * strategy.build(REQ) >> RESPONSE 1 * store.storeBuild(REQ.targetImage, RESPONSE, DURATION) >> null and: - REQ.workDir.resolve('Dockerfile').text == dockerFile + REQ.workDir.resolve('Dockerfile').text == new TemplateRenderer().render(dockerFile, [spack_cache_dir:'/mnt/cache', spack_key_file:'/mnt/secret']) REQ.workDir.resolve('conda.yml').text == condaFile REQ.workDir.resolve('spack.yaml').text == spackFile and: @@ -189,4 +194,55 @@ class ContainerBuildServiceTest extends Specification { cleanup: folder?.deleteDir() } + + def 'should resolve docker file' () { + given: + def folder = Files.createTempDirectory('test') + def builder = new ContainerBuildServiceImpl() + and: + def dockerFile = 'FROM something; {{foo}}' + def REQ = new BuildRequest(dockerFile, folder, 'box:latest', null, null, Mock(User), ContainerPlatform.of('amd64'), null, null, "") + and: + def spack = Mock(SpackConfig) + + when: + def result = builder.dockerFile0(REQ, spack) + then: + 0* spack.getCacheMountPath() >> null + 0* spack.getSecretMountPath() >> null + 0* spack.getBuilderImage() >> null + and: + result == 'FROM something; {{foo}}' + + cleanup: + folder?.deleteDir() + } + + def 'should resolve docker file with spack config' () { + given: + def folder = Files.createTempDirectory('test') + def builder = new ContainerBuildServiceImpl() + and: + def dockerFile = SpackHelper.builderTemplate() + def spackFile = 'some spack packages' + def REQ = new BuildRequest(dockerFile, folder, 'box:latest', null, spackFile, Mock(User), ContainerPlatform.of('amd64'), null, null, "") + and: + def spack = Mock(SpackConfig) + + when: + def result = builder.dockerFile0(REQ, spack) + then: + 1* spack.getCacheMountPath() >> '/mnt/cache' + 1* spack.getSecretMountPath() >> '/mnt/key' + 1* spack.getBuilderImage() >> 'spack-builder:2.0' + 1* spack.getRunnerImage() >> 'ubuntu:22.04' + and: + result.contains('FROM spack-builder:2.0 as builder') + result.contains('spack config add packages:all:target:[x86_64]') + result.contains('spack mirror add seqera-spack /mnt/cache') + result.contains('spack gpg trust /mnt/key') + + cleanup: + folder?.deleteDir() + } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuilderStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuilderStrategyTest.groovy index 7b0d9d2b7..5dbaa7c5c 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuilderStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuilderStrategyTest.groovy @@ -6,6 +6,7 @@ import java.nio.file.Path import io.micronaut.context.ApplicationContext import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.User /** @@ -17,12 +18,19 @@ class DockerBuilderStrategyTest extends Specification { def 'should get docker command' () { given: - def ctx = ApplicationContext.run() + def props = [ + 'wave.build.spack.cacheDirectory':'/host/spack/cache', + 'wave.build.spack.cacheMountPath':'/opt/spack/cache', + 'wave.build.spack.secretKeyFile':'/host/spack/key', + 'wave.build.spack.secretMountPath':'/opt/spack/key' ] + def ctx = ApplicationContext.run(props) + and: def service = ctx.getBean(DockerBuildStrategy) + def spackConfig = ctx.getBean(SpackConfig) and: def work = Path.of('/work/foo') when: - def cmd = service.dockerWrapper(work, null, null) + def cmd = service.dockerWrapper(work, null, null, null) then: cmd == ['docker', 'run', @@ -32,7 +40,7 @@ class DockerBuilderStrategyTest extends Specification { 'gcr.io/kaniko-project/executor:v1.9.1'] when: - cmd = service.dockerWrapper(work, Path.of('/foo/creds.json'), ContainerPlatform.of('arm64')) + cmd = service.dockerWrapper(work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64')) then: cmd == ['docker', 'run', @@ -43,6 +51,20 @@ class DockerBuilderStrategyTest extends Specification { '--platform', 'linux/arm64', 'gcr.io/kaniko-project/executor:v1.9.1'] + when: + cmd = service.dockerWrapper(work, Path.of('/foo/creds.json'), spackConfig, null) + then: + cmd == ['docker', + 'run', + '--rm', + '-w', '/work/foo', + '-v', '/work/foo:/work/foo', + '-v', '/foo/creds.json:/kaniko/.docker/config.json:ro', + '-v', '/host/spack/key:/opt/spack/key:ro', + '-v', '/host/spack/cache:/opt/spack/cache:rw', + 'gcr.io/kaniko-project/executor:v1.9.1'] + + cleanup: ctx.close() } diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index 7c5752d5a..304746a2c 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -79,7 +79,7 @@ class KubeBuildStrategyTest extends Specification { then: resp and: - 1 * k8sService.buildContainer(_, _, _, _, _, [service:'wave-build']) >> null + 1 * k8sService.buildContainer(_, _, _, _, _, _, [service:'wave-build']) >> null when: def req2 = new BuildRequest('from foo', PATH, repo, null, null, USER, ContainerPlatform.of('arm64'),'{}', cache, "") @@ -89,7 +89,7 @@ class KubeBuildStrategyTest extends Specification { then: resp2 and: - 1 * k8sService.buildContainer(_, _, _, _, _, [service:'wave-build-arm64']) >> null + 1 * k8sService.buildContainer(_, _, _, _, _, _, [service:'wave-build-arm64']) >> null } } diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy index 79123264a..b69546905 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy @@ -39,8 +39,10 @@ class K8sClientTest extends Specification { 'my-pod', 'busybox', ['cat','/kaniko/.docker/config.json'], - Path.of('/foo'), - 'my-creds') + Path.of('/work/dir'), + Path.of('/creds'), + Path.of('/spack/dir'), + ['my-creds': 'selector']) then: true diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index aa825fff0..af09048a1 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -7,6 +7,8 @@ import java.nio.file.Path import io.kubernetes.client.custom.Quantity import io.micronaut.context.ApplicationContext import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.configuration.SpackConfig + /** * * @author Paolo Di Tommaso @@ -143,6 +145,29 @@ class K8sServiceImplTest extends Specification { ctx.close() } + def 'should get spack dir vol' () { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.k8s.namespace': 'foo', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'bar', + 'wave.build.k8s.storage.mountPath': '/build' ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + + when: + def mount = k8sService.mountSpackCacheDir(Path.of('/foo/work/x1'), '/foo', '/opt/spack/cache') + then: + mount.name == 'build-data' + mount.mountPath == '/opt/spack/cache' + mount.subPath == 'work/x1' + !mount.readOnly + + cleanup: + ctx.close() + } def 'should create build pod' () { given: @@ -158,7 +183,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), Path.of('secret'),[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), Path.of('secret'), null, [:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -188,6 +213,60 @@ class K8sServiceImplTest extends Specification { ctx.close() } + def 'should create build pod with spack cache' () { + given: + def PROPS = [ + 'wave.build.workspace': '/build/work', + 'wave.build.timeout': '10s', + 'wave.build.k8s.namespace': 'my-ns', + 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.k8s.storage.claimName': 'build-claim', + 'wave.build.k8s.storage.mountPath': '/build', + 'wave.build.spack.cacheDirectory':'/build/host/spack/cache', + 'wave.build.spack.cacheMountPath':'/opt/container/spack/cache', + 'wave.build.spack.secretKeyFile':'/build/host/spack/key', + 'wave.build.spack.secretMountPath':'/opt/container/spack/key' + ] + and: + def ctx = ApplicationContext.run(PROPS) + def k8sService = ctx.getBean(K8sServiceImpl) + def spackConfig = ctx.getBean(SpackConfig) + when: + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, spackConfig, [:]) + then: + result.metadata.name == 'foo' + result.metadata.namespace == 'my-ns' + and: + result.spec.activeDeadlineSeconds == 10 + and: + result.spec.containers.get(0).name == 'foo' + result.spec.containers.get(0).image == 'my-image:latest' + result.spec.containers.get(0).args == ['this','that'] + and: + result.spec.containers.get(0).volumeMounts.size() == 3 + and: + result.spec.containers.get(0).volumeMounts.get(0).name == 'build-data' + result.spec.containers.get(0).volumeMounts.get(0).mountPath == '/build/work/xyz' + result.spec.containers.get(0).volumeMounts.get(0).subPath == 'work/xyz' + and: + result.spec.containers.get(0).volumeMounts.get(1).name == 'build-data' + result.spec.containers.get(0).volumeMounts.get(1).mountPath == '/opt/container/spack/cache' + result.spec.containers.get(0).volumeMounts.get(1).subPath == 'host/spack/cache' + !result.spec.containers.get(0).volumeMounts.get(1).readOnly + and: + result.spec.containers.get(0).volumeMounts.get(2).name == 'build-data' + result.spec.containers.get(0).volumeMounts.get(2).mountPath == '/opt/container/spack/key' + result.spec.containers.get(0).volumeMounts.get(2).subPath == 'host/spack/key' + result.spec.containers.get(0).volumeMounts.get(2).readOnly + + and: + result.spec.volumes.get(0).name == 'build-data' + result.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' + + cleanup: + ctx.close() + } + def 'should create build pod without init container' () { given: def PROPS = [ @@ -202,7 +281,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, null,[:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' @@ -245,7 +324,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, null,[:]) then: result.metadata.name == 'foo' result.metadata.labels.toString() == PROPS['wave.build.k8s.labels'].toString() @@ -273,7 +352,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, PROPS['wave.build.k8s.node-selector'] as Map) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, null, PROPS['wave.build.k8s.node-selector'] as Map) then: result.spec.nodeSelector.toString() == PROPS['wave.build.k8s.node-selector'].toString() and: @@ -298,7 +377,7 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null,[:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), null, null,[:]) then: result.spec.serviceAccount == PROPS['wave.build.k8s.service-account'] and: diff --git a/src/test/groovy/io/seqera/wave/util/SpackHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/SpackHelperTest.groovy new file mode 100644 index 000000000..5dca08e49 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/util/SpackHelperTest.groovy @@ -0,0 +1,38 @@ +package io.seqera.wave.util + +import spock.lang.Specification + +import io.seqera.wave.core.ContainerPlatform + +/** + * + * @author Paolo Di Tommaso + */ +class SpackHelperTest extends Specification { + + def 'should load builder template' () { + expect: + SpackHelper.builderTemplate().startsWith('# Builder image') + } + + def 'should prepend builder template' () { + expect: + SpackHelper.prependBuilderTemplate('foo').startsWith('# Builder image') + SpackHelper.prependBuilderTemplate('foo').endsWith('\nfoo') + } + + def 'should map platform to spack arch' () { + expect: + SpackHelper.toSpackArch(ContainerPlatform.of('x86_64')) == 'x86_64' + SpackHelper.toSpackArch(ContainerPlatform.of('linux/x86_64')) == 'x86_64' + SpackHelper.toSpackArch(ContainerPlatform.of('amd64')) == 'x86_64' + SpackHelper.toSpackArch(ContainerPlatform.of('aarch64')) == 'aarch64' + SpackHelper.toSpackArch(ContainerPlatform.of('arm64')) == 'aarch64' + SpackHelper.toSpackArch(ContainerPlatform.of('linux/arm64/v8')) == 'aarch64' + + when: + SpackHelper.toSpackArch(ContainerPlatform.of('linux/arm64/v7')) + then: + thrown(IllegalArgumentException) + } +} diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml index 5451be301..a3f490808 100644 --- a/src/test/resources/application-test.yml +++ b/src/test/resources/application-test.yml @@ -37,6 +37,9 @@ wave: password: ${AZURECR_PAT:test} europe-southwest1-docker.pkg.dev: credentials : ${GOOGLECR_KEYS:test} + build: + spack: + cacheDirectory: "spack-cache" --- logger: levels: diff --git a/wave-utils/src/main/java/io/seqera/wave/util/TemplateRenderer.java b/wave-utils/src/main/java/io/seqera/wave/util/TemplateRenderer.java new file mode 100644 index 000000000..af9d4b8f9 --- /dev/null +++ b/wave-utils/src/main/java/io/seqera/wave/util/TemplateRenderer.java @@ -0,0 +1,150 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.util; + +import java.io.InputStream; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Template rendering helper + * + * @author Paolo Di Tommaso + */ +public class TemplateRenderer { + + private static final Pattern PATTERN = Pattern.compile("\\{\\{([^}]+)}}"); + + private static final Pattern VAR1 = Pattern.compile("(\\s*)\\{\\{([\\d\\w_-]+)}}(\\s*$)"); + private static final Pattern VAR2 = Pattern.compile("(? ignoreNames = List.of(); + + public TemplateRenderer withIgnore(String... names) { + return withIgnore(List.of(names)); + } + + public TemplateRenderer withIgnore(List names) { + if( names!=null ) { + ignoreNames = List.copyOf(names); + } + return this; + } + + public String render(InputStream template, Map binding) { + String str = new Scanner(template).useDelimiter("\\A").next(); + return render(str, binding); + } + + public String render(String template, Map binding) { + final String[] lines = template.split("(?<=\n)"); + final StringBuilder result = new StringBuilder(); + for( String it : lines ) { + if( it==null || it.startsWith("##")) + continue; + final String resolved = replace0(it, binding); + if( resolved!=null ) + result.append(resolved); + } + return result.toString(); + } + + /** + * Simple template helper class replacing all variable enclosed by {{..}} + * with the corresponding variable specified in a map object + * + * @param template The template string + * @param binding The binding {@link Map} + * @return The templated having the variables replaced with the corresponding value + */ + String replace1(CharSequence template, Map binding) { + Matcher matcher = PATTERN.matcher(template); + + // Loop through each matched variable placeholder + StringBuilder builder = new StringBuilder(); + boolean isNull=false; + while (matcher.find()) { + String variable = matcher.group(1); + + // Check if the variable exists in the values map + if (binding.containsKey(variable)) { + Object value = binding.get(variable); + String str = value!=null ? value.toString() : ""; + isNull |= value==null; + matcher.appendReplacement(builder, str); + } + else if( !ignoreNames.contains(variable) ) { + throw new IllegalArgumentException(String.format("Unable to resolve template variable: {{%s}}", variable)); + } + } + matcher.appendTail(builder); + + final String result = builder.toString(); + return !isNull || !result.isBlank() ? result : null; + } + + String replace0(String line, Map binding) { + if( line==null || line.length()==0 ) + return line; + + Matcher matcher = VAR1.matcher(line); + if( matcher.matches() ) { + final String name = matcher.group(2); + if( ignoreNames.contains(name) ) + return line; + if( !binding.containsKey(name) ) + throw new IllegalArgumentException("Missing template key: "+name); + final String prefix = matcher.group(1); + final String value = binding.get(name); + if( value==null ) + return null; // <-- return null to skip this line + + final StringBuilder result = new StringBuilder(); + final String[] multi = value.split("(?<=\n)"); + for (String s : multi) { + result.append(prefix); + result.append(s); + } + result.append( matcher.group(3) ); + return result.toString(); + } + + final StringBuilder result = new StringBuilder(); + while( (matcher=VAR2.matcher(line)).find() ) { + final String name = matcher.group(1); + if( !binding.containsKey(name) && !ignoreNames.contains(name)) { + throw new IllegalArgumentException("Missing template key: "+name); + } + final String value = !ignoreNames.contains(name) + ? (binding.get(name)!=null ? binding.get(name) : "") + : "{{"+name+"}}"; + final int p = matcher.start(1); + final int q = matcher.end(1); + + result.append(line.substring(0,p-2)); + result.append(value); + line = line.substring(q+2); + } + result.append(line); + return result.toString(); + } + +} diff --git a/wave-utils/src/test/groovy/io/seqera/wave/util/TemplateRendererTest.groovy b/wave-utils/src/test/groovy/io/seqera/wave/util/TemplateRendererTest.groovy new file mode 100644 index 000000000..392893e16 --- /dev/null +++ b/wave-utils/src/test/groovy/io/seqera/wave/util/TemplateRendererTest.groovy @@ -0,0 +1,202 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.wave.util + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class TemplateRendererTest extends Specification { + + def 'should replace vars' () { + given: + def binding = [foo: 'Hello', bar: 'world'] + def render = new TemplateRenderer() + expect: + render.replace0('{{foo}}', binding) == 'Hello' + render.replace0('{{foo}} ', binding) == 'Hello ' + render.replace0('{{foo}}\n', binding) == 'Hello\n' + render.replace0(' {{foo}}', binding) == ' Hello' + render.replace0(' {{foo}}\n', binding) == ' Hello\n' + render.replace0(' ${foo}', binding) == ' ${foo}' + render.replace0(' ${{foo}}', binding) == ' ${{foo}}' + render.replace0('{{foo}}', [foo:'']) == '' + render.replace0('{{foo}}', [foo:null]) == null + render.replace0(' {{foo}}\n', [foo:null]) == null + render.replace0('', binding) == '' + render.replace0(null, binding) == null + + render.replace0('{{foo}} {{bar}}!', binding) == 'Hello world!' + render.replace0('abc {{foo}} pq {{bar}} xyz', binding) == 'abc Hello pq world xyz' + render.replace0('{{foo}} 123 {{bar}} xyz {{foo}}', binding) == 'Hello 123 world xyz Hello' + render.replace0('1{{foo}}2{{foo}}3', [foo:'']) == '123' + render.replace0('1{{foo}}2{{foo}}3', [foo:null]) == '123' + } + + def 'should throw an exception when missing variables' () { + when: + new TemplateRenderer().replace0('{{x1}}', [:]) + then: + def e = thrown(IllegalArgumentException) + e.message == 'Missing template key: x1' + + when: + new TemplateRenderer().replace0('{{foo}} {{x2}}', [foo:'ciao']) + then: + e = thrown(IllegalArgumentException) + e.message == 'Missing template key: x2' + } + + def 'should not throw an exception when missing variables' () { + when: + def result = new TemplateRenderer().withIgnore("x1").replace0('{{x1}}', [x1:'one']) + then: + result == '{{x1}}' + + when: + result = new TemplateRenderer().withIgnore('x1','x2').replace0('{{x1}} {{x2}}', [x1:'one']) + then: + result == '{{x1}} {{x2}}' + } + + def 'should render template' () { + given: + def template = "Hello, {{name}}!\n" + + "Today is {{day}} and the weather is {{weather}}."; + and: + def binding = new HashMap(); + binding.put("name", "John"); + binding.put("day", "Monday"); + binding.put("weather", "sunny"); + + when: + def renderer = new TemplateRenderer() + and: + def result = renderer.render(template, binding); + + then: + result == 'Hello, John!\nToday is Monday and the weather is sunny.' + } + + def 'should render a template with comment'() { + given: + def template = """\ + ## remove this comment + 1: {{alpha}} + 2: {{delta}} {{delta}} + 3: {{gamma}} {{gamma}} {{gamma}} + 4: end + """.stripIndent() + and: + def binding = new HashMap(); + binding.put("alpha", "one"); + binding.put("delta", "two"); + binding.put("gamma", "three"); + + when: + def renderer = new TemplateRenderer() + and: + def result = renderer.render(new ByteArrayInputStream(template.bytes), binding); + + then: + result == """\ + 1: one + 2: two two + 3: three three three + 4: end + """.stripIndent() + } + + + def 'should render a template using an input stream'() { + given: + def template = """\ + {{one}} + {{two}} + xxx + {{three}} + zzz + """.stripIndent() + and: + def binding = [ + one: '1', // this is rendered + two:null, // a line containing a null variable is not rendered + three:'' // empty value is considered ok + ] + + when: + def renderer = new TemplateRenderer() + and: + def result = renderer.render(new ByteArrayInputStream(template.bytes), binding); + + then: + result == """\ + 1 + xxx + + zzz + """.stripIndent() + } + + def 'should render template with indentations' () { + given: + def binding = [foo: 'Hello', bar: 'world'] + + when: + def renderer = new TemplateRenderer() + and: + def result = renderer.render('{{foo}}\n{{bar}}', binding) + then: + result == 'Hello\nworld' + + when: + def template = '''\ + {{foo}} + {{bar}} + '''.stripIndent() + result = renderer.render(template, [foo:'11\n22\n33', bar:'Hello world']) + then: + result == '''\ + 11 + 22 + 33 + Hello world + '''.stripIndent() + + + when: + template = '''\ + {{x1}} + {{x2}} + {{x3}} + '''.stripIndent() + result = renderer.render(template, [x1:'aa\nbb\n', x2:null, x3:'pp\nqq']) + then: + result == '''\ + aa + bb + + pp + qq + '''.stripIndent() + + } + +}