Skip to content

Commit

Permalink
fix: additional templating and logic fixes (#346)
Browse files Browse the repository at this point in the history 
* fix: no imagecache for pull through images if dockerhub credentials are provided

* fix: add missing LAGOON_SSH_PRIVATE_KEY build arg

* fix: fsgroup 0 on services that had it previously

* fix: trim whitespace from cron

* test: add test for rootless workload with cronjobs

* test: add not enough and too many field checks for cronjobs with better error

* fix: k8up newer version backup annotation on deployments
  • Loading branch information
@shreddedbacon
shreddedbacon authored Aug 5, 2024
1 parent 8c4437d commit 2fa66e0
Show file tree
Hide file tree
Showing 44 changed files with 1,282 additions and 64 deletions.
106 changes: 106 additions & 0 deletions cmd/identify_imagebuild_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"LAGOON_GIT_SHA": "abcdefg123456",
"LAGOON_GIT_BRANCH": "main",
"NODE_IMAGE": "example-project-main-node",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand Down Expand Up @@ -85,6 +86,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"CLI_IMAGE": "example-project-main-cli",
"NGINX_IMAGE": "example-project-main-nginx",
"PHP_IMAGE": "example-project-main-php",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand Down Expand Up @@ -172,6 +174,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"NGINX_IMAGE": "example-project-main-nginx",
"PHP_IMAGE": "example-project-main-php",
"LAGOON_FEATURE_FLAG_IMAGECACHE_REGISTRY": "imagecache.example.com",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand Down Expand Up @@ -247,6 +250,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"LND_IMAGE": "example-project-main-lnd",
"THUNDERHUB_IMAGE": "example-project-main-thunderhub",
"TOR_IMAGE": "example-project-main-tor",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand Down Expand Up @@ -315,6 +319,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"LAGOON_GIT_BRANCH": "main",
"LND_IMAGE": "example-project-main-lnd",
"TOR_IMAGE": "example-project-main-tor",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand Down Expand Up @@ -358,6 +363,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"LAGOON_BUILD_TYPE": "promote",
"LAGOON_GIT_SOURCE_REPOSITORY": "ssh://[email protected]/lagoon-demo.git",
"LAGOON_KUBERNETES": "remote-cluster1",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand Down Expand Up @@ -406,6 +412,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"LAGOON_GIT_SOURCE_REPOSITORY": "ssh://[email protected]/lagoon-demo.git",
"LAGOON_KUBERNETES": "remote-cluster1",
"NODE_IMAGE": "example-project-pr-123-node",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand Down Expand Up @@ -441,6 +448,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"LAGOON_BUILD_TYPE": "promote",
"LAGOON_GIT_SOURCE_REPOSITORY": "ssh://[email protected]/lagoon-demo.git",
"LAGOON_KUBERNETES": "remote-cluster1",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand Down Expand Up @@ -510,6 +518,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"NGINX_IMAGE": "example-project-main-nginx",
"PHP_IMAGE": "example-project-main-php",
"LAGOON_FEATURE_FLAG_IMAGECACHE_REGISTRY": "imagecache.example.com",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
ContainerRegistries: []generator.ContainerRegistry{
{
Expand Down Expand Up @@ -588,6 +597,7 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
"LAGOON_GIT_BRANCH": "main",
"NODE_IMAGE": "example-project-main-node",
"LAGOON_CACHE_node": "harbor.example/example-project/main/node@sha256:e90daba405cbf33bab23fe8a021146811b2c258df5f2afe7dadc92c0778eef45",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
Images: []imageBuilds{
{
Expand All @@ -602,6 +612,102 @@ func TestImageBuildConfigurationIdentification(t *testing.T) {
},
},
},
{
name: "test10 nginx-php external pull images from dockerhub",
args: testdata.GetSeedData(
testdata.TestData{
Namespace: "example-project-main",
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
LagoonYAML: "internal/testdata/complex/lagoon.varnish3.yml",
ProjectVariables: []lagoon.EnvironmentVariable{
{
Name: "LAGOON_FEATURE_FLAG_IMAGECACHE_REGISTRY",
Value: "imagecache.example.com",
Scope: "build",
},
},
}, true),
want: imageBuild{
BuildKit: false,
BuildArguments: map[string]string{
"LAGOON_BUILD_NAME": "lagoon-build-abcdefg",
"LAGOON_PROJECT": "example-project",
"LAGOON_ENVIRONMENT": "main",
"LAGOON_ENVIRONMENT_TYPE": "production",
"LAGOON_BUILD_TYPE": "branch",
"LAGOON_GIT_SOURCE_REPOSITORY": "ssh://[email protected]/lagoon-demo.git",
"LAGOON_KUBERNETES": "remote-cluster1",
"LAGOON_GIT_SHA": "0000000000000000000000000000000000000000",
"LAGOON_GIT_BRANCH": "main",
"CLI_IMAGE": "example-project-main-cli",
"NGINX_IMAGE": "example-project-main-nginx",
"PHP_IMAGE": "example-project-main-php",
"LAGOON_FEATURE_FLAG_IMAGECACHE_REGISTRY": "imagecache.example.com",
"LAGOON_SSH_PRIVATE_KEY": "-----BEGIN OPENSSH PRIVATE KEY-----\nthisisafakekey\n-----END OPENSSH PRIVATE KEY-----",
},
ContainerRegistries: []generator.ContainerRegistry{
{
Name: "my-custom-registry",
Username: "registry_user",
Password: "REGISTRY_PASSWORD",
SecretName: "lagoon-private-registry-my-custom-registry",
URL: "index.docker.io",
UsernameSource: ".lagoon.yml",
PasswordSource: ".lagoon.yml (we recommend using an environment variable, see the docs on container-registries for more information)",
},
{
Name: "my-other-custom-registry",
Username: "registry_user2",
Password: "REGISTRY_PASSWORD2",
SecretName: "lagoon-private-registry-my-other-custom-registry",
URL: "registry1.example.com",
UsernameSource: ".lagoon.yml",
PasswordSource: ".lagoon.yml (we recommend using an environment variable, see the docs on container-registries for more information)",
},
},
Images: []imageBuilds{
{
Name: "cli",
ImageBuild: generator.ImageBuild{
BuildImage: "harbor.example/example-project/main/cli:latest",
Context: "internal/testdata/complex/docker",
DockerFile: ".docker/Dockerfile.cli",
TemporaryImage: "example-project-main-cli",
},
}, {
Name: "nginx",
ImageBuild: generator.ImageBuild{
BuildImage: "harbor.example/example-project/main/nginx:latest",
Context: "internal/testdata/complex/docker",
DockerFile: ".docker/Dockerfile.nginx-drupal",
TemporaryImage: "example-project-main-nginx",
},
}, {
Name: "php",
ImageBuild: generator.ImageBuild{
BuildImage: "harbor.example/example-project/main/php:latest",
Context: "internal/testdata/complex/docker",
DockerFile: ".docker/Dockerfile.php",
TemporaryImage: "example-project-main-php",
},
}, {
Name: "redis",
ImageBuild: generator.ImageBuild{
BuildImage: "harbor.example/example-project/main/redis:latest",
PullImage: "registry1.example.com/amazeeio/redis:latest",
},
}, {
Name: "varnish",
ImageBuild: generator.ImageBuild{
BuildImage: "harbor.example/example-project/main/varnish:latest",
PullImage: "uselagoon/varnish-5-drupal:latest",
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
41 changes: 26 additions & 15 deletions cmd/template_lagoonservices_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,32 @@ func TestTemplateLagoonServices(t *testing.T) {
templatePath: "testoutput",
want: "internal/testdata/complex/service-templates/service2",
},
{
name: "test2b nginx-php deployment - rootless workloads enabled",
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
LagoonYAML: "internal/testdata/complex/lagoon.varnish.yml",
ImageReferences: map[string]string{
"nginx": "harbor.example/example-project/main/nginx@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"php": "harbor.example/example-project/main/php@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"cli": "harbor.example/example-project/main/cli@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"redis": "harbor.example/example-project/main/redis@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"varnish": "harbor.example/example-project/main/varnish@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
},
ProjectVariables: []lagoon.EnvironmentVariable{
{
Name: "LAGOON_FEATURE_FLAG_ROOTLESS_WORKLOAD",
Value: "enabled",
Scope: "build",
},
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/complex/service-templates/service5",
},
{
name: "test3 - funky pvcs",
description: "only create pvcs of the requested persistent-name in the docker-compose file",
Expand Down Expand Up @@ -283,21 +309,6 @@ func TestTemplateLagoonServices(t *testing.T) {
templatePath: "testoutput",
want: "internal/testdata/complex/service-templates/service4",
},
{
name: "test10 basic deployment polysite cronjobs",
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
LagoonYAML: "internal/testdata/basic/lagoon.polysite-cronjobs.yml",
ImageReferences: map[string]string{
"node": "harbor.example/example-project/main/node@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/basic/service-templates/service7",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions internal/generator/build_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,5 +85,6 @@ func collectImageBuildArguments(buildValues BuildValues) map[string]string {
for _, icba := range buildValues.ImageCacheBuildArguments {
buildArgs[fmt.Sprintf("LAGOON_CACHE_%s", icba.Name)] = icba.Image
}
buildArgs["LAGOON_SSH_PRIVATE_KEY"] = buildValues.SSHPrivateKey
return buildArgs
}
2 changes: 2 additions & 0 deletions internal/generator/buildvalues.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ type BuildValues struct {
BackupsEnabled bool `json:"backupsEnabled"`
RouteQuota *int `json:"routeQuota"`
ImageCacheBuildArguments []ImageCacheBuildArguments `json:"imageCacheBuildArgs"`
IgnoreImageCache bool `json:"ignoreImageCache"`
SSHPrivateKey string `json:"sshPrivateKey"`
}

type Resources struct {
Expand Down
1 change: 1 addition & 0 deletions internal/generator/container_registries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func configureContainerRegistries(buildValues *BuildValues) error {
}
if cr.URL == "" {
cr.URL = "index.docker.io"
buildValues.IgnoreImageCache = true
}
eru := cr.URL
u, _ := url.Parse(eru)
Expand Down
2 changes: 2 additions & 0 deletions internal/generator/generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ type GeneratorInput struct {
DynamicSecrets []string
DynamicDBaaSSecrets []string
ImageCacheBuildArgsJSON string
SSHPrivateKey string
}

func NewGenerator(
Expand Down Expand Up @@ -110,6 +111,7 @@ func NewGenerator(
dynamicSecrets := helpers.GetEnv("DYNAMIC_SECRETS", strings.Join(generator.DynamicSecrets, ","), generator.Debug)
dynamicDBaaSSecrets := helpers.GetEnv("DYNAMIC_DBAAS_SECRETS", strings.Join(generator.DynamicDBaaSSecrets, ","), generator.Debug)
imageCacheBuildArgsJSON := helpers.GetEnv("LAGOON_CACHE_BUILD_ARGS", generator.ImageCacheBuildArgsJSON, generator.Debug)
buildValues.SSHPrivateKey = helpers.GetEnv("SSH_PRIVATE_KEY", generator.SSHPrivateKey, generator.Debug)
// this is used by CI systems to influence builds, it is rarely used and should probably be abandoned
buildValues.IsCI = helpers.GetEnvBool("CI", generator.CI, generator.Debug)

Expand Down
2 changes: 1 addition & 1 deletion internal/generator/services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ func composeToServiceValues(
}
if !ContainsRegistry(buildValues.ContainerRegistry, pullImage) {
// if the image isn't in dockerhub, then the imagecache can't be used
if buildValues.ImageCache != "" && strings.Count(pullImage, "/") == 1 {
if buildValues.ImageCache != "" && strings.Count(pullImage, "/") == 1 && !buildValues.IgnoreImageCache {
imageBuild.PullImage = fmt.Sprintf("%s%s", buildValues.ImageCache, imageBuild.PullImage)
}
}
Expand Down
5 changes: 4 additions & 1 deletion internal/helpers/helpers_cron.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func ConvertCrontab(namespace, cron string) (string, error) {
// namespace, so will not change after a deployment for a given namespace.
seed := cksum.Cksum([]byte(fmt.Sprintf("%s\n", namespace)))
var minutes, hours, days, months, dayweek string
splitCron := strings.Split(cron, " ")
splitCron := strings.Split(strings.Trim(cron, " "), " ")
// check the provided cron splits into 5
if len(splitCron) == 5 {
for idx, val := range splitCron {
Expand Down Expand Up @@ -215,6 +215,9 @@ func ConvertCrontab(namespace, cron string) (string, error) {
}
return fmt.Sprintf("%v %v %v %v %v", minutes, hours, days, months, dayweek), nil
}
if len(splitCron) < 5 && len(splitCron) > 0 || len(splitCron) > 5 {
return "", fmt.Errorf("cron definition '%s' is invalid, %d fields provided, required 5", cron, len(splitCron))
}
return "", fmt.Errorf("cron definition '%s' is invalid", cron)
}

Expand Down
Loading

0 comments on commit 2fa66e0

Please sign in to comment.