devdb: use readinessProbe to check status (#201)

.github/workflows/integration.yaml

@@ -28,11 +28,14 @@ jobs:
           go-version-file: 'go.mod'
       - name: Create k8s Kind Cluster
         uses: helm/kind-action@v1
+      - name: Install Skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.3.1/skaffold-linux-amd64 && \
+          sudo install skaffold /usr/local/bin/
       - name: Run e2e tests
         run: |
-          make docker-build kind-image test-e2e \
+          make test-e2e \
             ATLAS_TOKEN=${{ secrets.ATLAS_TOKEN }} \
-            IMG=ariga/atlas-operator:e2e \
             KIND_CLUSTER=chart-testing
   test:
     runs-on: ubuntu-latest

internal/controller/devdb.go

@@ -161,9 +161,7 @@ func (r *devDBReconciler) devURL(ctx context.Context, sc client.Object, targetUR
 	case len(pods.Items) == 0:
 		return "", transient(errors.New("no pods found"))
 	}
-	idx := slices.IndexFunc(pods.Items, func(p corev1.Pod) bool {
-		return p.Status.Phase == corev1.PodRunning
-	})
+	idx := slices.IndexFunc(pods.Items, isPodReady)
 	if idx == -1 {
 		return "", transient(errors.New("no running pods found"))
 	}
@@ -185,6 +183,12 @@ func (r *devDBReconciler) devURL(ctx context.Context, sc client.Object, targetUR
 	return "", errors.New("no connection template annotation found")
 }
 
+func isPodReady(pod corev1.Pod) bool {
+	return slices.ContainsFunc(pod.Status.Conditions, func(c corev1.PodCondition) bool {
+		return c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue
+	})
+}
+
 // devDB contains values used to render a devDB pod template.
 type devDB struct {
 	types.NamespacedName
@@ -197,8 +201,8 @@ type devDB struct {
 	SchemaBound bool
 }
 
-// ConnTmpl returns a connection template for the devDB.
-func (d *devDB) ConnTmpl() string {
+// connTmpl returns a connection template for the devDB.
+func (d *devDB) connTmpl() string {
 	u := url.URL{
 		Scheme: d.Driver,
 		User:   url.UserPassword(d.User, d.Pass),
@@ -220,7 +224,7 @@ func (d *devDB) ConnTmpl() string {
 	return u.String()
 }
 
-func (d *devDB) Render(w io.Writer) error {
+func (d *devDB) render(w io.Writer) error {
 	return tmpl.ExecuteTemplate(w, "devdb.tmpl", d)
 }
 
@@ -253,13 +257,16 @@ func deploymentDevDB(name types.NamespacedName, drv string, schemaBound bool) (*
 		return nil, fmt.Errorf("unsupported driver %q", v.Driver)
 	}
 	b := &bytes.Buffer{}
-	if err := v.Render(b); err != nil {
+	if err := v.render(b); err != nil {
 		return nil, err
 	}
 	d := &appsv1.Deployment{}
 	if err := yaml.NewYAMLToJSONDecoder(b).Decode(d); err != nil {
 		return nil, err
 	}
+	d.Spec.Template.Annotations = map[string]string{
+		annoConnTmpl: v.connTmpl(),
+	}
 	if drv == "sqlserver" {
 		c := &d.Spec.Template.Spec.Containers[0]
 		if v := os.Getenv("MSSQL_ACCEPT_EULA"); v != "" {

internal/controller/templates/devdb.tmpl

@@ -20,8 +20,6 @@ spec:
         "app.kubernetes.io/part-of": "atlas-operator"
         "app.kubernetes.io/created-by": "controller-manager"
         "atlasgo.io/engine": "{{ .Driver }}"
-      annotations:
-        "atlasgo.io/conntmpl": "{{ .ConnTmpl }}"
     spec:
       {{- if ne .Driver "sqlserver" }}
       securityContext:
@@ -46,6 +44,14 @@ spec:
           - name: MYSQL_DATABASE
             value: {{ .DB }}
           {{- end }}
+          readinessProbe:
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 5
+            exec:
+              command: [
+                "mysql", "-u", "root", "-h", "127.0.0.1", "-ppass", "-e", "SELECT 1"
+              ]
           {{- else if eq .Driver "postgres" }}
           image: postgres:15
           env:
@@ -55,11 +61,27 @@ spec:
               value: pass
             - name: POSTGRES_USER
               value: root
+          readinessProbe:
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 5
+            exec:
+              command: [ "pg_isready" ]
           {{- else if eq .Driver "sqlserver" }}
           image: mcr.microsoft.com/mssql/server:2022-latest
           env:
             - name: MSSQL_SA_PASSWORD
               value: {{ .Pass }}
+          readinessProbe:
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            timeoutSeconds: 5
+            exec:
+              command: [
+                "/opt/mssql-tools18/bin/sqlcmd",
+                "-C", "-U", "sa", "-P", "{{ .Pass }}",
+                "-Q", "SELECT 1"
+              ]
           {{- end }}
           ports:
           - containerPort: {{ .Port }}

test/e2e/e2e_test.go

@@ -34,23 +34,18 @@ const (
 )
 
 func TestOperator(t *testing.T) {
-	img := os.Getenv("IMG")
-	if img == "" {
-		img = "ariga/atlas-operator:e2e"
-	}
 	kindCluster := os.Getenv("KIND_CLUSTER")
 	if kindCluster == "" {
 		kindCluster = "kind"
 	}
-	dir, err := getProjectDir()
-	require.NoError(t, err)
 	// Creating kubeconfig for the kind cluster
 	kubeconfig := filepath.Join(t.TempDir(), "kubeconfig")
-	err = pipeFile(kubeconfig, func(f io.Writer) error {
+	require.NoError(t, pipeFile(kubeconfig, func(f io.Writer) error {
 		cmd := exec.Command("kind", "get", "kubeconfig", "--name", kindCluster)
 		cmd.Stdout, cmd.Stderr = f, os.Stderr
 		return cmd.Run()
-	})
+	}))
+	dir, err := getProjectDir()
 	require.NoError(t, err)
 	// Kind run the command with kubeconfig set to kind cluster
 	kind := func(name string, args ...string) (string, error) {
@@ -66,41 +61,37 @@ func TestOperator(t *testing.T) {
 		return string(output), nil
 	}
 	// Deploying the controller-manager
-	_, err = kind("make", "deploy", "install", fmt.Sprintf("IMG=%s", img))
+	_, err = kind("skaffold", "run", "--wait-for-connection=true", "-p", "integration")
+	require.NoError(t, err)
+	// Installing the CRDs
+	_, err = kind("make", "install")
 	require.NoError(t, err)
+	t.Cleanup(func() {
+		_, err = kind("make", "undeploy", "ignore-not-found=true")
+		require.NoError(t, err)
+	})
 	// Accept the EULA and set the PID
 	_, err = kind("kubectl", "set", "env",
 		"-n", nsController, controller,
 		"MSSQL_ACCEPT_EULA=Y", "MSSQL_PID=Developer")
 	require.NoError(t, err)
-	// Restarting the controller-manager, to ensure the latest image is used
-	_, err = kind("kubectl", "rollout", "restart",
-		"-n", nsController, controller)
-	require.NoError(t, err)
-	// Waiting for the controller-manager to be ready
-	_, err = kind("kubectl", "rollout", "status",
-		"-n", nsController, controller,
-		"--timeout", "2m")
-	require.NoError(t, err)
 	var controllerPod string
 	for range 5 {
-		// Wait for the old pods to be deleted
-		<-time.After(time.Second)
 		// Getting the controller-manager pod name
-		output, err := kind("kubectl", "wait", "pod",
+		output, err := kind("kubectl", "get", "pod",
 			"-n", nsController,
 			"-l", "control-plane=controller-manager",
-			"--for", "condition=Ready",
-			"--timeout", "2m",
-			"-o", "go-template",
-			"--template", "{{.metadata.name}}",
+			"-o", "jsonpath",
+			"--template", "{.items[*].metadata.name}",
 		)
 		require.NoError(t, err)
-		pods := strings.Split(output, "\n")
+		pods := strings.Split(output, " ")
 		if len(pods) == 1 {
 			controllerPod = pods[0]
 			break
 		}
+		// Wait 5s before retrying
+		<-time.After(time.Second * 5)
 	}
 	require.NotEmpty(t, controllerPod, "controller-manager pod not found")
 	// Running the test script
@@ -127,36 +118,6 @@ func TestOperator(t *testing.T) {
 			return nil
 		},
 		Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
-			// expand reads a file and expands it with the environment variables
-			"expand": func(ts *testscript.TestScript, neg bool, args []string) {
-				if neg {
-					ts.Fatalf("unsupported: ! expand")
-				}
-				if len(args) < 1 {
-					ts.Fatalf("usage: expand filename")
-				}
-				w := ts.Stdout()
-				// If the last argument is >, write to a file
-				if l := len(args); l > 2 && args[l-2] == ">" {
-					outPath := filepath.Join(ts.Getenv("WORK"), args[l-1])
-					f, err := os.Create(outPath)
-					ts.Check(err)
-					defer f.Close()
-					w = f
-				}
-				content := os.Expand(ts.ReadFile(args[0]), ts.Getenv)
-				_, err := w.Write([]byte(content))
-				ts.Check(err)
-			},
-			// kubectl runs kubectl with the namespace set to the test namespace
-			"kubectl": func(ts *testscript.TestScript, neg bool, args []string) {
-				err := ts.Exec("kubectl", append([]string{"-n", ts.Getenv("NAMESPACE")}, args...)...)
-				if !neg {
-					ts.Check(err)
-				} else if err == nil {
-					ts.Fatalf("unexpected success")
-				}
-			},
 			// atlas runs the atlas binary in the controller-manager pod
 			"atlas": func(ts *testscript.TestScript, neg bool, args []string) {
 				err := ts.Exec("kubectl", "exec",
@@ -169,6 +130,15 @@ func TestOperator(t *testing.T) {
 					ts.Fatalf("unexpected success")
 				}
 			},
+			// kubectl runs kubectl with the namespace set to the test namespace
+			"kubectl": func(ts *testscript.TestScript, neg bool, args []string) {
+				err := ts.Exec("kubectl", append([]string{"-n", ts.Getenv("NAMESPACE")}, args...)...)
+				if !neg {
+					ts.Check(err)
+				} else if err == nil {
+					ts.Fatalf("unexpected success")
+				}
+			},
 			// envfile read the file and using its content as environment variables
 			"envfile": func(ts *testscript.TestScript, neg bool, args []string) {
 				if neg {
@@ -182,6 +152,27 @@ func TestOperator(t *testing.T) {
 					ts.Setenv(vals[0], ts.ReadFile(vals[1]))
 				}
 			},
+			// cat reads a file and expands it with the environment variables
+			"cat": func(ts *testscript.TestScript, neg bool, args []string) {
+				if neg {
+					ts.Fatalf("unsupported: ! cat")
+				}
+				if len(args) < 1 {
+					ts.Fatalf("usage: cat filename")
+				}
+				w := ts.Stdout()
+				// If the last argument is >, write to a file
+				if l := len(args); l > 2 && args[l-2] == ">" {
+					outPath := filepath.Join(ts.Getenv("WORK"), args[l-1])
+					f, err := os.Create(outPath)
+					ts.Check(err)
+					defer f.Close()
+					w = f
+				}
+				content := os.Expand(ts.ReadFile(args[0]), ts.Getenv)
+				_, err := w.Write([]byte(content))
+				ts.Check(err)
+			},
 		},
 	})
 }

test/e2e/testdata/atlas-schema/postgres.txtar

@@ -93,4 +93,4 @@ spec:
             periodSeconds: 2
             timeoutSeconds: 1
             exec:
-              command: [ "pg_isready", "-U", "postgres" ]
+              command: [ "pg_isready" ]

test/e2e/testdata/atlas-schema/sqlserver.txtar

@@ -96,6 +96,6 @@ spec:
             exec:
               command: [
                 "/opt/mssql-tools18/bin/sqlcmd",
-                "-C", "-U", "sa", "-P","P@ssw0rd0995",
+                "-C", "-U", "sa", "-P", "P@ssw0rd0995",
                 "-Q", "SELECT 1"
               ]