CLI Local Persistence (#827)

tembo-cli/src/cmd/apply.rs

@@ -67,6 +67,7 @@ const DOCKERFILE_NAME: &str = "Dockerfile";
 const DOCKERCOMPOSE_NAME: &str = "docker-compose.yml";
 const POSTGRESCONF_NAME: &str = "postgres.conf";
 const MAX_INT32: i32 = 2147483647;
+const PGDATA_NAME: &str = "init_pgdata.sh";
 
 /// Deploys a tembo.toml file
 #[derive(Args)]
@@ -312,6 +313,15 @@ fn docker_apply_instance(
         true,
     )?;
 
+    let rendered_pgdata_script: String = get_rendered_init_pgdata()?;
+
+    FileUtils::create_file(
+        PGDATA_NAME.to_string(),
+        instance_setting.instance_name.clone() + "/" + PGDATA_NAME,
+        rendered_pgdata_script,
+        true,
+    )?;
+
     instance_setting.final_extensions = extensions;
 
     FileUtils::create_file(
@@ -1080,6 +1090,17 @@ pub fn get_instance_settings(
     Ok(base_settings)
 }
 
+pub fn get_rendered_init_pgdata() -> Result<String, anyhow::Error> {
+    let contents = include_str!("../../tembo/init_pgdata.sh.template");
+
+    let mut tera = Tera::new("templates/**/*").unwrap();
+    let _ = tera.add_raw_template("init_pgdata", contents);
+    let context = Context::new();
+
+    let rendered_init_pgdata = tera.render("init_pgdata", &context)?;
+    Ok(rendered_init_pgdata)
+}
+
 pub fn get_rendered_dockerfile(
     trunk_installs: &Option<Vec<ControllerTrunkInstall>>,
     stack: &Stack,

tembo-cli/tembo/Dockerfile.template

@@ -24,10 +24,16 @@ RUN openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
     chown postgres:postgres /var/lib/postgresql/server.* && \
     chmod 600 /var/lib/postgresql/server.key
 
+# Copy the init_pgdata.sh script to the correct location
+COPY init_pgdata.sh /docker-entrypoint-initdb.d/init_pgdata.sh
+
+# Set permissions for the init_pgdata.sh script
+RUN chmod +x /docker-entrypoint-initdb.d/init_pgdata.sh
+
 USER postgres
 
-# Initialize the database
-RUN pg_ctl -c init
+# Initialize the database if not already initialized
+RUN if [ ! -s "$PGDATA/PG_VERSION" ]; then pg_ctl -c init; fi
 
 # Set permissive authentication (for local testing)
 RUN echo "hostssl all all 0.0.0.0/0 trust" >> ${PGDATA}/pg_hba.conf

tembo-cli/tembo/docker-compose.yml.template

@@ -3,9 +3,12 @@ version: '3.8'
 services:
 {% for key, instance in instance_settings %}
   {{instance.instance_name}}:
+    platform: linux/amd64
     build:
       context: ./{{instance.instance_name}}
     container_name: {{instance.instance_name}}
+    volumes:
+      - {{instance.instance_name}}-data:/var/lib/postgresql/data2
     networks:
       - tembo
     labels:
@@ -107,3 +110,8 @@ services:
 
 networks:
   tembo: {}
+
+volumes:
+{% for key, instance in instance_settings %}
+  {{instance.instance_name}}-data:
+{% endfor %}

tembo-cli/tembo/init_pgdata.sh.template

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Initialize the database if PGDATA is empty
+if [ ! -s "$PGDATA/PG_VERSION" ]; then
+    echo "Initializing database in $PGDATA"
+    initdb -D "$PGDATA" > /var/log/initdb.log 2>&1
+else
+    echo "Database already initialized in $PGDATA"
+fi
+
+# Start PostgreSQL
+exec postgres

tembo-cli/tests/integration_tests_docker.rs

@@ -348,6 +348,78 @@ async fn multiple_instances() -> Result<(), anyhow::Error> {
     Ok(())
 }
 
+#[tokio::test]
+async fn local_persistence() -> Result<(), anyhow::Error> {
+    let instance_name = "set";
+    let root_dir = env!("CARGO_MANIFEST_DIR");
+    let test_dir = PathBuf::from(root_dir).join("examples").join(instance_name);
+
+    env::set_current_dir(&test_dir)?;
+
+    // tembo init
+    let mut cmd = Command::cargo_bin(CARGO_BIN)?;
+    cmd.arg("init");
+    cmd.assert().success();
+
+    // tembo context set --name local
+    let mut cmd = Command::cargo_bin(CARGO_BIN)?;
+    cmd.arg("context");
+    cmd.arg("set");
+    cmd.arg("--name");
+    cmd.arg("local");
+    cmd.assert().success();
+
+    // tembo apply
+    let mut cmd = Command::cargo_bin(CARGO_BIN)?;
+    cmd.arg("--verbose");
+    cmd.arg("apply");
+    cmd.assert().success();
+
+    // Create a table and insert data
+    SqlxUtils::execute_sql(
+        instance_name.to_string(),
+        "CREATE TABLE test_table (id serial PRIMARY KEY, data TEXT NOT NULL);".to_string(),
+    )
+    .await?;
+
+    SqlxUtils::execute_sql(
+        instance_name.to_string(),
+        "INSERT INTO test_table (data) VALUES ('test data');".to_string(),
+    )
+    .await?;
+
+    // Stop the container
+    let mut cmd = Command::cargo_bin(CARGO_BIN)?;
+    cmd.arg("delete");
+    let _ = cmd.ok();
+
+    // Start the container again
+    let mut cmd = Command::cargo_bin(CARGO_BIN)?;
+    cmd.arg("--verbose");
+    cmd.arg("apply");
+    cmd.assert().success();
+
+    SqlxUtils::execute_sql(
+        instance_name.to_string(),
+        "SELECT * FROM test_table;".to_string(),
+    )
+    .await?;
+
+    // Stop the container
+    let mut cmd = Command::cargo_bin(CARGO_BIN)?;
+    cmd.arg("delete");
+    let _ = cmd.ok();
+
+    // Remove the Docker volume
+    let mut cmd = Command::new("docker");
+    cmd.arg("volume");
+    cmd.arg("rm");
+    cmd.arg("set_set-data");
+    cmd.assert().success();
+
+    Ok(())
+}
+
 async fn get_output_from_sql(instance_name: String, sql: String) -> Result<String, anyhow::Error> {
     // Configure SQLx connection options
     let connect_options = PgConnectOptions::new()