Clean retro (#127)

* Add a flag to only capture metadata
* Add TPC-C benchmark
* Fixed replay and retro algorithm.
* Fixed postgres demo

postgres-demo/pom.xml

@@ -108,32 +108,6 @@
 					</execution>
 				</executions>
 			</plugin>
-			<plugin>
-				<artifactId>maven-assembly-plugin</artifactId>
-				<version>3.2.0</version>
-				<configuration>
-					<descriptors>
-						<descriptor>src/main/assembly/assembly.xml</descriptor>
-					</descriptors>
-				</configuration>
-				<executions>
-					<execution>
-						<id>rollback-assembly</id>
-						<phase>package</phase>
-						<goals>
-							<goal>single</goal>
-						</goals>
-						<configuration>
-							<archive>
-								<manifest>
-									<mainClass>org.dbos.apiary.postgresdemo.executable.RollbackExecutable</mainClass>
-								</manifest>
-							</archive>
-							<finalName>demo-rollback-exec</finalName>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
 		</plugins>
 	</build>
 

postgres-demo/src/main/java/org/dbos/apiary/postgresdemo/NectarController.java

@@ -31,6 +31,10 @@ public class NectarController {
     ApiaryWorkerClient client;
 
     public NectarController() throws SQLException {
+        ApiaryConfig.captureUpdates = true;
+        ApiaryConfig.captureReads = true;
+        ApiaryConfig.provenancePort = 5432;  // Store provenance data in the same database.
+
         PostgresConnection conn = new PostgresConnection("localhost", ApiaryConfig.postgresPort, "postgres", "dbos");
         conn.dropTable("WebsiteLogins"); // For testing.
         conn.dropTable("WebsitePosts"); // For testing.

scripts/initialize_postgres_docker.sh

@@ -7,20 +7,28 @@ SCRIPT_DIR=$(dirname $(realpath $0))
 docker pull postgres:14.5-bullseye
 
 PGCONFIG=""  # Postgres config file.
+NAME="apiary-postgres"
+PORT=5432
 if [[ $# -eq 1 ]]; then
     PGCONFIG="$PWD/$1"
 fi
 
+if [[ $# -eq 3 ]]; then
+    PGCONFIG="$PWD/$1"
+    NAME="$2"
+    PORT="$3"
+fi
+
 # Set the password to dbos, default user is postgres.
 if [[ -z "$PGCONFIG" ]]; then
-    docker run -d --network host --rm --name="apiary-postgres" --env POSTGRES_PASSWORD=dbos postgres:14.5-bullseye
+    docker run -d --network host --rm --name="$NAME" --env PGDATA=/var/lib/postgresql-static/data --env POSTGRES_PASSWORD=dbos postgres:14.5-bullseye
 else
     # Use customized config file
-    docker run -d --network host --rm --name="apiary-postgres" --env POSTGRES_PASSWORD=dbos \
+    docker run -d --network host --rm --name="$NAME" --env PGDATA=/var/lib/postgresql-static/data --env POSTGRES_PASSWORD=dbos \
     -v "$PGCONFIG":/etc/postgresql/postgresql.conf \
     postgres:14.5-bullseye -c 'config_file=/etc/postgresql/postgresql.conf'
 fi
 
 sleep 10
 
-docker exec -i apiary-postgres psql -h localhost -U postgres -t < ${SCRIPT_DIR}/init_postgres.sql
+docker exec -i $NAME psql -h localhost -U postgres -p $PORT -t < ${SCRIPT_DIR}/init_postgres.sql

scripts/pg_provenance.conf

@@ -0,0 +1,42 @@
+# -----------------------------
+# Changed for Apiary
+# -----------------------------
+track_commit_timestamp = off	# default off, collect timestamp of transaction commit
+				# (change requires restart)
+max_connections = 200			# default 100, (change requires restart)
+
+# -----------------------------
+# Default settings.
+# -----------------------------
+listen_addresses = '*'
+					# comma-separated list of addresses;
+					# defaults to 'localhost'; use '*' for all
+					# (change requires restart)
+port = 5433		    # (change requires restart)
+shared_buffers = 128MB			# min 128kB
+					# (change requires restart)
+dynamic_shared_memory_type = posix	# the default is the first option
+					# supported by the operating system:
+					#   posix
+					#   sysv
+					#   windows
+					#   mmap
+					# (change requires restart)
+max_wal_size = 1GB
+min_wal_size = 80MB
+log_timezone = 'Etc/UTC'
+
+# - Locale and Formatting -
+datestyle = 'iso, mdy'
+#intervalstyle = 'postgres'
+timezone = 'Etc/UTC'
+
+# These settings are initialized by initdb, but they can be changed.
+lc_messages = 'en_US.utf8'			# locale for system error message
+					# strings
+lc_monetary = 'en_US.utf8'			# locale for monetary formatting
+lc_numeric = 'en_US.utf8'			# locale for number formatting
+lc_time = 'en_US.utf8'				# locale for time formatting
+
+# default configuration for text search
+default_text_search_config = 'pg_catalog.english'

scripts/postgres_replay.conf

@@ -0,0 +1,44 @@
+# -----------------------------
+# Changed for Apiary
+# -----------------------------
+track_commit_timestamp = off	# default off, collect timestamp of transaction commit
+				# (change requires restart)
+max_connections = 200			# default 100, (change requires restart)
+synchronous_commit = off		# synchronization level;
+					# off, local, remote_write, remote_apply, or on
+
+# -----------------------------
+# Default settings.
+# -----------------------------
+listen_addresses = '*'
+					# comma-separated list of addresses;
+					# defaults to 'localhost'; use '*' for all
+					# (change requires restart)
+#port = 5432				# (change requires restart)
+shared_buffers = 8GB			# min 128kB
+					# (change requires restart)
+dynamic_shared_memory_type = posix	# the default is the first option
+					# supported by the operating system:
+					#   posix
+					#   sysv
+					#   windows
+					#   mmap
+					# (change requires restart)
+max_wal_size = 1GB
+min_wal_size = 80MB
+log_timezone = 'Etc/UTC'
+
+# - Locale and Formatting -
+datestyle = 'iso, mdy'
+#intervalstyle = 'postgres'
+timezone = 'Etc/UTC'
+
+# These settings are initialized by initdb, but they can be changed.
+lc_messages = 'en_US.utf8'			# locale for system error message
+					# strings
+lc_monetary = 'en_US.utf8'			# locale for monetary formatting
+lc_numeric = 'en_US.utf8'			# locale for number formatting
+lc_time = 'en_US.utf8'				# locale for time formatting
+
+# default configuration for text search
+default_text_search_config = 'pg_catalog.english'

scripts/restore_postgres_docker.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -ex
+
+SCRIPT_DIR=$(dirname $(realpath $0))
+
+# Start Postgres Docker image.
+docker pull postgres:14.5-bullseye
+
+PGCONFIG=""  # Postgres config file.
+NAME="tpcc-postgres"
+PORT=5432
+
+if [[ $# -eq 1 ]]; then
+    PGCONFIG="$PWD/$1"
+fi
+
+
+# Set the password to dbos, default user is postgres.
+if [[ -z "$PGCONFIG" ]]; then
+    docker run -d --network host --rm --name="$NAME" --env PGDATA=/var/lib/postgresql-static/data --env POSTGRES_PASSWORD=dbos tpcc-postgres:latest
+else
+    # Use customized config file
+    docker run -d --network host --rm --name="$NAME" --env PGDATA=/var/lib/postgresql-static/data --env POSTGRES_PASSWORD=dbos \
+    -v "$PGCONFIG":/etc/postgresql/postgresql.conf \
+    tpcc-postgres:latest -c 'config_file=/etc/postgresql/postgresql.conf'
+fi
+
+

scripts/track_xact_ts.conf

@@ -1,10 +1,9 @@
 # -----------------------------
 # Changed for Apiary
 # -----------------------------
-track_commit_timestamp = on	# default off, collect timestamp of transaction commit
+track_commit_timestamp = off	# default off, collect timestamp of transaction commit
 				# (change requires restart)
 max_connections = 200			# default 100, (change requires restart)
-
 # -----------------------------
 # Default settings.
 # -----------------------------
@@ -13,7 +12,7 @@ listen_addresses = '*'
 					# defaults to 'localhost'; use '*' for all
 					# (change requires restart)
 #port = 5432				# (change requires restart)
-shared_buffers = 128MB			# min 128kB
+shared_buffers = 8GB			# min 128kB
 					# (change requires restart)
 dynamic_shared_memory_type = posix	# the default is the first option
 					# supported by the operating system:
@@ -39,4 +38,4 @@ lc_numeric = 'en_US.utf8'			# locale for number formatting
 lc_time = 'en_US.utf8'				# locale for time formatting
 
 # default configuration for text search
-default_text_search_config = 'pg_catalog.english'
+default_text_search_config = 'pg_catalog.english'

src/main/java/org/dbos/apiary/benchmarks/BenchmarkingExecutable.java

@@ -22,7 +22,7 @@ public static void main(String[] args) throws Exception {
         options.addOption("d", true, "Duration (sec)?");
         options.addOption("i", true, "Benchmark Interval (μs)");
         options.addOption("mainHostAddr", true, "Address of the main host to connect to.");
-        options.addOption("s", true, "Service Name ([moodle, wordpress] in retro benchmark)");
+        options.addOption("s", true, "Service Name ([moodle, wordpress, tpcc] in retro benchmark)");
         options.addOption("p1", true, "Percentage 1");
         options.addOption("p2", true, "Percentage 2");
         options.addOption("p3", true, "Percentage 3");
@@ -64,11 +64,13 @@ public static void main(String[] args) throws Exception {
             logger.info("Disabling provenance tracing!");
             ApiaryConfig.captureReads = false;
             ApiaryConfig.captureUpdates = false;
+            ApiaryConfig.captureMetadata = false;
             ApiaryConfig.recordInput = false;
         } else {
             logger.info("Provenance tracing enabled!");
             ApiaryConfig.captureReads = true;
             ApiaryConfig.captureUpdates = true;
+            ApiaryConfig.captureMetadata = true;
             ApiaryConfig.recordInput = true;
         }
 

src/main/java/org/dbos/apiary/benchmarks/RetroBenchmark.java

@@ -2,6 +2,7 @@
 
 import com.google.protobuf.InvalidProtocolBufferException;
 import org.dbos.apiary.benchmarks.retro.MoodleBenchmark;
+import org.dbos.apiary.benchmarks.tpcc.TPCCBenchmark;
 import org.dbos.apiary.benchmarks.retro.WordPressBenchmark;
 import org.dbos.apiary.client.ApiaryWorkerClient;
 import org.dbos.apiary.function.FunctionOutput;
@@ -14,14 +15,32 @@
 public class RetroBenchmark {
     private static final Logger logger = LoggerFactory.getLogger(RetroBenchmark.class);
 
+    public static int provenancePort = 5433;  // Change to 5433 for a separate Postgres or Vertica.
+    public static String provenanceDB = ApiaryConfig.vertica; // Change to Vertica as needed.
+    public static String provenanceAddr = "localhost"; // Change to other addresses as needed.
+
     public static void benchmark(String appName, String dbAddr, Integer interval, Integer duration, boolean skipLoad, int retroMode, long startExecId, long endExecId, String bugFix, List<Integer> percentages) throws Exception {
 
-        // Always disable read capture for retro replay mode. Only capture read metadata.
+        // Disable provenance tracking, only capture func invocations and input.
         ApiaryConfig.captureReads = false;
+        ApiaryConfig.captureUpdates = false;
+        ApiaryConfig.captureMetadata = false;
+
+        // Change provenance info.
+        if (!ApiaryConfig.recordInput) {
+            // Use postgres itself just to pass init.
+            provenanceAddr = dbAddr;
+            provenanceDB = ApiaryConfig.postgres;
+            provenancePort = ApiaryConfig.postgresPort;
+        }
+        ApiaryConfig.provenancePort = provenancePort;
+
         if (appName.equalsIgnoreCase("moodle")) {
             MoodleBenchmark.benchmark(dbAddr, interval, duration, skipLoad, retroMode, startExecId, endExecId, bugFix, percentages);
         } else if (appName.equalsIgnoreCase("wordpress")) {
             WordPressBenchmark.benchmark(dbAddr, interval, duration, skipLoad, retroMode, startExecId, endExecId, bugFix, percentages);
+        } else if (appName.equalsIgnoreCase("tpcc")) {
+            TPCCBenchmark.benchmark(dbAddr, interval, duration, retroMode, startExecId, endExecId, percentages);
         }
     }