Test and fixes for cdb2_printlog

Signed-off-by: Mark Hannum <[email protected]>

berkdb/build/db.h

@@ -1348,8 +1348,9 @@ typedef enum {
 #define	DB_UPDATE_SECONDARY	34	/* c_get(), c_del() (internal) */
 #define	DB_WRITECURSOR		35	/* cursor() */
 #define	DB_WRITELOCK		36	/* cursor() (internal) */
-#define DB_PREV_VALUE		37	/* prev record without moving cursor */
-#define DB_NEXT_VALUE		38	/* next record without moving cursor */
+#define	DB_PREV_VALUE		37	/* prev record without moving cursor */
+#define	DB_NEXT_VALUE		38	/* next record without moving cursor */
+#define	DB_FIRST_FILE		39	/* first log-record in file */  
 
 /* This has to change when the max opcode hits 255. */
 #define	DB_OPFLAGS_MASK	0x000000ff	/* Mask for operations flags. */

berkdb/log/log_get.c

@@ -322,6 +322,7 @@ __log_c_get_pp(logc, alsn, dbt, flags)
 	switch (flags) {
 	case DB_CURRENT:
 	case DB_FIRST:
+	case DB_FIRST_FILE:
 	case DB_LAST:
 	case DB_NEXT:
 	case DB_PREV:
@@ -404,9 +405,10 @@ __log_c_get_timed(logc, alsn, dbt, flags)
 		return (ret);
 	}
 	if (alsn->offset == 0 && (flags == DB_FIRST ||
-	    flags == DB_NEXT || flags == DB_LAST || flags == DB_PREV)) {
+		flags == DB_NEXT || flags == DB_LAST || flags == DB_PREV || flags == DB_FIRST_FILE)) {
 		switch (flags) {
 		case DB_FIRST:
+		case DB_FIRST_FILE:
 			flags = DB_NEXT;
 			break;
 		case DB_LAST:
@@ -489,6 +491,10 @@ __log_c_get_int(logc, alsn, dbt, flags)
 
 	nlsn = logc->c_lsn;
 	switch (flags) {
+	case DB_FIRST_FILE:
+		nlsn.file = alsn->file;
+		nlsn.offset = 0;
+		break;
 	case DB_NEXT:				/* Next log record. */
 		if (!IS_ZERO_LSN(nlsn)) {
 			/* Increment the cursor by the cursor record size. */

db/comdb2.c

@@ -5601,6 +5601,25 @@ static void hash_no_op_callback(hash_t *const restrict hash, plhash_event_t even
 void hostinfo_init(void);
 void clienthost_init(void);
 
+int tool_index(const char *tool)
+{
+    for (int i = 0; tool_callbacks[i].tool; i++) {
+        if (strcmp(tool_callbacks[i].tool, tool) == 0)
+            return i;
+    }
+    return -1;
+}
+
+void exec_tool(int argc, char *argv[])
+{
+    int tidx = tool_index(argv[0]);
+    if (tidx >= 0) {
+        optind = 1;
+        int rc = tool_callbacks[tidx].main_func(argc, argv);
+        exit(rc);
+    }
+}
+
 int main(int argc, char **argv)
 {
     int rc;
@@ -5638,9 +5657,9 @@ int main(int argc, char **argv)
 
     init_peer_hash();
 
-    for (int i = 0; tool_callbacks[i].tool; i++) {
-       if (strcmp(tool_callbacks[i].tool, exe) == 0)
-          return tool_callbacks[i].main_func(argc, argv);
+    int tidx = tool_index(exe);
+    if (tidx >= 0) {
+        return tool_callbacks[tidx].main_func(argc, argv);
     }
 
     /* Initialize the tunables. */

db/config.c

@@ -71,6 +71,7 @@ static int pre_read_option(char *, int);
 static int read_lrl_option(struct dbenv *, char *, struct read_lrl_option_type *, int, int *);
 static void fallback_from_snap_impl();
 
+// clang-format off
 static struct option long_options[] = {
     {"lrl", required_argument, NULL, 0},
     {"repopnewlrl", required_argument, NULL, 0},
@@ -87,7 +88,9 @@ static struct option long_options[] = {
     {"version", no_argument, NULL, 'v'},
     {"insecure", no_argument, &gbl_disable_access_controls, 1},
     {"admin-mode", no_argument, &gbl_server_admin_mode, 1},
+    {"tool", required_argument, NULL, 't'},
     {NULL, 0, NULL, 0}};
+// clang-format on
 
 static const char *help_text =
     "Usage: comdb2 [OPTION]... NAME\n"
@@ -204,12 +207,15 @@ static void read_cmd_line_tunables(struct dbenv *dbenv)
     cmd_line_tunables = NULL;
 }
 
+void exec_tool(int argc, char *argv[]);
+
 int handle_cmdline_options(int argc, char **argv, char **lrlname)
 {
     char *p;
     int c;
     int options_idx;
 
+    // clang-format off
     while ((c = bb_getopt_long(argc, argv, "hv", long_options, &options_idx)) != -1) {
         if (c == 'h') print_usage_and_exit(0);
         if (c == 'v') print_version_and_exit();
@@ -245,8 +251,14 @@ int handle_cmdline_options(int argc, char **argv, char **lrlname)
         case 5: /* pidfile */ write_pidfile(optarg); break;
         case 10: /* dir */ set_dbdir(optarg); break;
         case 11: /* tunable */ add_cmd_line_tunable(optarg); break;
+        case 15: /* tool */
+            exec_tool(argc - (optind - 1), &argv[optind - 1]);
+            logmsg(LOGMSG_WARN, "Invalid tool option, %s\n", optarg);
+            exit(1);
+            break;
         }
     }
+    // clang-format on
     return 0;
 }
 

tests/printlog.test/Makefile

@@ -0,0 +1,8 @@
+ifeq ($(TESTSROOTDIR),)
+  include ../testcase.mk
+else
+  include $(TESTSROOTDIR)/testcase.mk
+endif
+ifeq ($(TEST_TIMEOUT),)
+	export TEST_TIMEOUT=10m
+endif

tests/printlog.test/README

@@ -0,0 +1 @@
+Simple test for basic features of cdb2_printlog

tests/printlog.test/runit

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+bash -n "$0" | exit 1
+
+export debug=1
+export firstnode=""
+
+[[ "$debug" == "1" ]] && set -x
+
+. ${TESTSROOTDIR}/tools/cluster_utils.sh
+. ${TESTSROOTDIR}/tools/ddl.sh
+. ${TESTSROOTDIR}/tools/runit_common.sh
+
+function flush_db
+{
+    if [[ -n "$CLUSTER" ]]; then
+        for node in $CLUSTER ; do 
+            $CDB2SQL_EXE $CDB2_OPTIONS $DBNAME --host $node "exec procedure sys.cmd.send('flush')"
+        done
+    else
+        $CDB2SQL_EXE $CDB2_OPTIONS $DBNAME default "exec procedure sys.cmd.send('flush')"
+    fi
+}
+
+function create_lots_of_btrees
+{
+    j=0
+    while [[ "$j" -lt "100" ]]; do
+        $CDB2SQL_EXE $CDB2_OPTIONS $DBNAME default "create table if not exists x${j}(a int, b blob, c blob, d blob, e blob)"
+        let j=j+1
+    done
+}
+
+function printlog_multiple_files()
+{
+    output=./multiple.txt
+    if [[ -z "$CLUSTER" ]]; then
+        $COMDB2_EXE --tool cdb2_printlog -h $DBDIR/logs -l 2-3  > $output
+    else
+        ssh -n -o StrictHostKeyChecking=no -tt $firstnode "$COMDB2_EXE --tool cdb2_printlog -h $DBDIR/logs -l 2-3" < /dev/null 2>/dev/null > $output
+    fi
+
+    cnt=$(egrep "^\[1" $output | wc -l)
+    [[ "$cnt" == "0" ]] || failexit "printlog_one_file has records from logfile 1"
+    cnt=$(egrep "^\[4" $output | wc -l)
+    [[ "$cnt" == "0" ]] || failexit "printlog_one_file has records from logfile 4"
+    cnt=$(egrep "^\[2" $output | wc -l)
+    [[ "$cnt" > "1000" ]] || failexit "printlog_one_file has less than 1000 records from logfile 2"
+    cnt=$(egrep "^\[3" $output | wc -l)
+    [[ "$cnt" > "1000" ]] || failexit "printlog_one_file has less than 1000 records from logfile 3"
+}
+
+function printlog_one_file()
+{
+    output=./onefile.txt
+    if [[ -z "$CLUSTER" ]]; then
+        $COMDB2_EXE --tool cdb2_printlog -h $DBDIR/logs -l 2  > $output
+    else
+        ssh -n -o StrictHostKeyChecking=no -tt $firstnode "$COMDB2_EXE --tool cdb2_printlog -h $DBDIR/logs -l 2" < /dev/null 2>/dev/null > $output
+    fi
+
+    # Verify output file has no records from logfile 1 or 3
+    cnt=$(egrep "^\[1" $output | wc -l)
+    [[ "$cnt" == "0" ]] || failexit "printlog_one_file has records from logfile 1"
+    cnt=$(egrep "^\[3" $output | wc -l)
+    [[ "$cnt" == "0" ]] || failexit "printlog_one_file has records from logfile 3"
+
+    # Verify output flie has multiple records from logfile 2
+    cnt=$(egrep "^\[2" $output | wc -l)
+    [[ "$cnt" < "1000" ]] && failexit "printlog_one_file has less than 1000 records from logfile 2"
+
+    # Specify non-existent log
+    noexist=./nonexist.txt
+    if [[ -z "$CLUSTER" ]]; then
+        $COMDB2_EXE --tool cdb2_printlog -h $DBDIR/logs -l 100  > $noexist
+    else
+        ssh -n -o StrictHostKeyChecking=no -tt $firstnode "$COMDB2_EXE --tool cdb2_printlog -h $DBDIR/logs -l 100" < /dev/null 2>/dev/null > $noexist
+    fi
+
+    # Just verify an empty file
+    cnt=$(cat $noexist | wc -l)
+    [[ "$cnt" == 0  ]] || failexit "printlog_one_file has records from non-existent logfile"
+}
+
+if [[ -n "$CLUSTER" ]]; then
+    for node in $CLUSTER ; do
+        export firstnode=$node
+        break
+    done
+fi
+
+function printlog_test()
+{
+    printlog_one_file
+    printlog_multiple_files
+    # TODO: emit every record-type (but dont commit)
+}
+
+create_lots_of_btrees
+printlog_test
+flush_db
+

tools/cdb2_printlog/cdb2_printlog.c

@@ -64,19 +64,6 @@ extern pthread_key_t comdb2_open_key;
 
 extern int comdb2ma_init(size_t init_sz, size_t max_cap);
 
-static char *orig_dir;
-#define PRINTLOG_RANGE_DIR "printlog_range"
-void remove_tempdir()
-{
-	if (orig_dir && chdir(orig_dir) == 0) {
-		free(orig_dir);
-		int lrc = system("rm -rf " PRINTLOG_RANGE_DIR);
-        if (lrc) {
-            printf("ERROR: %s:%d system() returns rc = %d\n",__FILE__,__LINE__, lrc);
-        }
-	}
-}
-
 int tool_cdb2_printlog_main(argc, argv)
 	int argc;
 	char *argv[];
@@ -88,6 +75,8 @@ int tool_cdb2_printlog_main(argc, argv)
 	int first = 1;
 	unsigned int start_file = 0;
 	unsigned int start_offset = 0;
+	unsigned int end_file = 0;
+	unsigned int end_offset = 0;
 	char *p;
 	DB *dbp;
 	DBC *dbc;
@@ -124,6 +113,7 @@ int tool_cdb2_printlog_main(argc, argv)
 	while ((ch = getopt(argc, argv, "h:s:gNP:rVl:")) != EOF)
 		switch (ch) {
 		case 'h':
+			chdir(optarg);
 			home = optarg;
 			break;
 		case 'N':
@@ -167,35 +157,14 @@ int tool_cdb2_printlog_main(argc, argv)
 			return (EXIT_SUCCESS);
 		case 'l': {
 			/* Printing only a few of the logs. */
-			int start, end;
 			char *p;
 			if ((p = strchr(optarg, '-')) != NULL) {
 				*p = '\0';
-				start = atoi(optarg);
-				end = atoi(&p[1]);
+				start_file = atoi(optarg);
+				end_file = atoi(&p[1]) + 1;
 			} else {
-				start = end = atoi(optarg);
-			}
-			if (mkdir(PRINTLOG_RANGE_DIR, S_IRWXU) != 0) {
-				fprintf(stderr, "mkdir " PRINTLOG_RANGE_DIR " failed -- %s\n", strerror(errno));
-				exit(1);
-			}
-			orig_dir = getcwd(NULL, 0);
-			atexit(remove_tempdir);
-			if (chdir(PRINTLOG_RANGE_DIR) != 0) {
-				fprintf(stderr, "chdir failed errno:%d %s\n", errno, strerror(errno));
-				exit(1);
-			}
-
-			/* Copy logfiles into this directory. */
-			for (int i = start; i <= end; i++) {
-				char fl[PATH_MAX];
-				snprintf(fl, sizeof(fl), "../" LFNAME, i);
-				if (symlink(fl, fl + 3) != 0) {
-					fprintf(stderr, "symlink %s %s failed errno:%d -- %s\n",
-					    fl, fl + 3, errno, strerror(errno));
-					exit(1);
-				}
+				start_file = atoi(optarg);
+				end_file = start_file + 1;
 			}
 			break;
 		}
@@ -319,6 +288,8 @@ int tool_cdb2_printlog_main(argc, argv)
 
 	memset(&data, 0, sizeof(data));
 	memset(&keydbt, 0, sizeof(keydbt));
+	key.file = 1;
+	key.offset = 0;
 	for (;;) {
 		if (repflag) {
 			ret = dbc->c_get(dbc,
@@ -328,7 +299,7 @@ int tool_cdb2_printlog_main(argc, argv)
 		} else if (first && start_file) {
 			key.file = start_file;
 			key.offset = start_offset;
-			ret = logc->get(logc, &key, &data, DB_SET);
+			ret = logc->get(logc, &key, &data, !start_offset ? DB_FIRST_FILE : DB_SET);
 		} else {
 			ret = logc->get(logc,
 			    &key, &data, rflag ? DB_PREV : DB_NEXT);
@@ -341,6 +312,12 @@ int tool_cdb2_printlog_main(argc, argv)
 			goto shutdown;
 		}
 
+		if (end_file) {
+			if (key.file > end_file || (key.file == end_file && key.offset >= end_offset)) {
+				break;
+			}
+		}
+
 		ret = __db_dispatch(dbenv,
 		    dtab, dtabsize, &data, &key, DB_TXN_PRINT, NULL);