backend: Add scalable PostgreSQL backend

cmd/dcrtime_dumpdb/config.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"path/filepath"
+
+	"github.com/jessevdk/go-flags"
+)
+
+const defaultConfigFilename = "dcrtimed.conf"
+
+var (
+	defaultConfigFile = filepath.Join(defaultHomeDir, defaultConfigFilename)
+	defaultBackend    = "filesystem"
+)
+
+// config defines the configuration options for dcrtime_fsck
+//
+// See loadConfig for details on the configuration load process.
+type config struct {
+	HomeDir           string   `short:"A" long:"appdata" description:"Path to application home directory"`
+	ShowVersion       bool     `short:"V" long:"version" description:"Display version information and exit"`
+	ConfigFile        string   `short:"C" long:"configfile" description:"Path to configuration file"`
+	DataDir           string   `short:"b" long:"datadir" description:"Directory to store data"`
+	LogDir            string   `long:"logdir" description:"Directory to log output."`
+	TestNet           bool     `long:"testnet" description:"Use the test network"`
+	SimNet            bool     `long:"simnet" description:"Use the simulation test network"`
+	Profile           string   `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`
+	CPUProfile        string   `long:"cpuprofile" description:"Write CPU profile to the specified file"`
+	MemProfile        string   `long:"memprofile" description:"Write mem profile to the specified file"`
+	DebugLevel        string   `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`
+	Listeners         []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 49152, testnet: 59152)"`
+	WalletHost        string   `long:"wallethost" description:"Hostname for wallet server"`
+	WalletCert        string   `long:"walletcert" description:"Certificate path for wallet server"`
+	WalletPassphrase  string   `long:"walletpassphrase" description:"Passphrase for wallet server"`
+	Version           string
+	HTTPSCert         string   `long:"httpscert" description:"File containing the https certificate file"`
+	HTTPSKey          string   `long:"httpskey" description:"File containing the https certificate key"`
+	StoreHost         string   `long:"storehost" description:"Enable proxy mode - send requests to the specified ip:port"`
+	StoreCert         string   `long:"storecert" description:"File containing the https certificate file for storehost"`
+	EnableCollections bool     `long:"enablecollections" description:"Allow clients to query collection timestamps."`
+	APITokens         []string `long:"apitoken" description:"Token used to grant access to privileged API resources"`
+	APIVersions       string   `long:"apiversions" description:"Enables API versions on the daemon"`
+	Backend           string   `long:"backend" description:"Sets the cache layer type 'filesystem'/'postgres'"`
+	PostgresHost      string   `long:"postgreshost" description:"Postgres ip:port"`
+	PostgresRootCert  string   `long:"postgresrootcert" description:"File containing the CA certificate for postgres"`
+	PostgresCert      string   `long:"postgrescert" description:"File containing the dcrtimed client certificate for postgres"`
+	PostgresKey       string   `long:"postgreskey" description:"File containing the dcrtimed client certificate key for postgres"`
+}
+
+// loadConfig initializes and parses the config using a config file
+func loadConfig() (*config, error) {
+	// Default config.
+	cfg := config{
+		Backend: defaultBackend,
+	}
+
+	err := flags.IniParse(defaultConfigFile, &cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}

cmd/dcrtime_dumpdb/dcrtime_dumpdb.go

@@ -8,7 +8,9 @@ import (
 
 	"github.com/decred/dcrd/chaincfg/v2"
 	"github.com/decred/dcrd/dcrutil/v2"
+	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/backend/filesystem"
+	"github.com/decred/dcrtime/dcrtimed/backend/postgres"
 )
 
 var (
@@ -24,18 +26,9 @@ var (
 func _main() error {
 	flag.Parse()
 
-	if *restore {
-		if *destination == "" {
-			return fmt.Errorf("-destination must be set")
-		}
-
-		fs, err := filesystem.NewRestore(*destination)
-		if err != nil {
-			return err
-		}
-		defer fs.Close()
-
-		return fs.Restore(os.Stdin, true, *destination)
+	loadedCfg, err := loadConfig()
+	if err != nil {
+		return fmt.Errorf("Could not load configuration file: %v", err)
 	}
 
 	root := *fsRoot
@@ -48,17 +41,45 @@ func _main() error {
 		}
 	}
 
-	// Dump
-
-	fmt.Printf("=== Root: %v\n", root)
-
-	fs, err := filesystem.NewDump(root)
+	var b backend.Backend
+	switch (*loadedCfg).Backend {
+	case "filesystem":
+		if *restore {
+			if *destination == "" {
+				return fmt.Errorf("-destination must be set")
+			}
+			b, err = filesystem.NewRestore(*destination)
+			break
+		}
+		b, err = filesystem.NewDump(root)
+		if !*dumpJSON {
+			fmt.Printf("=== Root: %v\n", root)
+		}
+	case "postgres":
+		var net string
+		switch loadedCfg.TestNet {
+		case true:
+			net = "testnet"
+		default:
+			net = "mainnet"
+		}
+		b, err = postgres.NewDB(loadedCfg.PostgresHost,
+			net,
+			loadedCfg.PostgresRootCert,
+			loadedCfg.PostgresCert,
+			loadedCfg.PostgresKey)
+	default:
+		err = fmt.Errorf("Unsupported backend type: %v", (*loadedCfg).Backend)
+	}
 	if err != nil {
 		return err
 	}
-	defer fs.Close()
+	defer b.Close()
 
-	return fs.Dump(os.Stdout, !*dumpJSON)
+	if *restore {
+		return b.Restore(os.Stdin, true, *destination)
+	}
+	return b.Dump(os.Stdout, !*dumpJSON)
 }
 
 func main() {

cmd/dcrtime_fsck/README.md

@@ -4,6 +4,10 @@ dcrtime_fsck
 The filesystem backend can under rare circumstances become incoherent. This
 tool iterates over all timestamp directories and corrects known failures.
 
+PostgreSQL backend also supported, `dcrtime_fsck` digs in `dcrtimed.conf`
+file to determine which backend to run, and uses the provided db params 
+in case postgres backend is running.
+
 ## Flags
 
 ```

cmd/dcrtime_fsck/config.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"path/filepath"
+
+	"github.com/jessevdk/go-flags"
+)
+
+const defaultConfigFilename = "dcrtimed.conf"
+
+var (
+	defaultConfigFile = filepath.Join(defaultHomeDir, defaultConfigFilename)
+	defaultBackend    = "filesystem"
+)
+
+// config defines the configuration options for dcrtime_fsck
+//
+// See loadConfig for details on the configuration load process.
+type config struct {
+	HomeDir           string   `short:"A" long:"appdata" description:"Path to application home directory"`
+	ShowVersion       bool     `short:"V" long:"version" description:"Display version information and exit"`
+	ConfigFile        string   `short:"C" long:"configfile" description:"Path to configuration file"`
+	DataDir           string   `short:"b" long:"datadir" description:"Directory to store data"`
+	LogDir            string   `long:"logdir" description:"Directory to log output."`
+	TestNet           bool     `long:"testnet" description:"Use the test network"`
+	SimNet            bool     `long:"simnet" description:"Use the simulation test network"`
+	Profile           string   `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`
+	CPUProfile        string   `long:"cpuprofile" description:"Write CPU profile to the specified file"`
+	MemProfile        string   `long:"memprofile" description:"Write mem profile to the specified file"`
+	DebugLevel        string   `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`
+	Listeners         []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 49152, testnet: 59152)"`
+	WalletHost        string   `long:"wallethost" description:"Hostname for wallet server"`
+	WalletCert        string   `long:"walletcert" description:"Certificate path for wallet server"`
+	WalletPassphrase  string   `long:"walletpassphrase" description:"Passphrase for wallet server"`
+	Version           string
+	HTTPSCert         string   `long:"httpscert" description:"File containing the https certificate file"`
+	HTTPSKey          string   `long:"httpskey" description:"File containing the https certificate key"`
+	StoreHost         string   `long:"storehost" description:"Enable proxy mode - send requests to the specified ip:port"`
+	StoreCert         string   `long:"storecert" description:"File containing the https certificate file for storehost"`
+	EnableCollections bool     `long:"enablecollections" description:"Allow clients to query collection timestamps."`
+	APITokens         []string `long:"apitoken" description:"Token used to grant access to privileged API resources"`
+	APIVersions       string   `long:"apiversions" description:"Enables API versions on the daemon"`
+	Backend           string   `long:"backend" description:"Sets the cache layer type 'filesystem'/'postgres'"`
+	PostgresHost      string   `long:"postgreshost" description:"Postgres ip:port"`
+	PostgresRootCert  string   `long:"postgresrootcert" description:"File containing the CA certificate for postgres"`
+	PostgresCert      string   `long:"postgrescert" description:"File containing the dcrtimed client certificate for postgres"`
+	PostgresKey       string   `long:"postgreskey" description:"File containing the dcrtimed client certificate key for postgres"`
+}
+
+// loadConfig initializes and parses the config using a config file
+func loadConfig() (*config, error) {
+	// Default config.
+	cfg := config{
+		Backend: defaultBackend,
+	}
+
+	err := flags.IniParse(defaultConfigFile, &cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}

cmd/dcrtime_fsck/dcrtime_fsck.go

@@ -11,6 +11,7 @@ import (
 	"github.com/decred/dcrd/dcrutil/v2"
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/backend/filesystem"
+	"github.com/decred/dcrtime/dcrtimed/backend/postgres"
 )
 
 var (
@@ -28,6 +29,11 @@ var (
 func _main() error {
 	flag.Parse()
 
+	loadedCfg, err := loadConfig()
+	if err != nil {
+		return fmt.Errorf("Could not load configuration file: %v", err)
+	}
+
 	root := *fsRoot
 	if root == "" {
 		root = filepath.Join(defaultHomeDir, "data")
@@ -52,13 +58,32 @@ func _main() error {
 
 	fmt.Printf("=== Root: %v\n", root)
 
-	fs, err := filesystem.NewDump(root)
+	var b backend.Backend
+	switch (*loadedCfg).Backend {
+	case "filesystem":
+		b, err = filesystem.NewDump(root)
+	case "postgres":
+		var net string
+		switch loadedCfg.TestNet {
+		case true:
+			net = "testnet"
+		default:
+			net = "mainnet"
+		}
+		b, err = postgres.NewDB(loadedCfg.PostgresHost,
+			net,
+			loadedCfg.PostgresRootCert,
+			loadedCfg.PostgresCert,
+			loadedCfg.PostgresKey)
+	default:
+		err = fmt.Errorf("Unsupported backend type: %v", (*loadedCfg).Backend)
+	}
 	if err != nil {
 		return err
 	}
-	defer fs.Close()
+	defer b.Close()
 
-	return fs.Fsck(&backend.FsckOptions{
+	return b.Fsck(&backend.FsckOptions{
 		Verbose:     *verbose,
 		PrintHashes: *printHashes,
 		Fix:         *fix,

cmd/dcrtime_timestamp/dcrtime_timestamp.go

@@ -28,7 +28,7 @@ func _main() error {
 			continue
 		}
 
-		// Try timestam second
+		// Try timestamp second
 		timestamp, err := time.Parse(fStr, a)
 		if err == nil {
 			fmt.Printf("%v\n", timestamp.Unix())

dcrtimed/backend/filesystem/filesystem.go

@@ -934,33 +934,34 @@ func (fs *FileSystem) LastAnchor() (*backend.LastAnchorResult, error) {
 	var fr *backend.FlushRecord
 	var me backend.LastAnchorResult
 	payload, err := db.Get([]byte(flushedKey), nil)
-	if err == nil {
-		fr, err = DecodeFlushRecord(payload)
-		if err != nil {
-			return &me, err
-		}
-		me.Tx = fr.Tx
+	if err != nil {
+		return &backend.LastAnchorResult{}, err
+	}
 
-		// Close db conection as we may
-		// write & update it
-		db.Close()
+	fr, err = DecodeFlushRecord(payload)
+	if err != nil {
+		return &me, err
+	}
+	me.Tx = fr.Tx
 
-		// Lookup anchored tx info,
-		// and update db if info changed.
-		txWalletInfo, err := fs.lazyFlush(flushedTs, fr)
+	// Close db conection as we may
+	// write & update it
+	db.Close()
 
-		// If no error, or no enough confirmations
-		// err continue, else return err.
-		if err != nil && err != errNotEnoughConfirmation {
-			return &backend.LastAnchorResult{}, err
-		}
-		me.ChainTimestamp = fr.ChainTimestamp
-		me.BlockHash = txWalletInfo.BlockHash.String()
-		me.BlockHeight = txWalletInfo.BlockHeight
-		return &me, nil
+	// Lookup anchored tx info,
+	// and update db if info changed.
+	txWalletInfo, err := fs.lazyFlush(flushedTs, fr)
+
+	// If no error, or no enough confirmations
+	// err continue, else return err.
+	if err != nil && err != errNotEnoughConfirmation {
+		return &backend.LastAnchorResult{}, err
 	}
 
-	return &backend.LastAnchorResult{}, err
+	me.ChainTimestamp = fr.ChainTimestamp
+	me.BlockHash = txWalletInfo.BlockHash.String()
+	me.BlockHeight = txWalletInfo.BlockHeight
+	return &me, nil
 }
 
 // GetBalance provides the balance of the wallet and satisfies the

dcrtimed/backend/filesystem/filesystem_test.go

@@ -178,7 +178,7 @@ func TestGetDigests(t *testing.T) {
 			gr.ErrorCode != foundGlobal) {
 			t.Fatalf("invalid digest got %x want %x ErrorCode "+
 				"got %v want %v", gr.Digest[:], hashes[i][:],
-				gr.ErrorCode, foundLocal)
+				gr.ErrorCode, foundGlobal)
 		}
 		if i >= count && gr.ErrorCode != backend.ErrorNotFound {
 			t.Fatalf("invalid ErrorCode got %x want %x",