From 572090599b9694131ac875272c6b2cb122c5fd9c Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 4 Aug 2020 18:02:28 +0300
Subject: [PATCH 01/66] last anchor cleanup

---
 dcrtimed/backend/filesystem/filesystem.go | 49 ++++++++++++-----------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/dcrtimed/backend/filesystem/filesystem.go b/dcrtimed/backend/filesystem/filesystem.go
index b871c2b..25c2ab4 100644
--- a/dcrtimed/backend/filesystem/filesystem.go
+++ b/dcrtimed/backend/filesystem/filesystem.go
@@ -711,7 +711,7 @@ func (fs *FileSystem) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResul
 	}
 	defer current.Close()
 
-	// Create a Put batch for provided digests.  Obviously looking things
+	// Create a Put batch for provided digests. Obviously looking things
 	// up without a lock will make the lookups racy however when a
 	// container is committed it is locked and therefore overall an atomic
 	// operation.
@@ -754,7 +754,7 @@ func (fs *FileSystem) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResul
 			continue
 		}
 
-		// Lookup in previous not flushed dirs
+		// Lookup in previous not flushed dirs.
 		// Get Dirs.
 		files, err := ioutil.ReadDir(fs.root)
 		if err != nil {
@@ -934,33 +934,34 @@ func (fs *FileSystem) LastAnchor() (*backend.LastAnchorResult, error) {
 	var fr *backend.FlushRecord
 	var me backend.LastAnchorResult
 	payload, err := db.Get([]byte(flushedKey), nil)
-	if err == nil {
-		fr, err = DecodeFlushRecord(payload)
-		if err != nil {
-			return &me, err
-		}
-		me.Tx = fr.Tx
+	if err != nil {
+		return &backend.LastAnchorResult{}, err
+	}
 
-		// Close db conection as we may
-		// write & update it
-		db.Close()
+	fr, err = DecodeFlushRecord(payload)
+	if err != nil {
+		return &me, err
+	}
+	me.Tx = fr.Tx
 
-		// Lookup anchored tx info,
-		// and update db if info changed.
-		txWalletInfo, err := fs.lazyFlush(flushedTs, fr)
+	// Close db conection as we may
+	// write & update it
+	db.Close()
 
-		// If no error, or no enough confirmations
-		// err continue, else return err.
-		if err != nil && err != errNotEnoughConfirmation {
-			return &backend.LastAnchorResult{}, err
-		}
-		me.ChainTimestamp = fr.ChainTimestamp
-		me.BlockHash = txWalletInfo.BlockHash.String()
-		me.BlockHeight = txWalletInfo.BlockHeight
-		return &me, nil
+	// Lookup anchored tx info,
+	// and update db if info changed.
+	txWalletInfo, err := fs.lazyFlush(flushedTs, fr)
+
+	// If no error, or no enough confirmations
+	// err continue, else return err.
+	if err != nil && err != errNotEnoughConfirmation {
+		return &backend.LastAnchorResult{}, err
 	}
 
-	return &backend.LastAnchorResult{}, err
+	me.ChainTimestamp = fr.ChainTimestamp
+	me.BlockHash = txWalletInfo.BlockHash.String()
+	me.BlockHeight = txWalletInfo.BlockHeight
+	return &me, nil
 }
 
 // GetBalance provides the balance of the wallet and satisfies the

From 5177c5776686f8ac8731c06d94c7a7780d3ab1e8 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 5 Aug 2020 21:48:07 +0300
Subject: [PATCH 02/66] add schema.md to describe sql tables

---
 dcrtimed/backend/relational/schema.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 dcrtimed/backend/relational/schema.md

diff --git a/dcrtimed/backend/relational/schema.md b/dcrtimed/backend/relational/schema.md
new file mode 100644
index 0000000..6eed340
--- /dev/null
+++ b/dcrtimed/backend/relational/schema.md
@@ -0,0 +1,13 @@
+# Dcrtime Relational Database
+
+This document describes the SQL relational tables used in dcrtime.
+
+We have two tables storing all timestamped digests information, `records` 
+and `anchors` where the first is used to store all timestamped digest and 
+the second for storing anchors information, where each succesful anchor 
+will result in new entry. Each `record` which was included in an `anchor` 
+will be connected to the corresponding entry in the anchors table using 
+the col `anchor_merkle` which defined as forgein key & indexed in 
+records table, below you find the detailed description of the two table:
+
+

From bca63416e63354b567c639471f55834c4e87f222 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 5 Aug 2020 23:08:02 +0300
Subject: [PATCH 03/66] add records table description

---
 dcrtimed/backend/filesystem/filesystem.go |  2 +-
 dcrtimed/backend/relational/schema.md     | 21 ++++++++++++++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/dcrtimed/backend/filesystem/filesystem.go b/dcrtimed/backend/filesystem/filesystem.go
index 25c2ab4..d11327e 100644
--- a/dcrtimed/backend/filesystem/filesystem.go
+++ b/dcrtimed/backend/filesystem/filesystem.go
@@ -754,7 +754,7 @@ func (fs *FileSystem) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResul
 			continue
 		}
 
-		// Lookup in previous not flushed dirs.
+		// Lookup in previous not flushed dirs
 		// Get Dirs.
 		files, err := ioutil.ReadDir(fs.root)
 		if err != nil {
diff --git a/dcrtimed/backend/relational/schema.md b/dcrtimed/backend/relational/schema.md
index 6eed340..a4b647e 100644
--- a/dcrtimed/backend/relational/schema.md
+++ b/dcrtimed/backend/relational/schema.md
@@ -3,11 +3,22 @@
 This document describes the SQL relational tables used in dcrtime.
 
 We have two tables storing all timestamped digests information, `records` 
-and `anchors` where the first is used to store all timestamped digest and 
-the second for storing anchors information, where each succesful anchor 
-will result in new entry. Each `record` which was included in an `anchor` 
-will be connected to the corresponding entry in the anchors table using 
-the col `anchor_merkle` which defined as forgein key & indexed in 
+and `anchors` where the first is the first is key/value like used to store all 
+timestamped digests and the second for storing anchors information, where each 
+succesful anchor will result in a new entry. Each `record` which was included 
+in an `anchor` will be connected to the corresponding entry in the anchors 
+table using the col `anchor_merkle` which defined as forgein key & indexed in 
 records table, below you find the detailed description of the two table:
 
 
+### Tables
+
+**Records:**
+| Col Name             | Type              | Not Null | P. Key | F. Key | Indexed | Description                                                    |
+|----------------------|-------------------|----------|--------|--------|---------|----------------------------------------------------------------|
+| key                  | serial            | x        | x      |        |         | Auto incremented  identifier                                   |
+| collection_timestamp | text              | x        |        |        | x       | Unix timestamp of collection                                   |
+| digest               | bytea             | x        |        |        |         | Timestamped digest                                             |
+| anchor_merkle        | char. varying(64) |          |        | x      | x       | Merkle root of corresponding anchor,   nil if not anchored yet |
+
+

From 33df8746af82501ffa7b659e1da2c9a388d47d77 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 5 Aug 2020 23:22:51 +0300
Subject: [PATCH 04/66] add anchors table

---
 dcrtimed/backend/relational/schema.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/dcrtimed/backend/relational/schema.md b/dcrtimed/backend/relational/schema.md
index a4b647e..7c42abc 100644
--- a/dcrtimed/backend/relational/schema.md
+++ b/dcrtimed/backend/relational/schema.md
@@ -19,6 +19,14 @@ records table, below you find the detailed description of the two table:
 | key                  | serial            | x        | x      |        |         | Auto incremented  identifier                                   |
 | collection_timestamp | text              | x        |        |        | x       | Unix timestamp of collection                                   |
 | digest               | bytea             | x        |        |        |         | Timestamped digest                                             |
-| anchor_merkle        | char. varying(64) |          |        | x      | x       | Merkle root of corresponding anchor,   nil if not anchored yet |
+| anchor_merkle        | char. varying(64) |          |        | x      | x       | Merkle root of corresponding anchor - linking to anchors table, nil if not anchored yet |
 
+**Anchors:**
+| Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Description                     |
+|------------------|-------------------|----------|--------|--------|---------|---------------------------------|
+| merkle           | char. varying(64) | x        | x      |        |         | Anchor merkle root              |
+| hashes           | text[]            | x        |        |        |         | Anchored hashes                 |
+| tx_hash          | text              |          |        |        |         | Anchor tx hash                  |
+| chain_timestamp  | bigint            |          |        |        | x       | Anchor timestamp on blockchain  |
+| flush_timestamp  | bigint            |          |        |        | x       | When anchor actually  flushed   |
 

From c566934b2424bdb56d225f4d610daa8a3f4848ad Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 5 Aug 2020 23:32:31 +0300
Subject: [PATCH 05/66] opps

---
 dcrtimed/backend/relational/schema.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dcrtimed/backend/relational/schema.md b/dcrtimed/backend/relational/schema.md
index 7c42abc..e809ceb 100644
--- a/dcrtimed/backend/relational/schema.md
+++ b/dcrtimed/backend/relational/schema.md
@@ -3,7 +3,7 @@
 This document describes the SQL relational tables used in dcrtime.
 
 We have two tables storing all timestamped digests information, `records` 
-and `anchors` where the first is the first is key/value like used to store all 
+and `anchors` where the first is key/value like table used to store all 
 timestamped digests and the second for storing anchors information, where each 
 succesful anchor will result in a new entry. Each `record` which was included 
 in an `anchor` will be connected to the corresponding entry in the anchors 

From b9dae825d21c9b0a9307c7ff98b89ffc55c7e924 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 5 Aug 2020 23:50:04 +0300
Subject: [PATCH 06/66] typo

---
 dcrtimed/backend/relational/schema.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dcrtimed/backend/relational/schema.md b/dcrtimed/backend/relational/schema.md
index e809ceb..7033ae0 100644
--- a/dcrtimed/backend/relational/schema.md
+++ b/dcrtimed/backend/relational/schema.md
@@ -8,7 +8,7 @@ timestamped digests and the second for storing anchors information, where each
 succesful anchor will result in a new entry. Each `record` which was included 
 in an `anchor` will be connected to the corresponding entry in the anchors 
 table using the col `anchor_merkle` which defined as forgein key & indexed in 
-records table, below you find the detailed description of the two table:
+records table, below you find the detailed description of the two tables:
 
 
 ### Tables

From 2533029a1c9ddf207dbe80edebb5eed0b15e96d1 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Fri, 7 Aug 2020 11:12:07 +0300
Subject: [PATCH 07/66] add postgres package & struct

---
 dcrtimed/backend/postgres/postgres.go         | 35 +++++++++++++++++++
 .../{relational => postgres}/schema.md        |  0
 2 files changed, 35 insertions(+)
 create mode 100644 dcrtimed/backend/postgres/postgres.go
 rename dcrtimed/backend/{relational => postgres}/schema.md (100%)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
new file mode 100644
index 0000000..4eb02b3
--- /dev/null
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -0,0 +1,35 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package postgres
+
+import (
+	"sync"
+	"time"
+)
+
+var (
+	_ backend.Backend = (*Postgres)(nil)
+
+	// duration and flushSchedule must match or bad things will happen. By
+	// matching we mean both are hourly or every so many minutes.
+	//
+	// Seconds Minutes Hours Days Months DayOfWeek
+	flushSchedule = "10 0 * * * *" // On the hour + 10 seconds
+	duration      = time.Hour      // Default how often we combine digests
+)
+
+// Postgres is a postgreSQL implementation of a backend, it stores all uploaded
+// digests in records table, on flush it stores all anchor info as well and
+// link all anchored records with the corresponding anchor.
+type Postgres struct {
+	sync.RWMutex
+
+	cron *cron.Cron // Scheduler for periodic tasks
+	// db       *leveldb.DB   // Global database [hash]timestamp
+	duration time.Duration // How often we combine digests
+	commit   uint          // Current version, incremented during flush
+
+	wallet *dcrtimewallet.DcrtimeWallet // Wallet context.
+}
diff --git a/dcrtimed/backend/relational/schema.md b/dcrtimed/backend/postgres/schema.md
similarity index 100%
rename from dcrtimed/backend/relational/schema.md
rename to dcrtimed/backend/postgres/schema.md

From 52ec32f6cbf9c8f4313afc35f46d897d98a644d0 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Fri, 7 Aug 2020 22:03:37 +0300
Subject: [PATCH 08/66] add backend config & interface funcs signatures

---
 dcrtimed/backend/postgres/postgres.go | 58 +++++++++++++++++++++++++++
 dcrtimed/config.go                    |  6 +++
 2 files changed, 64 insertions(+)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 4eb02b3..ab2d495 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -5,8 +5,14 @@
 package postgres
 
 import (
+	"crypto/sha256"
+	"os"
 	"sync"
 	"time"
+
+	"github.com/decred/dcrtime/dcrtimed/backend"
+	"github.com/decred/dcrtime/dcrtimed/dcrtimewallet"
+	"github.com/robfig/cron"
 )
 
 var (
@@ -33,3 +39,55 @@ type Postgres struct {
 
 	wallet *dcrtimewallet.DcrtimeWallet // Wallet context.
 }
+
+// Return timestamp information for given digests.
+func (pg *Postgres) Get([][sha256.Size]byte) ([]backend.GetResult, error) {
+	return nil, nil
+}
+
+// Return all hashes for given timestamps.
+func (pg *Postgres) GetTimestamps([]int64) ([]backend.TimestampResult, error) {
+	return nil, nil
+}
+
+// Store hashes and return timestamp and associated errors.  Put is
+// allowed to return transient errors.
+func (pg *Postgres) Put([][sha256.Size]byte) (int64, []backend.PutResult, error) {
+	return 0, nil, nil
+}
+
+// Close performs cleanup of the backend.
+func (pg *Postgres) Close() {
+}
+
+// Dump dumps database to the provided file descriptor. If the
+// human flag is set to true it pretty prints the database content
+// otherwise it dumps a JSON stream.
+func (pg *Postgres) Dump(*os.File, bool) error {
+	return nil
+}
+
+// Restore recreates the the database from the provided file
+// descriptor. The verbose flag is set to true to indicate that this
+// call may parint to stdout. The provided string describes the target
+// location and is implementation specific.
+func (pg *Postgres) Restore(*os.File, bool, string) error {
+	return nil
+}
+
+// Fsck walks all data and verifies its integrity. In addition it
+// verifies anchored timestamps' existence on the blockchain.
+func (pg *Postgres) Fsck(*backend.FsckOptions) error {
+	return nil
+}
+
+// GetBalance retrieves balance information for the wallet
+// backing this instance
+func (pg *Postgres) GetBalance() (*backend.GetBalanceResult, error) {
+	return nil, nil
+}
+
+// LastAnchor retrieves last successful anchor details
+func (pg *Postgres) LastAnchor() (*backend.LastAnchorResult, error) {
+	return nil, nil
+}
diff --git a/dcrtimed/config.go b/dcrtimed/config.go
index 528b81e..51514b3 100644
--- a/dcrtimed/config.go
+++ b/dcrtimed/config.go
@@ -40,6 +40,7 @@ var (
 	defaultHTTPSCertFile = filepath.Join(defaultHomeDir, "https.cert")
 	defaultLogDir        = filepath.Join(defaultHomeDir, defaultLogDirname)
 	defaultAPIVersions   = fmt.Sprintf("%v,%v", v1.APIVersion, v2.APIVersion)
+	defaultBackend       = "filesystem"
 )
 
 // runServiceCommand is only set to a real function on Windows.  It is used
@@ -73,6 +74,7 @@ type config struct {
 	EnableCollections bool     `long:"enablecollections" description:"Allow clients to query collection timestamps."`
 	APITokens         []string `long:"apitoken" description:"Token used to grant access to privileged API resources"`
 	APIVersions       string   `long:"apiversions" description:"Enables API versions on the daemon"`
+	Backend           string   `long:"backend" description:"Sets the cache layer type 'filesystem'/'postgres'"`
 }
 
 // serviceOptions defines the configuration options for the daemon as a service
@@ -278,6 +280,7 @@ func loadConfig() (*config, []string, error) {
 		HTTPSCert:   defaultHTTPSCertFile,
 		Version:     version(),
 		APIVersions: defaultAPIVersions,
+		Backend:     defaultBackend,
 	}
 
 	// Service options which are only added on Windows.
@@ -543,6 +546,9 @@ func loadConfig() (*config, []string, error) {
 		cfg.APITokens = validTokens
 	}
 
+	fmt.Println(11111111111)
+	fmt.Println(cfg.Backend)
+
 	// Warn about missing config file only after all other configuration is
 	// done.  This prevents the warning on help messages and invalid
 	// options.  Note this should go directly before the return.

From a3bbf176e77bc274edd5c3a3beb5ca45d5963d7b Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 8 Aug 2020 00:08:56 +0300
Subject: [PATCH 09/66] setup postgres when configured

---
 dcrtimed/backend/postgres/log.go | 25 +++++++++++++++++++++++++
 dcrtimed/config.go               |  3 ---
 dcrtimed/dcrtimed.go             | 28 +++++++++++++++++++---------
 3 files changed, 44 insertions(+), 12 deletions(-)
 create mode 100644 dcrtimed/backend/postgres/log.go

diff --git a/dcrtimed/backend/postgres/log.go b/dcrtimed/backend/postgres/log.go
new file mode 100644
index 0000000..6eeb2e2
--- /dev/null
+++ b/dcrtimed/backend/postgres/log.go
@@ -0,0 +1,25 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package postgres
+
+import "github.com/decred/slog"
+
+// log is a logger that is initialized with no output filters.  This
+// means the package will not perform any logging by default until the caller
+// requests it.
+var log = slog.Disabled
+
+// DisableLog disables all library log output.  Logging output is disabled
+// by default until either UseLogger or SetLogWriter are called.
+func DisableLog() {
+	log = slog.Disabled
+}
+
+// UseLogger uses a specified Logger to output package logging info.
+// This should be used in preference to SetLogWriter if the caller is also
+// using slog.
+func UseLogger(logger slog.Logger) {
+	log = logger
+}
diff --git a/dcrtimed/config.go b/dcrtimed/config.go
index 51514b3..2c2d435 100644
--- a/dcrtimed/config.go
+++ b/dcrtimed/config.go
@@ -546,9 +546,6 @@ func loadConfig() (*config, []string, error) {
 		cfg.APITokens = validTokens
 	}
 
-	fmt.Println(11111111111)
-	fmt.Println(cfg.Backend)
-
 	// Warn about missing config file only after all other configuration is
 	// done.  This prevents the warning on help messages and invalid
 	// options.  Note this should go directly before the return.
diff --git a/dcrtimed/dcrtimed.go b/dcrtimed/dcrtimed.go
index 1135fa9..be45832 100644
--- a/dcrtimed/dcrtimed.go
+++ b/dcrtimed/dcrtimed.go
@@ -26,6 +26,7 @@ import (
 	v2 "github.com/decred/dcrtime/api/v2"
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/backend/filesystem"
+	"github.com/decred/dcrtime/dcrtimed/backend/postgres"
 	"github.com/decred/dcrtime/util"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
@@ -1364,15 +1365,24 @@ func _main() error {
 		}
 	} else {
 		// Setup backend.
-		filesystem.UseLogger(fsbeLog)
-		b, err := filesystem.New(loadedCfg.DataDir,
-			loadedCfg.WalletCert,
-			loadedCfg.WalletHost,
-			loadedCfg.EnableCollections,
-			[]byte(loadedCfg.WalletPassphrase))
-
-		if err != nil {
-			return err
+		var b backend.Backend
+		switch loadedCfg.Backend {
+		case "filesystem":
+			filesystem.UseLogger(fsbeLog)
+			b, err = filesystem.New(loadedCfg.DataDir,
+				loadedCfg.WalletCert,
+				loadedCfg.WalletHost,
+				loadedCfg.EnableCollections,
+				[]byte(loadedCfg.WalletPassphrase))
+			if err != nil {
+				return err
+			}
+		case "postgres":
+			postgres.UseLogger(fsbeLog)
+			b, err = postgres.New()
+			if err != nil {
+				return nil
+			}
 		}
 
 		d.backend = b

From 62798d4bb2587daa8fa72611fd8730072319793c Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 8 Aug 2020 17:29:32 +0300
Subject: [PATCH 10/66] add postgres connection info to config and validate
 when postgres is used as BE

---
 dcrtimed/backend/filesystem/filesystem.go |  2 +-
 dcrtimed/backend/postgres/postgres.go     | 96 ++++++++++++++++++++++-
 dcrtimed/config.go                        | 51 ++++++++++++
 dcrtimed/sample-dcrtimed.conf             | 15 ++++
 go.mod                                    |  1 +
 go.sum                                    |  2 +
 6 files changed, 164 insertions(+), 3 deletions(-)

diff --git a/dcrtimed/backend/filesystem/filesystem.go b/dcrtimed/backend/filesystem/filesystem.go
index d11327e..3f8d07c 100644
--- a/dcrtimed/backend/filesystem/filesystem.go
+++ b/dcrtimed/backend/filesystem/filesystem.go
@@ -711,7 +711,7 @@ func (fs *FileSystem) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResul
 	}
 	defer current.Close()
 
-	// Create a Put batch for provided digests. Obviously looking things
+	// Create a Put batch for provided digests.  Obviously looking things
 	// up without a lock will make the lookups racy however when a
 	// container is committed it is locked and therefore overall an atomic
 	// operation.
diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index ab2d495..dfea992 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -6,7 +6,11 @@ package postgres
 
 import (
 	"crypto/sha256"
+	"database/sql"
+	"fmt"
+	"net/url"
 	"os"
+	"path/filepath"
 	"sync"
 	"time"
 
@@ -32,12 +36,18 @@ var (
 type Postgres struct {
 	sync.RWMutex
 
-	cron *cron.Cron // Scheduler for periodic tasks
-	// db       *leveldb.DB   // Global database [hash]timestamp
+	cron     *cron.Cron    // Scheduler for periodic tasks
+	db       *sql.DB       // Postgres database
 	duration time.Duration // How often we combine digests
 	commit   uint          // Current version, incremented during flush
 
+	enableCollections bool // Set to true to enable collection query
+
 	wallet *dcrtimewallet.DcrtimeWallet // Wallet context.
+
+	// testing only entries
+	myNow   func() time.Time // Override time.Now()
+	testing bool             // Enabled during test
 }
 
 // Return timestamp information for given digests.
@@ -91,3 +101,85 @@ func (pg *Postgres) GetBalance() (*backend.GetBalanceResult, error) {
 func (pg *Postgres) LastAnchor() (*backend.LastAnchorResult, error) {
 	return nil, nil
 }
+
+func buildQueryString(rootCert, cert, key string) string {
+	v := url.Values{}
+	v.Set("sslmode", "require")
+	v.Set("sslrootcert", filepath.Clean(rootCert))
+	v.Set("sslcert", filepath.Join(cert))
+	v.Set("sslkey", filepath.Join(key))
+	return v.Encode()
+}
+
+// internalNew creates the Pstgres context but does not launch background
+// bits.  This is used by the test packages.
+func internalNew(user, host, net, rootCert, cert, key string) (*Postgres, error) {
+	// Connect to database
+	dbName := net + "_dcrtime"
+	h := "postgresql://" + user + "@" + host + "/" + dbName
+	u, err := url.Parse(h)
+	if err != nil {
+		return nil, fmt.Errorf("parse url '%v': %v", h, err)
+	}
+
+	qs := buildQueryString(rootCert, cert, key)
+	addr := u.String() + "?" + qs
+
+	db, err := sql.Open("postgres", addr)
+	if err != nil {
+		return nil, fmt.Errorf("connect to database '%v': %v", addr, err)
+	}
+
+	pg := &Postgres{
+		cron:     cron.New(),
+		db:       db,
+		duration: duration,
+		myNow:    time.Now,
+	}
+
+	return pg, nil
+}
+
+// New creates a new backend instance.  The caller should issue a Close once
+// the Postgres backend is no longer needed.
+func New(user, host, net, rootCert, cert, key, walletCert, walletHost string, enableCollections bool, walletPassphrase []byte) (*Postgres, error) {
+	// XXX log more stuff
+	log.Tracef("New: %v %v %v %v %v %v", user, host, net, rootCert, cert, key)
+
+	pg, err := internalNew(user, host, net, rootCert, cert, key)
+	if err != nil {
+		return nil, err
+	}
+	pg.enableCollections = enableCollections
+
+	// Runtime bits
+	dcrtimewallet.UseLogger(log)
+	pg.wallet, err = dcrtimewallet.New(walletCert, walletHost, walletPassphrase)
+	if err != nil {
+		return nil, err
+	}
+
+	// Flushing backend reconciles uncommitted work to the global database.
+	//start := time.Now()
+	//flushed, err := pg.doFlush()
+	//end := time.Since(start)
+	//if err != nil {
+	//return nil, err
+	//}
+
+	//if flushed != 0 {
+	//log.Infof("Startup flusher: directories %v in %v", flushed, end)
+	//}
+
+	// Launch cron.
+	//err = pg.cron.AddFunc(flushSchedule, func() {
+	//pg.flusher()
+	///})
+	//if err != nil {
+	//return nil, err
+	//}
+
+	//pg.cron.Start()
+
+	return pg, nil
+}
diff --git a/dcrtimed/config.go b/dcrtimed/config.go
index 2c2d435..89c893d 100644
--- a/dcrtimed/config.go
+++ b/dcrtimed/config.go
@@ -6,7 +6,11 @@
 package main
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
@@ -75,6 +79,10 @@ type config struct {
 	APITokens         []string `long:"apitoken" description:"Token used to grant access to privileged API resources"`
 	APIVersions       string   `long:"apiversions" description:"Enables API versions on the daemon"`
 	Backend           string   `long:"backend" description:"Sets the cache layer type 'filesystem'/'postgres'"`
+	PostgresHost      string   `long:"postgreshost" description:"Postgres ip:port"`
+	PostgresRootCert  string   `long:"postgresrootcert" description:"File containing the CA certificate for postgres"`
+	PostgresCert      string   `long:"postgrescert" description:"File containing the dcrtimed client certificate for postgres"`
+	PostgresKey       string   `long:"postgreskey" description:"File containing the dcrtimed client certificate key for postgres"`
 }
 
 // serviceOptions defines the configuration options for the daemon as a service
@@ -546,6 +554,49 @@ func loadConfig() (*config, []string, error) {
 		cfg.APITokens = validTokens
 	}
 
+	if cfg.Backend == "postgres" {
+		switch {
+		case cfg.PostgresHost == "":
+			return nil, nil, fmt.Errorf("postgres backend can " +
+				"not be used without the postgreshost param")
+		case cfg.PostgresRootCert == "":
+			return nil, nil, fmt.Errorf("postgres backend can " +
+				"not be used without the postgresrootcert param")
+		case cfg.PostgresCert == "":
+			return nil, nil, fmt.Errorf("postgres backend can " +
+				"not be used without the postgrescert param")
+		case cfg.PostgresKey == "":
+			return nil, nil, fmt.Errorf("postgres backend can " +
+				"not be used without the postgreskey param")
+		}
+
+		cfg.PostgresRootCert = cleanAndExpandPath(cfg.PostgresRootCert)
+		cfg.PostgresCert = cleanAndExpandPath(cfg.PostgresCert)
+		cfg.PostgresKey = cleanAndExpandPath(cfg.PostgresKey)
+
+		// Validate cache root cert.
+		b, err := ioutil.ReadFile(cfg.PostgresRootCert)
+		if err != nil {
+			return nil, nil, fmt.Errorf("read cacherootcert: %v", err)
+		}
+		block, _ := pem.Decode(b)
+		if block == nil {
+			return nil, nil, fmt.Errorf("%s is not a valid certificate",
+				cfg.PostgresRootCert)
+		}
+		_, err = x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return nil, nil, fmt.Errorf("parse cacherootcert: %v", err)
+		}
+
+		// Validate cache key pair.
+		_, err = tls.LoadX509KeyPair(cfg.PostgresCert, cfg.PostgresKey)
+		if err != nil {
+			return nil, nil, fmt.Errorf("load key pair cachecert "+
+				"and cachekey: %v", err)
+		}
+	}
+
 	// Warn about missing config file only after all other configuration is
 	// done.  This prevents the warning on help messages and invalid
 	// options.  Note this should go directly before the return.
diff --git a/dcrtimed/sample-dcrtimed.conf b/dcrtimed/sample-dcrtimed.conf
index c55fd87..0bf652f 100644
--- a/dcrtimed/sample-dcrtimed.conf
+++ b/dcrtimed/sample-dcrtimed.conf
@@ -41,3 +41,18 @@
 
 ; API Versions is a comma-separated list of versions to enable support on the daemon.
 ;apiversions=1,2
+
+; Backend used to set type of cache, possbile values: 'filesystem', 'postgres'
+;backend=filesystem
+
+; PostgreSQL connection information, required when using working with postgres
+; backend
+;
+; PostgreSQL host
+;postgreshost=localhost:5432
+; Postgres root cert
+;postgresrootcert="~/.postgresql/certs/clients/dcrtimed/root.crt"
+; Postgres client cert
+;postgrescert="~/.postgresql/certs/clients/dcrtimed/client.dcrtimed.crt"
+; Postgres client key
+;postgreskey="~/.postgresql/certs/clients/dcrtimed/client.dcrtimed.key"
diff --git a/go.mod b/go.mod
index ed66d1b..4d4e5b8 100644
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/jrick/logrotate v1.0.0
 	github.com/kr/pretty v0.1.0 // indirect
+	github.com/lib/pq v1.8.0 // indirect
 	github.com/robfig/cron v1.2.0
 	github.com/syndtr/goleveldb v1.0.0
 	google.golang.org/grpc v1.27.1
diff --git a/go.sum b/go.sum
index c2d2ae3..17bf957 100644
--- a/go.sum
+++ b/go.sum
@@ -124,6 +124,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
+github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=

From 73eb7fd877af27cceacee2031a9bf912d5ca95ef Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 8 Aug 2020 18:52:31 +0300
Subject: [PATCH 11/66] provide all postgres ssl vars & establish connection

---
 dcrtimed/config.go   |  6 +++---
 dcrtimed/dcrtimed.go | 22 ++++++++++++++++++++--
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/dcrtimed/config.go b/dcrtimed/config.go
index 89c893d..336692b 100644
--- a/dcrtimed/config.go
+++ b/dcrtimed/config.go
@@ -577,7 +577,7 @@ func loadConfig() (*config, []string, error) {
 		// Validate cache root cert.
 		b, err := ioutil.ReadFile(cfg.PostgresRootCert)
 		if err != nil {
-			return nil, nil, fmt.Errorf("read cacherootcert: %v", err)
+			return nil, nil, fmt.Errorf("read postgresrootcert: %v", err)
 		}
 		block, _ := pem.Decode(b)
 		if block == nil {
@@ -586,13 +586,13 @@ func loadConfig() (*config, []string, error) {
 		}
 		_, err = x509.ParseCertificate(block.Bytes)
 		if err != nil {
-			return nil, nil, fmt.Errorf("parse cacherootcert: %v", err)
+			return nil, nil, fmt.Errorf("parse postgresrootcert: %v", err)
 		}
 
 		// Validate cache key pair.
 		_, err = tls.LoadX509KeyPair(cfg.PostgresCert, cfg.PostgresKey)
 		if err != nil {
-			return nil, nil, fmt.Errorf("load key pair cachecert "+
+			return nil, nil, fmt.Errorf("load key pair postgrescert "+
 				"and cachekey: %v", err)
 		}
 	}
diff --git a/dcrtimed/dcrtimed.go b/dcrtimed/dcrtimed.go
index be45832..5a0b94d 100644
--- a/dcrtimed/dcrtimed.go
+++ b/dcrtimed/dcrtimed.go
@@ -35,7 +35,8 @@ import (
 const (
 	fStr = "20060102.150405"
 
-	forward = "X-Forwarded-For"
+	forward      = "X-Forwarded-For"
+	postgresUser = "dcrtimed"
 )
 
 var (
@@ -1379,7 +1380,24 @@ func _main() error {
 			}
 		case "postgres":
 			postgres.UseLogger(fsbeLog)
-			b, err = postgres.New()
+			var net string
+			switch loadedCfg.TestNet {
+			case true:
+				net = "testnet"
+			default:
+				net = "mainnet"
+			}
+			b, err = postgres.New(
+				postgresUser,
+				loadedCfg.PostgresHost,
+				net,
+				loadedCfg.PostgresRootCert,
+				loadedCfg.PostgresCert,
+				loadedCfg.PostgresKey,
+				loadedCfg.WalletCert,
+				loadedCfg.WalletHost,
+				loadedCfg.EnableCollections,
+				[]byte(loadedCfg.WalletPassphrase))
 			if err != nil {
 				return nil
 			}

From 3317ae5096287b5362144c17b1cd50affcfa8a63 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 8 Aug 2020 21:08:28 +0300
Subject: [PATCH 12/66] add scripts to generate postgres ssl certs & initial
 setup

---
 scripts/postgrescerts.sh | 149 +++++++++++++++++++++++++++++++++++++++
 scripts/postgressetup.sh |  69 ++++++++++++++++++
 2 files changed, 218 insertions(+)
 create mode 100755 scripts/postgrescerts.sh
 create mode 100755 scripts/postgressetup.sh

diff --git a/scripts/postgrescerts.sh b/scripts/postgrescerts.sh
new file mode 100755
index 0000000..53eb274
--- /dev/null
+++ b/scripts/postgrescerts.sh
@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+# This script creates the certificates required to run a PostgreSQL node
+# locally. This includes creating a CA certificate, a node certificate, one 
+# root client to connect via cli, and one more client certificate for dcrtimed
+#
+# NOTE: this scripts creates and copies over the server files (server.crt, 
+# server.key & root.crt) to postgrees data dir, is uses $PGDATA environment
+# variable to determine where to copy the files to, make sure it's exported
+# before running the script.
+# when done creating & moving certs this script restarts postgres server
+# in order to load created server certs.
+#
+# More information on PostgreSQL ssl connection usage can be found at:
+# https://www.postgresql.org/docs/9.5/ssl-tcp.html
+
+set -ex
+
+# Database usernames
+readonly USER_DCRTIMED="dcrtimedd"
+
+# POSTGRES_DIR is where all of the certificates will be created.
+POSTGRES_DIR=$1
+if [ "${POSTGRES_DIR}" == "" ]; then
+  POSTGRES_DIR="${HOME}/.postgresql"
+fi
+
+# Create postgresdb clients directories.
+mkdir -p "${POSTGRES_DIR}/certs/clients/root"
+mkdir -p "${POSTGRES_DIR}/certs/clients/${USER_DCRTIMED}"
+
+# Create a CA private key
+echo "Generating root.key, please type a password:"
+openssl genrsa -des3 -out root.key 4096
+# Remove passphrase
+echo "Removing root.key password, please re-type it:"
+openssl rsa -in root.key -out root.key -passout pass:123
+
+# Create a root Certificate Authority (CA)
+openssl \
+    req -new -x509 \
+    -days 365 \
+    -subj "/CN=CA" \
+    -key root.key \
+    -out root.crt
+
+# Create server key
+echo "Generating server.key, please type a password:"
+openssl genrsa -des3 -out server.key 4096 -passout pass:123
+#Remove a passphrase
+echo "Removing server.key password, please re-type it:"
+openssl rsa -in server.key -out server.key -passout pass:123
+
+# Create a root certificate signing request
+openssl \
+    req -new \
+    -key server.key \
+    -subj "/CN=localhost" \
+    -text \
+    -out server.csr
+
+# Create server certificate
+openssl \
+    x509 -req \
+    -in server.csr \
+    -text \
+    -days 365 \
+    -CA root.crt \
+    -CAkey root.key \
+    -CAcreateserial \
+    -out server.crt
+
+# Copy server.key, server.crt & root.crt to postgres' data dir as discribed in
+# PostgresSQL ssl connection documentation, it uses environment variable PGDATA
+# as postgres' data dir
+echo "Copying server.key server.crt root.crt to $PGDATA as postgres sys user"
+sudo -u postgres cp server.key server.crt root.crt $PGDATA
+
+# Create root client key - used to connect via cli
+openssl genrsa -out client.root.key 4096
+# Remove passphrase
+openssl rsa -in client.root.key -out client.root.key
+
+chmod og-rwx client.root.key
+
+# Create client certificate signing request
+# Note: CN should be equal to db username
+openssl \
+    req -new \
+    -key client.root.key \
+    -subj "/CN=postgres" \
+    -out client.root.csr
+
+# Create client certificate
+openssl \
+    x509 -req \
+    -in client.root.csr \
+    -CA root.crt \
+    -CAkey root.key \
+    -CAcreateserial \
+    -days 365 \
+    -text \
+    -out client.root.crt
+
+# Copy client to certs dir
+cp client.root.key client.root.crt root.crt \
+  ${POSTGRES_DIR}/certs/clients/root
+
+# Create client key for dcrtimed
+openssl genrsa -out client.${USER_DCRTIMED}.key 4096
+# Remove passphrase
+openssl rsa -in client.${USER_DCRTIMED}.key -out client.${USER_DCRTIMED}.key
+
+chmod og-rwx client.${USER_DCRTIMED}.key
+
+# Create client certificate signing request
+# Note: CN should be equal to db username
+openssl \
+    req -new \
+    -key client.${USER_DCRTIMED}.key \
+    -subj "/CN=${USER_DCRTIMED}" \
+    -out client.${USER_DCRTIMED}.csr
+
+# Create client certificate
+openssl \
+    x509 -req \
+    -in client.${USER_DCRTIMED}.csr \
+    -CA root.crt \
+    -CAkey root.key \
+    -CAcreateserial \
+    -days 365 \
+    -text \
+    -out client.${USER_DCRTIMED}.crt
+
+# Copy client to certs dir
+cp client.${USER_DCRTIMED}.key client.${USER_DCRTIMED}.crt root.crt \
+  ${POSTGRES_DIR}/certs/clients/${USER_DCRTIMED}
+
+# "On Unix systems, the permissions on 
+# server.key must disallow any access to world or group"
+# Source: PostgresSQL docs - link above
+#
+sudo chmod og-rwx $PGDATA/server.key
+sudo chmod og-rwx $POSTGRES_DIR/certs/clients/${USER_DCRTIMED}/client.${USER_DCRTIMED}.key
+
+# Cleanup
+rm *.crt *.key *.srl *.csr
+
+# Restart postgres to load server certs
+sudo -u postgres pg_ctl -D $PGDATA restart
diff --git a/scripts/postgressetup.sh b/scripts/postgressetup.sh
new file mode 100755
index 0000000..849c070
--- /dev/null
+++ b/scripts/postgressetup.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# This script sets up the PostgresSQL databases for the dcrtimed cache and
+# assigns user privileges.
+# This script requires that you have already created PostgresSQL certificates
+# using the postgrescerts.sh script and that you have a PostgresSQL instance
+# listening on the default port localhost:5432.
+
+set -ex
+
+# POSTGRES_DIR must be the same directory that was used with the
+# postgrescerts.sh script.
+POSTGRES_DIR=$1
+if [ "${POSTGRES_DIR}" == "" ]; then
+  POSTGRES_DIR="${HOME}/.postgresql"
+fi
+
+# ROOT_CERTS_DIR must contain client.root.crt, client.root.key, and root.crt.
+readonly ROOT_CERTS_DIR="${POSTGRES_DIR}/certs/clients/root"
+
+if [ ! -f "${ROOT_CERTS_DIR}/client.root.crt" ]; then
+  >&2 echo "error: file not found ${ROOT_CERTS_DIR}/client.root.crt"
+  exit
+elif [ ! -f "${ROOT_CERTS_DIR}/client.root.key" ]; then
+  >&2 echo "error: file not found ${ROOT_CERTS_DIR}/client.root.key"
+  exit
+elif [ ! -f "${ROOT_CERTS_DIR}/root.crt" ]; then
+  >&2 echo "error: file not found ${ROOT_CERTS_DIR}/root.crt"
+  exit
+fi
+
+# Database names.
+readonly DB_MAINNET="mainnet_dcrtime"
+readonly DB_TESTNET="testnet_dcrtime"
+
+# Database usernames.
+readonly USER_DCRTIMED="dcrtimed"
+
+# Psql connection string
+readonly CONNECTION_STRING="host=localhost \
+  sslmode=verify-full  \
+  sslrootcert=${ROOT_CERTS_DIR}/root.crt \
+  sslcert=${ROOT_CERTS_DIR}/client.root.crt \
+  sslkey=${ROOT_CERTS_DIR}/client.root.key \
+  port=5432 \
+  user=postgres"
+
+# Create the mainnet and testnet databases for the dcrtimed records cache.
+psql "$CONNECTION_STRING" \
+  -c "CREATE DATABASE ${DB_MAINNET}"
+
+psql "$CONNECTION_STRING" \
+  -c "CREATE DATABASE ${DB_TESTNET}"
+
+# Create the dcrtimed user(if not exists) and assign privileges.
+psql "$CONNECTION_STRING" \
+   -c "DO \$\$
+BEGIN
+CREATE USER ${USER_DCRTIMED};
+EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
+END
+\$\$;";
+
+psql "$CONNECTION_STRING" \
+  -c "GRANT CREATE \
+  ON DATABASE ${DB_MAINNET} TO ${USER_DCRTIMED}"
+
+psql "$CONNECTION_STRING" \
+  -c "GRANT CREATE \
+  ON DATABASE ${DB_TESTNET} TO ${USER_DCRTIMED}"

From c7129c4355d16e17982375296f0deaa8a58029c2 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 9 Aug 2020 01:27:07 +0300
Subject: [PATCH 13/66] postgress connection last touches

---
 dcrtimed/backend/postgres/postgres.go | 15 ++++++++-------
 dcrtimed/dcrtimed.go                  |  2 +-
 go.mod                                |  2 +-
 scripts/postgrescerts.sh              |  2 +-
 4 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index dfea992..7991599 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -16,6 +16,7 @@ import (
 
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/dcrtimewallet"
+	_ "github.com/lib/pq"
 	"github.com/robfig/cron"
 )
 
@@ -172,14 +173,14 @@ func New(user, host, net, rootCert, cert, key, walletCert, walletHost string, en
 	//}
 
 	// Launch cron.
-	//err = pg.cron.AddFunc(flushSchedule, func() {
-	//pg.flusher()
-	///})
-	//if err != nil {
-	//return nil, err
-	//}
+	err = pg.cron.AddFunc(flushSchedule, func() {
+		fmt.Println("brrrrrr")
+	})
+	if err != nil {
+		return nil, err
+	}
 
-	//pg.cron.Start()
+	pg.cron.Start()
 
 	return pg, nil
 }
diff --git a/dcrtimed/dcrtimed.go b/dcrtimed/dcrtimed.go
index 5a0b94d..6049b4e 100644
--- a/dcrtimed/dcrtimed.go
+++ b/dcrtimed/dcrtimed.go
@@ -1399,7 +1399,7 @@ func _main() error {
 				loadedCfg.EnableCollections,
 				[]byte(loadedCfg.WalletPassphrase))
 			if err != nil {
-				return nil
+				return err
 			}
 		}
 
diff --git a/go.mod b/go.mod
index 4d4e5b8..07c1e9c 100644
--- a/go.mod
+++ b/go.mod
@@ -19,7 +19,7 @@ require (
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/jrick/logrotate v1.0.0
 	github.com/kr/pretty v0.1.0 // indirect
-	github.com/lib/pq v1.8.0 // indirect
+	github.com/lib/pq v1.8.0
 	github.com/robfig/cron v1.2.0
 	github.com/syndtr/goleveldb v1.0.0
 	google.golang.org/grpc v1.27.1
diff --git a/scripts/postgrescerts.sh b/scripts/postgrescerts.sh
index 53eb274..ffb8ab0 100755
--- a/scripts/postgrescerts.sh
+++ b/scripts/postgrescerts.sh
@@ -16,7 +16,7 @@
 set -ex
 
 # Database usernames
-readonly USER_DCRTIMED="dcrtimedd"
+readonly USER_DCRTIMED="dcrtimed"
 
 # POSTGRES_DIR is where all of the certificates will be created.
 POSTGRES_DIR=$1

From 49bcd143ac4e4b7fd6c67f6ce814e7b3d128b795 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 9 Aug 2020 22:25:26 +0300
Subject: [PATCH 14/66] add createTables func

---
 dcrtimed/backend/postgres/postgres.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 7991599..28116db 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -112,6 +112,16 @@ func buildQueryString(rootCert, cert, key string) string {
 	return v.Encode()
 }
 
+func createTables(db *sql.DB) error {
+	//if !tx.HasTable(tableKeyValue) {
+	//err := tx.CreateTable(&KeyValue{}).Error
+	//if err != nil {
+	//return err
+	//}
+
+	return nil
+}
+
 // internalNew creates the Pstgres context but does not launch background
 // bits.  This is used by the test packages.
 func internalNew(user, host, net, rootCert, cert, key string) (*Postgres, error) {

From 086a6f7333046145fb4e07872f82f3bfd6cdd5d7 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Mon, 10 Aug 2020 01:29:58 +0300
Subject: [PATCH 15/66] create anchors table if doesn't exsist

---
 dcrtimed/backend/postgres/postgres.go | 61 +++++++++++++++++++++++++--
 1 file changed, 58 insertions(+), 3 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 28116db..1533975 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -20,6 +20,11 @@ import (
 	"github.com/robfig/cron"
 )
 
+const (
+	tableRecords = "records"
+	tableAnchors = "anchors"
+)
+
 var (
 	_ backend.Backend = (*Postgres)(nil)
 
@@ -112,13 +117,57 @@ func buildQueryString(rootCert, cert, key string) string {
 	return v.Encode()
 }
 
+func hasTable(db *sql.DB, name string) (bool, error) {
+	rows, err := db.Query(`SELECT EXISTS (SELECT FROM information_schema.tables 
+	WHERE table_schema = 'public' AND table_name  = $1)`, name)
+	if err != nil {
+		return false, err
+	}
+	defer rows.Close()
+	var exists bool
+	for rows.Next() {
+		err = rows.Scan(&exists)
+		if err != nil {
+			return false, err
+		}
+	}
+	return exists, nil
+}
+
+func createAnchorsTable(db *sql.DB) error {
+	_, err := db.Exec(`CREATE TABLE public.anchors
+(
+    merkle character varying(64) COLLATE pg_catalog."default" NOT NULL,
+    hashes text[] COLLATE pg_catalog."default" NOT NULL,
+    tx_hash text COLLATE pg_catalog."default",
+    chain_timestamp bigint,
+    flush_timestamp bigint,
+    CONSTRAINT anchors_pkey PRIMARY KEY (merkle)
+)`)
+	if err != nil {
+		return err
+	}
+	fmt.Println("anchors creared")
+	return nil
+}
+
 func createTables(db *sql.DB) error {
-	//if !tx.HasTable(tableKeyValue) {
-	//err := tx.CreateTable(&KeyValue{}).Error
+	exists, err := hasTable(db, tableAnchors)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		err := createAnchorsTable(db)
+		if err != nil {
+			return err
+		}
+	}
+	//if !db.HasTable(tableRecords) {
+	//err := db.CreateTable(&Anchor{}).Error
 	//if err != nil {
 	//return err
 	//}
-
+	//}
 	return nil
 }
 
@@ -141,6 +190,12 @@ func internalNew(user, host, net, rootCert, cert, key string) (*Postgres, error)
 		return nil, fmt.Errorf("connect to database '%v': %v", addr, err)
 	}
 
+	// Create tables
+	err = createTables(db)
+	if err != nil {
+		return nil, err
+	}
+
 	pg := &Postgres{
 		cron:     cron.New(),
 		db:       db,

From 9d387313a6211523dfa20499c6c1d066a1ca11ab Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Mon, 10 Aug 2020 18:45:35 +0300
Subject: [PATCH 16/66] add pg log & create pg tables if doesn't exist

---
 dcrtimed/backend/postgres/postgres.go | 70 ++++++++++++++++++++++-----
 dcrtimed/dcrtimed.go                  |  2 +-
 dcrtimed/log.go                       |  2 +
 3 files changed, 62 insertions(+), 12 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 1533975..00bd27c 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -119,7 +119,7 @@ func buildQueryString(rootCert, cert, key string) string {
 
 func hasTable(db *sql.DB, name string) (bool, error) {
 	rows, err := db.Query(`SELECT EXISTS (SELECT FROM information_schema.tables 
-	WHERE table_schema = 'public' AND table_name  = $1)`, name)
+		WHERE table_schema = 'public' AND table_name  = $1)`, name)
 	if err != nil {
 		return false, err
 	}
@@ -143,11 +143,56 @@ func createAnchorsTable(db *sql.DB) error {
     chain_timestamp bigint,
     flush_timestamp bigint,
     CONSTRAINT anchors_pkey PRIMARY KEY (merkle)
-)`)
+);
+-- Index: idx_chain_timestamp
+CREATE INDEX idx_chain_timestamp
+    ON public.anchors USING btree
+    (chain_timestamp ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_flush_timestamp
+CREATE INDEX idx_flush_timestamp
+    ON public.anchors USING btree
+    (flush_timestamp ASC NULLS LAST)
+    TABLESPACE pg_default;
+`)
 	if err != nil {
 		return err
 	}
-	fmt.Println("anchors creared")
+	log.Infof("Anchors table created")
+	return nil
+}
+
+func createRecordsTable(db *sql.DB) error {
+	_, err := db.Exec(`CREATE TABLE public.records
+(
+    digest bytea NOT NULL,
+    anchor_merkle character varying(64) COLLATE pg_catalog."default",
+    key serial NOT NULL,
+    collection_timestamp text COLLATE pg_catalog."default" NOT NULL,
+    CONSTRAINT records_pkey PRIMARY KEY (key),
+    CONSTRAINT records_anchors_fkey FOREIGN KEY (anchor_merkle)
+        REFERENCES public.anchors (merkle) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION
+        NOT VALID
+);
+
+-- Index: fki_records_anchors_fkey
+CREATE INDEX fki_records_anchors_fkey
+    ON public.records USING btree
+    (anchor_merkle COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+
+-- Index: idx_collection_timestamp
+CREATE INDEX idx_collection_timestamp
+    ON public.records USING btree
+    (collection_timestamp COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+`)
+	if err != nil {
+		return err
+	}
+	log.Infof("Records table created")
 	return nil
 }
 
@@ -157,17 +202,21 @@ func createTables(db *sql.DB) error {
 		return err
 	}
 	if !exists {
-		err := createAnchorsTable(db)
+		err = createAnchorsTable(db)
+		if err != nil {
+			return err
+		}
+	}
+	exists, err = hasTable(db, tableRecords)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		err = createRecordsTable(db)
 		if err != nil {
 			return err
 		}
 	}
-	//if !db.HasTable(tableRecords) {
-	//err := db.CreateTable(&Anchor{}).Error
-	//if err != nil {
-	//return err
-	//}
-	//}
 	return nil
 }
 
@@ -239,7 +288,6 @@ func New(user, host, net, rootCert, cert, key, walletCert, walletHost string, en
 
 	// Launch cron.
 	err = pg.cron.AddFunc(flushSchedule, func() {
-		fmt.Println("brrrrrr")
 	})
 	if err != nil {
 		return nil, err
diff --git a/dcrtimed/dcrtimed.go b/dcrtimed/dcrtimed.go
index 6049b4e..a78ddd7 100644
--- a/dcrtimed/dcrtimed.go
+++ b/dcrtimed/dcrtimed.go
@@ -1379,7 +1379,7 @@ func _main() error {
 				return err
 			}
 		case "postgres":
-			postgres.UseLogger(fsbeLog)
+			postgres.UseLogger(pgbeLog)
 			var net string
 			switch loadedCfg.TestNet {
 			case true:
diff --git a/dcrtimed/log.go b/dcrtimed/log.go
index 115005b..cffe111 100644
--- a/dcrtimed/log.go
+++ b/dcrtimed/log.go
@@ -42,12 +42,14 @@ var (
 
 	log     = backendLog.Logger("DCRT")
 	fsbeLog = backendLog.Logger("FSBE")
+	pgbeLog = backendLog.Logger("PGBE")
 )
 
 // subsystemLoggers maps each subsystem identifier to its associated logger.
 var subsystemLoggers = map[string]slog.Logger{
 	"DCRT": log,
 	"FSBE": fsbeLog,
+	"PGBE": pgbeLog,
 }
 
 // initLogRotator initializes the logging rotater to write logs to logFile and

From d9afd5abb3b4426dd2992bf660410cb33d495f1f Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 01:30:06 +0300
Subject: [PATCH 17/66] Implement backend Put func

---
 dcrtimed/backend/postgres/postgres.go | 119 +++++++++++++++++++++++++-
 1 file changed, 117 insertions(+), 2 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 00bd27c..44ecad9 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -7,6 +7,7 @@ package postgres
 import (
 	"crypto/sha256"
 	"database/sql"
+	"encoding/binary"
 	"fmt"
 	"net/url"
 	"os"
@@ -16,6 +17,7 @@ import (
 
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/dcrtimewallet"
+	"github.com/lib/pq"
 	_ "github.com/lib/pq"
 	"github.com/robfig/cron"
 )
@@ -23,6 +25,9 @@ import (
 const (
 	tableRecords = "records"
 	tableAnchors = "anchors"
+
+	// errorFound is thrown if digest was found in records table
+	errorFound = 1001
 )
 
 var (
@@ -56,6 +61,18 @@ type Postgres struct {
 	testing bool             // Enabled during test
 }
 
+// now returns current time stamp rounded down to 1 hour.  All timestamps are
+// UTC.
+func (pg *Postgres) now() time.Time {
+	return pg.truncate(pg.myNow().UTC(), pg.duration)
+}
+
+// truncate rounds time down to the provided duration.  This is split out in
+// order to test.
+func (pg *Postgres) truncate(t time.Time, d time.Duration) time.Time {
+	return t.Truncate(d)
+}
+
 // Return timestamp information for given digests.
 func (pg *Postgres) Get([][sha256.Size]byte) ([]backend.GetResult, error) {
 	return nil, nil
@@ -66,10 +83,108 @@ func (pg *Postgres) GetTimestamps([]int64) ([]backend.TimestampResult, error) {
 	return nil, nil
 }
 
+func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
+	rows, err := pg.db.Query(`SELECT EXISTS (SELECT FROM records 
+		WHERE digest = $1)`, hash)
+	if err != nil {
+		return false, err
+	}
+	defer rows.Close()
+	var exists bool
+	for rows.Next() {
+		err = rows.Scan(&exists)
+		if err != nil {
+			return false, err
+		}
+	}
+	return exists, nil
+}
+
 // Store hashes and return timestamp and associated errors.  Put is
 // allowed to return transient errors.
-func (pg *Postgres) Put([][sha256.Size]byte) (int64, []backend.PutResult, error) {
-	return 0, nil, nil
+func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult, error) {
+	// Poor mans two-phase commit.
+	pg.Lock()
+	commit := pg.commit
+	pg.Unlock()
+
+	// Get current time rounded down.
+	ts := pg.now().Unix()
+	//now := fs.now().Format(fStr)
+	timestamp := make([]byte, 8)
+	binary.LittleEndian.PutUint64(timestamp, uint64(ts))
+
+	// Prep return and unwind bits before taking mutex.
+	me := make([]backend.PutResult, 0, len(hashes))
+
+	// Create batch transaction
+	txn, err := pg.db.Begin()
+	if err != nil {
+		return 0, []backend.PutResult{}, err
+	}
+
+	stmt, err := txn.Prepare(pq.CopyIn("records", "digest",
+		"collection_timestamp"))
+	if err != nil {
+		return 0, []backend.PutResult{}, err
+	}
+
+	for _, hash := range hashes {
+		// exists ?
+		exists, err := pg.checkIfDigestExists(hash[:])
+		if err != nil {
+			return 0, []backend.PutResult{}, err
+		}
+		if exists {
+			me = append(me, backend.PutResult{
+				Digest:    hash,
+				ErrorCode: backend.ErrorExists,
+			})
+
+			// Override error code during testing
+			if pg.testing {
+				me[len(me)-1].ErrorCode = errorFound
+			}
+			continue
+		}
+
+		// Insert record
+		_, err = stmt.Exec(hash, ts)
+		if err != nil {
+			return 0, []backend.PutResult{}, err
+		}
+
+		// Mark as successful
+		me = append(me, backend.PutResult{
+			Digest:    hash,
+			ErrorCode: backend.ErrorOK,
+		})
+	}
+
+	// From this point on the operation must be atomic.
+	pg.Lock()
+	defer pg.Unlock()
+
+	// Make sure we are on the same commit.
+	if commit != pg.commit {
+		return 0, []backend.PutResult{}, backend.ErrTryAgainLater
+	}
+
+	// Write to db
+	_, err = stmt.Exec()
+	if err != nil {
+		return 0, []backend.PutResult{}, err
+	}
+	err = stmt.Close()
+	if err != nil {
+		return 0, []backend.PutResult{}, err
+	}
+	err = txn.Commit()
+	if err != nil {
+		return 0, []backend.PutResult{}, err
+	}
+
+	return ts, me, nil
 }
 
 // Close performs cleanup of the backend.

From f6a96e0b622cc0e1e24fa420f74255f9f7c39790 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 14:55:19 +0300
Subject: [PATCH 18/66] polish Put func

---
 dcrtimed/backend/postgres/postgres.go | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 44ecad9..4bec305 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -7,7 +7,6 @@ package postgres
 import (
 	"crypto/sha256"
 	"database/sql"
-	"encoding/binary"
 	"fmt"
 	"net/url"
 	"os"
@@ -23,6 +22,7 @@ import (
 )
 
 const (
+	fStr         = "20060102.150405"
 	tableRecords = "records"
 	tableAnchors = "anchors"
 
@@ -103,16 +103,14 @@ func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
 // Store hashes and return timestamp and associated errors.  Put is
 // allowed to return transient errors.
 func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult, error) {
-	// Poor mans two-phase commit.
+
+	// Two-phase commit.
 	pg.Lock()
 	commit := pg.commit
 	pg.Unlock()
 
 	// Get current time rounded down.
 	ts := pg.now().Unix()
-	//now := fs.now().Format(fStr)
-	timestamp := make([]byte, 8)
-	binary.LittleEndian.PutUint64(timestamp, uint64(ts))
 
 	// Prep return and unwind bits before taking mutex.
 	me := make([]backend.PutResult, 0, len(hashes))
@@ -130,7 +128,7 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 	}
 
 	for _, hash := range hashes {
-		// exists ?
+		// Check if digest exists
 		exists, err := pg.checkIfDigestExists(hash[:])
 		if err != nil {
 			return 0, []backend.PutResult{}, err
@@ -147,9 +145,8 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 			}
 			continue
 		}
-
 		// Insert record
-		_, err = stmt.Exec(hash, ts)
+		_, err = stmt.Exec(hash[:], ts)
 		if err != nil {
 			return 0, []backend.PutResult{}, err
 		}

From 708c017f66580070288e16d608ef0532f84dcf9a Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 18:30:47 +0300
Subject: [PATCH 19/66] add unique indexes for sql cols

---
 dcrtimed/backend/postgres/schema.md | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/dcrtimed/backend/postgres/schema.md b/dcrtimed/backend/postgres/schema.md
index 7033ae0..b9f1925 100644
--- a/dcrtimed/backend/postgres/schema.md
+++ b/dcrtimed/backend/postgres/schema.md
@@ -14,19 +14,19 @@ records table, below you find the detailed description of the two tables:
 ### Tables
 
 **Records:**
-| Col Name             | Type              | Not Null | P. Key | F. Key | Indexed | Description                                                    |
-|----------------------|-------------------|----------|--------|--------|---------|----------------------------------------------------------------|
-| key                  | serial            | x        | x      |        |         | Auto incremented  identifier                                   |
-| collection_timestamp | text              | x        |        |        | x       | Unix timestamp of collection                                   |
-| digest               | bytea             | x        |        |        |         | Timestamped digest                                             |
-| anchor_merkle        | char. varying(64) |          |        | x      | x       | Merkle root of corresponding anchor - linking to anchors table, nil if not anchored yet |
+| Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                     |
+|------------------|-------------------|----------|--------|--------|---------|--------|---------------------------------|
+| merkle           | char. varying(64) | x        | x      |        | x       | x      | Anchor merkle root              |
+| hashes           | text[]            | x        |        |        | x       | x      | Anchored hashes                 |
+| tx_hash          | text              |          |        |        | x       | x      | Anchor tx hash                  |
+| chain_timestamp  | bigint            |          |        |        |         |        | Anchor timestamp on blockchain  |
 
 **Anchors:**
-| Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Description                     |
-|------------------|-------------------|----------|--------|--------|---------|---------------------------------|
-| merkle           | char. varying(64) | x        | x      |        |         | Anchor merkle root              |
-| hashes           | text[]            | x        |        |        |         | Anchored hashes                 |
-| tx_hash          | text              |          |        |        |         | Anchor tx hash                  |
-| chain_timestamp  | bigint            |          |        |        | x       | Anchor timestamp on blockchain  |
-| flush_timestamp  | bigint            |          |        |        | x       | When anchor actually  flushed   |
+| Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                     |
+|------------------|-------------------|----------|--------|--------|---------|--------|---------------------------------|
+| merkle           | char. varying(64) | x        | x      |        | x       | x      | Anchor merkle root              |
+| hashes           | text[]            | x        |        |        | x       | x      | Anchored hashes                 |
+| tx_hash          | text              |          |        |        | x       | x      | Anchor tx hash                  |
+| chain_timestamp  | bigint            |          |        |        |         |        | Anchor timestamp on blockchain  |
+| flush_timestamp  | bigint            |          |        |        |         |        | When anchor actually  flushed   |
 

From 07dc8c77a458df22c390d6f9c408f7c27def7887 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 18:38:22 +0300
Subject: [PATCH 20/66] .md land

---
 dcrtimed/backend/postgres/schema.md | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/dcrtimed/backend/postgres/schema.md b/dcrtimed/backend/postgres/schema.md
index b9f1925..2f5154d 100644
--- a/dcrtimed/backend/postgres/schema.md
+++ b/dcrtimed/backend/postgres/schema.md
@@ -14,12 +14,14 @@ records table, below you find the detailed description of the two tables:
 ### Tables
 
 **Records:**
-| Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                     |
-|------------------|-------------------|----------|--------|--------|---------|--------|---------------------------------|
-| merkle           | char. varying(64) | x        | x      |        | x       | x      | Anchor merkle root              |
-| hashes           | text[]            | x        |        |        | x       | x      | Anchored hashes                 |
-| tx_hash          | text              |          |        |        | x       | x      | Anchor tx hash                  |
-| chain_timestamp  | bigint            |          |        |        |         |        | Anchor timestamp on blockchain  |
+| Col Name             | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                  |
+|----------------------|-------------------|----------|--------|--------|---------|--------|------------------------------|
+| key                  | serial            | x        | x      |        |         |        | Auto incremented  identifier |
+| collection_timestamp | text              | x        |        |        | x       |        | Unix timestamp of collection |
+| digest               | bytea             | x        |        |        | x       | x      | Timestamped digest           |
+| anchor_merkle        | char. varying(64) |          |        | x      | x       |        | Anchor merkle root           |
+
+**Note:** `anchor_merkle` linking to anchors table, nil if not anchored yet
 
 **Anchors:**
 | Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                     |

From c16c9e8ad19e49afcf6eeb36f89a1c869c55c33a Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 18:48:36 +0300
Subject: [PATCH 21/66] add unique indexes in sql table def.

---
 dcrtimed/backend/postgres/postgres.go | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 4bec305..1a1d045 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -266,6 +266,21 @@ CREATE INDEX idx_flush_timestamp
     ON public.anchors USING btree
     (flush_timestamp ASC NULLS LAST)
     TABLESPACE pg_default;
+-- Index: idx_hashes
+CREATE UNIQUE INDEX idx_hashes
+    ON public.anchors USING btree
+    (hashes COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_merkle
+CREATE UNIQUE INDEX idx_merkle
+    ON public.anchors USING btree
+    (merkle COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_tx_hash
+CREATE UNIQUE INDEX idx_tx_hash
+    ON public.anchors USING btree
+    (tx_hash COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
 `)
 	if err != nil {
 		return err
@@ -294,12 +309,16 @@ CREATE INDEX fki_records_anchors_fkey
     ON public.records USING btree
     (anchor_merkle COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
-
 -- Index: idx_collection_timestamp
 CREATE INDEX idx_collection_timestamp
     ON public.records USING btree
     (collection_timestamp COLLATE pg_catalog."default" ASC NULLS LAST)
     TABLESPACE pg_default;
+-- Index: idx_digest
+CREATE UNIQUE INDEX idx_digest
+    ON public.records USING btree
+    (digest ASC NULLS LAST)
+    TABLESPACE pg_default;
 `)
 	if err != nil {
 		return err

From 68add0f0076b858ed4067027297a55207df11b40 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 20:34:06 +0300
Subject: [PATCH 22/66] move sql logic to separate file - postgres/sql.go

---
 dcrtimed/backend/postgres/postgres.go | 151 +-------------------------
 dcrtimed/backend/postgres/sql.go      | 140 ++++++++++++++++++++++++
 2 files changed, 146 insertions(+), 145 deletions(-)
 create mode 100644 dcrtimed/backend/postgres/sql.go

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 1a1d045..a1ff93c 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -83,23 +83,6 @@ func (pg *Postgres) GetTimestamps([]int64) ([]backend.TimestampResult, error) {
 	return nil, nil
 }
 
-func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
-	rows, err := pg.db.Query(`SELECT EXISTS (SELECT FROM records 
-		WHERE digest = $1)`, hash)
-	if err != nil {
-		return false, err
-	}
-	defer rows.Close()
-	var exists bool
-	for rows.Next() {
-		err = rows.Scan(&exists)
-		if err != nil {
-			return false, err
-		}
-	}
-	return exists, nil
-}
-
 // Store hashes and return timestamp and associated errors.  Put is
 // allowed to return transient errors.
 func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult, error) {
@@ -229,128 +212,6 @@ func buildQueryString(rootCert, cert, key string) string {
 	return v.Encode()
 }
 
-func hasTable(db *sql.DB, name string) (bool, error) {
-	rows, err := db.Query(`SELECT EXISTS (SELECT FROM information_schema.tables 
-		WHERE table_schema = 'public' AND table_name  = $1)`, name)
-	if err != nil {
-		return false, err
-	}
-	defer rows.Close()
-	var exists bool
-	for rows.Next() {
-		err = rows.Scan(&exists)
-		if err != nil {
-			return false, err
-		}
-	}
-	return exists, nil
-}
-
-func createAnchorsTable(db *sql.DB) error {
-	_, err := db.Exec(`CREATE TABLE public.anchors
-(
-    merkle character varying(64) COLLATE pg_catalog."default" NOT NULL,
-    hashes text[] COLLATE pg_catalog."default" NOT NULL,
-    tx_hash text COLLATE pg_catalog."default",
-    chain_timestamp bigint,
-    flush_timestamp bigint,
-    CONSTRAINT anchors_pkey PRIMARY KEY (merkle)
-);
--- Index: idx_chain_timestamp
-CREATE INDEX idx_chain_timestamp
-    ON public.anchors USING btree
-    (chain_timestamp ASC NULLS LAST)
-    TABLESPACE pg_default;
--- Index: idx_flush_timestamp
-CREATE INDEX idx_flush_timestamp
-    ON public.anchors USING btree
-    (flush_timestamp ASC NULLS LAST)
-    TABLESPACE pg_default;
--- Index: idx_hashes
-CREATE UNIQUE INDEX idx_hashes
-    ON public.anchors USING btree
-    (hashes COLLATE pg_catalog."default" ASC NULLS LAST)
-    TABLESPACE pg_default;
--- Index: idx_merkle
-CREATE UNIQUE INDEX idx_merkle
-    ON public.anchors USING btree
-    (merkle COLLATE pg_catalog."default" ASC NULLS LAST)
-    TABLESPACE pg_default;
--- Index: idx_tx_hash
-CREATE UNIQUE INDEX idx_tx_hash
-    ON public.anchors USING btree
-    (tx_hash COLLATE pg_catalog."default" ASC NULLS LAST)
-    TABLESPACE pg_default;
-`)
-	if err != nil {
-		return err
-	}
-	log.Infof("Anchors table created")
-	return nil
-}
-
-func createRecordsTable(db *sql.DB) error {
-	_, err := db.Exec(`CREATE TABLE public.records
-(
-    digest bytea NOT NULL,
-    anchor_merkle character varying(64) COLLATE pg_catalog."default",
-    key serial NOT NULL,
-    collection_timestamp text COLLATE pg_catalog."default" NOT NULL,
-    CONSTRAINT records_pkey PRIMARY KEY (key),
-    CONSTRAINT records_anchors_fkey FOREIGN KEY (anchor_merkle)
-        REFERENCES public.anchors (merkle) MATCH SIMPLE
-        ON UPDATE NO ACTION
-        ON DELETE NO ACTION
-        NOT VALID
-);
-
--- Index: fki_records_anchors_fkey
-CREATE INDEX fki_records_anchors_fkey
-    ON public.records USING btree
-    (anchor_merkle COLLATE pg_catalog."default" ASC NULLS LAST)
-    TABLESPACE pg_default;
--- Index: idx_collection_timestamp
-CREATE INDEX idx_collection_timestamp
-    ON public.records USING btree
-    (collection_timestamp COLLATE pg_catalog."default" ASC NULLS LAST)
-    TABLESPACE pg_default;
--- Index: idx_digest
-CREATE UNIQUE INDEX idx_digest
-    ON public.records USING btree
-    (digest ASC NULLS LAST)
-    TABLESPACE pg_default;
-`)
-	if err != nil {
-		return err
-	}
-	log.Infof("Records table created")
-	return nil
-}
-
-func createTables(db *sql.DB) error {
-	exists, err := hasTable(db, tableAnchors)
-	if err != nil {
-		return err
-	}
-	if !exists {
-		err = createAnchorsTable(db)
-		if err != nil {
-			return err
-		}
-	}
-	exists, err = hasTable(db, tableRecords)
-	if err != nil {
-		return err
-	}
-	if !exists {
-		err = createRecordsTable(db)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // internalNew creates the Pstgres context but does not launch background
 // bits.  This is used by the test packages.
 func internalNew(user, host, net, rootCert, cert, key string) (*Postgres, error) {
@@ -370,12 +231,6 @@ func internalNew(user, host, net, rootCert, cert, key string) (*Postgres, error)
 		return nil, fmt.Errorf("connect to database '%v': %v", addr, err)
 	}
 
-	// Create tables
-	err = createTables(db)
-	if err != nil {
-		return nil, err
-	}
-
 	pg := &Postgres{
 		cron:     cron.New(),
 		db:       db,
@@ -383,6 +238,12 @@ func internalNew(user, host, net, rootCert, cert, key string) (*Postgres, error)
 		myNow:    time.Now,
 	}
 
+	// Create tables
+	err = pg.createTables()
+	if err != nil {
+		return nil, err
+	}
+
 	return pg, nil
 }
 
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
new file mode 100644
index 0000000..4a3e6fe
--- /dev/null
+++ b/dcrtimed/backend/postgres/sql.go
@@ -0,0 +1,140 @@
+package postgres
+
+func (pg *Postgres) hasTable(name string) (bool, error) {
+	rows, err := pg.db.Query(`SELECT EXISTS (SELECT FROM information_schema.tables 
+		WHERE table_schema = 'public' AND table_name  = $1)`, name)
+	if err != nil {
+		return false, err
+	}
+	defer rows.Close()
+	var exists bool
+	for rows.Next() {
+		err = rows.Scan(&exists)
+		if err != nil {
+			return false, err
+		}
+	}
+	return exists, nil
+}
+
+func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
+	rows, err := pg.db.Query(`SELECT EXISTS (SELECT FROM records 
+		WHERE digest = $1)`, hash)
+	if err != nil {
+		return false, err
+	}
+	defer rows.Close()
+	var exists bool
+	for rows.Next() {
+		err = rows.Scan(&exists)
+		if err != nil {
+			return false, err
+		}
+	}
+	return exists, nil
+}
+
+func (pg *Postgres) createAnchorsTable() error {
+	_, err := pg.db.Exec(`CREATE TABLE public.anchors
+(
+    merkle character varying(64) COLLATE pg_catalog."default" NOT NULL,
+    hashes text[] COLLATE pg_catalog."default" NOT NULL,
+    tx_hash text COLLATE pg_catalog."default",
+    chain_timestamp bigint,
+    flush_timestamp bigint,
+    CONSTRAINT anchors_pkey PRIMARY KEY (merkle)
+);
+-- Index: idx_chain_timestamp
+CREATE INDEX idx_chain_timestamp
+    ON public.anchors USING btree
+    (chain_timestamp ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_flush_timestamp
+CREATE INDEX idx_flush_timestamp
+    ON public.anchors USING btree
+    (flush_timestamp ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_hashes
+CREATE UNIQUE INDEX idx_hashes
+    ON public.anchors USING btree
+    (hashes COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_merkle
+CREATE UNIQUE INDEX idx_merkle
+    ON public.anchors USING btree
+    (merkle COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_tx_hash
+CREATE UNIQUE INDEX idx_tx_hash
+    ON public.anchors USING btree
+    (tx_hash COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+`)
+	if err != nil {
+		return err
+	}
+	log.Infof("Anchors table created")
+	return nil
+}
+
+func (pg *Postgres) createRecordsTable() error {
+	_, err := pg.db.Exec(`CREATE TABLE public.records
+(
+    digest bytea NOT NULL,
+    anchor_merkle character varying(64) COLLATE pg_catalog."default",
+    key serial NOT NULL,
+    collection_timestamp bigint NOT NULL,
+    CONSTRAINT records_pkey PRIMARY KEY (key),
+    CONSTRAINT records_anchors_fkey FOREIGN KEY (anchor_merkle)
+        REFERENCES public.anchors (merkle) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION
+        NOT VALID
+);
+
+-- Index: fki_records_anchors_fkey
+CREATE INDEX fki_records_anchors_fkey
+    ON public.records USING btree
+    (anchor_merkle COLLATE pg_catalog."default" ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_collection_timestamp
+CREATE INDEX idx_collection_timestamp
+    ON public.records USING btree
+    (collection_timestamp ASC NULLS LAST)
+    TABLESPACE pg_default;
+-- Index: idx_digest
+CREATE UNIQUE INDEX idx_digest
+    ON public.records USING btree
+    (digest ASC NULLS LAST)
+    TABLESPACE pg_default;
+`)
+	if err != nil {
+		return err
+	}
+	log.Infof("Records table created")
+	return nil
+}
+
+func (pg *Postgres) createTables() error {
+	exists, err := pg.hasTable(tableAnchors)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		err = pg.createAnchorsTable()
+		if err != nil {
+			return err
+		}
+	}
+	exists, err = pg.hasTable(tableRecords)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		err = pg.createRecordsTable()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

From 9f8c03e23224035a74c310d2f5382e1cd23eff1a Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 20:41:06 +0300
Subject: [PATCH 23/66] move postgres user to postgres.go

---
 dcrtimed/backend/postgres/postgres.go | 11 ++++++-----
 dcrtimed/dcrtimed.go                  |  4 +---
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index a1ff93c..e4ff81f 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -25,6 +25,7 @@ const (
 	fStr         = "20060102.150405"
 	tableRecords = "records"
 	tableAnchors = "anchors"
+	dbUser       = "dcrtimed"
 
 	// errorFound is thrown if digest was found in records table
 	errorFound = 1001
@@ -214,10 +215,10 @@ func buildQueryString(rootCert, cert, key string) string {
 
 // internalNew creates the Pstgres context but does not launch background
 // bits.  This is used by the test packages.
-func internalNew(user, host, net, rootCert, cert, key string) (*Postgres, error) {
+func internalNew(host, net, rootCert, cert, key string) (*Postgres, error) {
 	// Connect to database
 	dbName := net + "_dcrtime"
-	h := "postgresql://" + user + "@" + host + "/" + dbName
+	h := "postgresql://" + dbUser + "@" + host + "/" + dbName
 	u, err := url.Parse(h)
 	if err != nil {
 		return nil, fmt.Errorf("parse url '%v': %v", h, err)
@@ -249,11 +250,11 @@ func internalNew(user, host, net, rootCert, cert, key string) (*Postgres, error)
 
 // New creates a new backend instance.  The caller should issue a Close once
 // the Postgres backend is no longer needed.
-func New(user, host, net, rootCert, cert, key, walletCert, walletHost string, enableCollections bool, walletPassphrase []byte) (*Postgres, error) {
+func New(host, net, rootCert, cert, key, walletCert, walletHost string, enableCollections bool, walletPassphrase []byte) (*Postgres, error) {
 	// XXX log more stuff
-	log.Tracef("New: %v %v %v %v %v %v", user, host, net, rootCert, cert, key)
+	log.Tracef("New: %v %v %v %v %v %v", dbUser, host, net, rootCert, cert, key)
 
-	pg, err := internalNew(user, host, net, rootCert, cert, key)
+	pg, err := internalNew(host, net, rootCert, cert, key)
 	if err != nil {
 		return nil, err
 	}
diff --git a/dcrtimed/dcrtimed.go b/dcrtimed/dcrtimed.go
index a78ddd7..c0ef211 100644
--- a/dcrtimed/dcrtimed.go
+++ b/dcrtimed/dcrtimed.go
@@ -35,8 +35,7 @@ import (
 const (
 	fStr = "20060102.150405"
 
-	forward      = "X-Forwarded-For"
-	postgresUser = "dcrtimed"
+	forward = "X-Forwarded-For"
 )
 
 var (
@@ -1388,7 +1387,6 @@ func _main() error {
 				net = "mainnet"
 			}
 			b, err = postgres.New(
-				postgresUser,
 				loadedCfg.PostgresHost,
 				net,
 				loadedCfg.PostgresRootCert,

From dda53312199e71002f1f3752bb0eee7a229760cd Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 20:57:00 +0300
Subject: [PATCH 24/66] delete dbuser from log

---
 dcrtimed/backend/postgres/postgres.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index e4ff81f..c06a943 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -252,7 +252,7 @@ func internalNew(host, net, rootCert, cert, key string) (*Postgres, error) {
 // the Postgres backend is no longer needed.
 func New(host, net, rootCert, cert, key, walletCert, walletHost string, enableCollections bool, walletPassphrase []byte) (*Postgres, error) {
 	// XXX log more stuff
-	log.Tracef("New: %v %v %v %v %v %v", dbUser, host, net, rootCert, cert, key)
+	log.Tracef("New: %v %v %v %v %v %v", host, net, rootCert, cert, key)
 
 	pg, err := internalNew(host, net, rootCert, cert, key)
 	if err != nil {

From f52214ef74fa2d5ee1b8f11fe45f30a77a31c551 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 21:07:03 +0300
Subject: [PATCH 25/66] ditch `key` col & use digest as primary key in records
 table

---
 dcrtimed/backend/postgres/postgres.go | 1 -
 dcrtimed/backend/postgres/schema.md   | 3 +--
 dcrtimed/backend/postgres/sql.go      | 3 +--
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index c06a943..41af35c 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -251,7 +251,6 @@ func internalNew(host, net, rootCert, cert, key string) (*Postgres, error) {
 // New creates a new backend instance.  The caller should issue a Close once
 // the Postgres backend is no longer needed.
 func New(host, net, rootCert, cert, key, walletCert, walletHost string, enableCollections bool, walletPassphrase []byte) (*Postgres, error) {
-	// XXX log more stuff
 	log.Tracef("New: %v %v %v %v %v %v", host, net, rootCert, cert, key)
 
 	pg, err := internalNew(host, net, rootCert, cert, key)
diff --git a/dcrtimed/backend/postgres/schema.md b/dcrtimed/backend/postgres/schema.md
index 2f5154d..596072a 100644
--- a/dcrtimed/backend/postgres/schema.md
+++ b/dcrtimed/backend/postgres/schema.md
@@ -16,9 +16,8 @@ records table, below you find the detailed description of the two tables:
 **Records:**
 | Col Name             | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                  |
 |----------------------|-------------------|----------|--------|--------|---------|--------|------------------------------|
-| key                  | serial            | x        | x      |        |         |        | Auto incremented  identifier |
 | collection_timestamp | text              | x        |        |        | x       |        | Unix timestamp of collection |
-| digest               | bytea             | x        |        |        | x       | x      | Timestamped digest           |
+| digest               | bytea             | x        | x      |        | x       | x      | Timestamped digest           |
 | anchor_merkle        | char. varying(64) |          |        | x      | x       |        | Anchor merkle root           |
 
 **Note:** `anchor_merkle` linking to anchors table, nil if not anchored yet
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 4a3e6fe..1fd9659 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -82,9 +82,8 @@ func (pg *Postgres) createRecordsTable() error {
 (
     digest bytea NOT NULL,
     anchor_merkle character varying(64) COLLATE pg_catalog."default",
-    key serial NOT NULL,
     collection_timestamp bigint NOT NULL,
-    CONSTRAINT records_pkey PRIMARY KEY (key),
+    CONSTRAINT records_pkey PRIMARY KEY (digest),
     CONSTRAINT records_anchors_fkey FOREIGN KEY (anchor_merkle)
         REFERENCES public.anchors (merkle) MATCH SIMPLE
         ON UPDATE NO ACTION

From b6cddb6bff8eb2f4465279e74b4762b3b2bd9217 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 23:23:55 +0300
Subject: [PATCH 26/66] Implement Close interface func

---
 dcrtimed/backend/postgres/postgres.go | 16 +++++++++++++++-
 dcrtimed/backend/postgres/schema.md   |  2 +-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 41af35c..be658a0 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -168,8 +168,22 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 	return ts, me, nil
 }
 
-// Close performs cleanup of the backend.
+// Close performs cleanup of the backend. In our case closes postgres
+// connection
 func (pg *Postgres) Close() {
+	// Block until last command is complete.
+	pg.Lock()
+	defer pg.Unlock()
+	defer log.Infof("Exiting")
+
+	// We need nil tests when in dump/restore mode.
+	if pg.cron != nil {
+		pg.cron.Stop()
+	}
+	if pg.wallet != nil {
+		pg.wallet.Close()
+	}
+	pg.db.Close()
 }
 
 // Dump dumps database to the provided file descriptor. If the
diff --git a/dcrtimed/backend/postgres/schema.md b/dcrtimed/backend/postgres/schema.md
index 596072a..8fd7704 100644
--- a/dcrtimed/backend/postgres/schema.md
+++ b/dcrtimed/backend/postgres/schema.md
@@ -16,7 +16,7 @@ records table, below you find the detailed description of the two tables:
 **Records:**
 | Col Name             | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                  |
 |----------------------|-------------------|----------|--------|--------|---------|--------|------------------------------|
-| collection_timestamp | text              | x        |        |        | x       |        | Unix timestamp of collection |
+| collection_timestamp | bigint            | x        |        |        | x       |        | Unix timestamp of collection |
 | digest               | bytea             | x        | x      |        | x       | x      | Timestamped digest           |
 | anchor_merkle        | char. varying(64) |          |        | x      | x       |        | Anchor merkle root           |
 

From 0e893c466ceaa3897e20e4d2182dbb038d85e2fe Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 11 Aug 2020 23:40:25 +0300
Subject: [PATCH 27/66] add GetBalance impl.

---
 dcrtimed/backend/postgres/postgres.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index be658a0..fe563be 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -210,7 +210,15 @@ func (pg *Postgres) Fsck(*backend.FsckOptions) error {
 // GetBalance retrieves balance information for the wallet
 // backing this instance
 func (pg *Postgres) GetBalance() (*backend.GetBalanceResult, error) {
-	return nil, nil
+	result, err := pg.wallet.GetWalletBalance()
+	if err != nil {
+		return nil, err
+	}
+	return &backend.GetBalanceResult{
+		Total:       result.Total,
+		Spendable:   result.Spendable,
+		Unconfirmed: result.Unconfirmed,
+	}, nil
 }
 
 // LastAnchor retrieves last successful anchor details

From 7f3d4a7c52cffd0424c3539cf9b63730150bfcc4 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 12 Aug 2020 21:47:47 +0300
Subject: [PATCH 28/66] get wip

---
 dcrtimed/backend/postgres/postgres.go | 28 ++++++++++++++++++++++++---
 dcrtimed/backend/postgres/sql.go      | 25 ++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index fe563be..82d4eec 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -75,8 +75,30 @@ func (pg *Postgres) truncate(t time.Time, d time.Duration) time.Time {
 }
 
 // Return timestamp information for given digests.
-func (pg *Postgres) Get([][sha256.Size]byte) ([]backend.GetResult, error) {
-	return nil, nil
+func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error) {
+	gdmes := make([]backend.GetResult, 0, len(digests))
+
+	// We need to be read locked from here on out.  Note that we are not
+	// locking/releasing.  This is by design in order to let all readers
+	// finish before a potential write occurs.
+	pg.RLock()
+	defer pg.RUnlock()
+
+	// Iterate over digests and translate results to backend interface.
+	for _, d := range digests {
+		gdme := backend.GetResult{
+			Digest: d,
+		}
+		found, err := pg.getRecordByDigest(d[:], &gdme)
+		if err != nil {
+			return nil, err
+		}
+		if !found {
+			gdme.ErrorCode = backend.ErrorNotFound
+		}
+		gdmes = append(gdmes, gdme)
+	}
+	return gdmes, nil
 }
 
 // Return all hashes for given timestamps.
@@ -273,7 +295,7 @@ func internalNew(host, net, rootCert, cert, key string) (*Postgres, error) {
 // New creates a new backend instance.  The caller should issue a Close once
 // the Postgres backend is no longer needed.
 func New(host, net, rootCert, cert, key, walletCert, walletHost string, enableCollections bool, walletPassphrase []byte) (*Postgres, error) {
-	log.Tracef("New: %v %v %v %v %v %v", host, net, rootCert, cert, key)
+	log.Tracef("New: %v %v %v %v %v", host, net, rootCert, cert, key)
 
 	pg, err := internalNew(host, net, rootCert, cert, key)
 	if err != nil {
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 1fd9659..f75b398 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -1,5 +1,30 @@
 package postgres
 
+import (
+	"github.com/decred/dcrtime/dcrtimed/backend"
+)
+
+func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool, error) {
+	q := "SELECT collection_timestamp FROM records WHERE digest = $1"
+
+	rows, err := pg.db.Query(q, hash)
+	if err != nil {
+		return false, err
+	}
+	defer rows.Close()
+
+	var ts int64
+	for rows.Next() {
+		err = rows.Scan(&ts)
+		if err != nil {
+			return false, err
+		}
+		(*r).Timestamp = ts
+	}
+
+	return true, nil
+}
+
 func (pg *Postgres) hasTable(name string) (bool, error) {
 	rows, err := pg.db.Query(`SELECT EXISTS (SELECT FROM information_schema.tables 
 		WHERE table_schema = 'public' AND table_name  = $1)`, name)

From 6328827237481c4427cb2e9167fd5bdb5ce83170 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 12 Aug 2020 22:00:27 +0300
Subject: [PATCH 29/66] handle digest doesn't exist case

---
 dcrtimed/backend/postgres/sql.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index f75b398..bd3da0e 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -20,9 +20,10 @@ func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool,
 			return false, err
 		}
 		(*r).Timestamp = ts
+		return true, nil
 	}
 
-	return true, nil
+	return false, nil
 }
 
 func (pg *Postgres) hasTable(name string) (bool, error) {

From 34b3dfc39cc46e579bd9bac04701dc9900d3bc5f Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Thu, 13 Aug 2020 02:53:50 +0300
Subject: [PATCH 30/66] add missing cols to get & change merkle root col type
 to bytea

---
 dcrtimed/backend/postgres/postgres.go |  1 +
 dcrtimed/backend/postgres/sql.go      | 38 ++++++++++++++++++++-------
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 82d4eec..cad4799 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -96,6 +96,7 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 		if !found {
 			gdme.ErrorCode = backend.ErrorNotFound
 		}
+		log.Infof("ffffffferererer: %v, %v", gdme.ErrorCode, gdme.Timestamp)
 		gdmes = append(gdmes, gdme)
 	}
 	return gdmes, nil
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index bd3da0e..f08310b 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -1,11 +1,19 @@
 package postgres
 
 import (
+	"crypto/sha256"
+	"database/sql"
+
 	"github.com/decred/dcrtime/dcrtimed/backend"
 )
 
 func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool, error) {
-	q := "SELECT collection_timestamp FROM records WHERE digest = $1"
+	q := `SELECT r.anchor_merkle, r.collection_timestamp, an.tx_hash, 
+an.chain_timestamp
+FROM records as r
+LEFT JOIN anchors as an
+ON r.anchor_merkle = an.merkle
+WHERE digest = $1`
 
 	rows, err := pg.db.Query(q, hash)
 	if err != nil {
@@ -13,13 +21,24 @@ func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool,
 	}
 	defer rows.Close()
 
-	var ts int64
+	var merkle []byte
+	var txHash sql.NullString
+	var chainTs sql.NullInt64
+	var serverTs int64
 	for rows.Next() {
-		err = rows.Scan(&ts)
+		err = rows.Scan(&merkle, &serverTs, &txHash, &chainTs)
 		if err != nil {
 			return false, err
 		}
-		(*r).Timestamp = ts
+		(*r).Timestamp = serverTs
+		copy(merkle[:], (*r).MerkleRoot[:sha256.Size])
+		// txHash & chainTs can be NULL - handle safely
+		if txHash.Valid {
+			copy((*r).Tx[:], []byte(txHash.String))
+		}
+		if chainTs.Valid {
+			(*r).AnchoredTimestamp = chainTs.Int64
+		}
 		return true, nil
 	}
 
@@ -27,7 +46,8 @@ func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool,
 }
 
 func (pg *Postgres) hasTable(name string) (bool, error) {
-	rows, err := pg.db.Query(`SELECT EXISTS (SELECT FROM information_schema.tables 
+	rows, err := pg.db.Query(`SELECT EXISTS (SELECT 
+		FROM information_schema.tables 
 		WHERE table_schema = 'public' AND table_name  = $1)`, name)
 	if err != nil {
 		return false, err
@@ -63,7 +83,7 @@ func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
 func (pg *Postgres) createAnchorsTable() error {
 	_, err := pg.db.Exec(`CREATE TABLE public.anchors
 (
-    merkle character varying(64) COLLATE pg_catalog."default" NOT NULL,
+    merkle bytea NOT NULL,
     hashes text[] COLLATE pg_catalog."default" NOT NULL,
     tx_hash text COLLATE pg_catalog."default",
     chain_timestamp bigint,
@@ -88,7 +108,7 @@ CREATE UNIQUE INDEX idx_hashes
 -- Index: idx_merkle
 CREATE UNIQUE INDEX idx_merkle
     ON public.anchors USING btree
-    (merkle COLLATE pg_catalog."default" ASC NULLS LAST)
+		(merkle ASC NULLS LAST)
     TABLESPACE pg_default;
 -- Index: idx_tx_hash
 CREATE UNIQUE INDEX idx_tx_hash
@@ -107,7 +127,7 @@ func (pg *Postgres) createRecordsTable() error {
 	_, err := pg.db.Exec(`CREATE TABLE public.records
 (
     digest bytea NOT NULL,
-    anchor_merkle character varying(64) COLLATE pg_catalog."default",
+    anchor_merkle bytea,
     collection_timestamp bigint NOT NULL,
     CONSTRAINT records_pkey PRIMARY KEY (digest),
     CONSTRAINT records_anchors_fkey FOREIGN KEY (anchor_merkle)
@@ -120,7 +140,7 @@ func (pg *Postgres) createRecordsTable() error {
 -- Index: fki_records_anchors_fkey
 CREATE INDEX fki_records_anchors_fkey
     ON public.records USING btree
-    (anchor_merkle COLLATE pg_catalog."default" ASC NULLS LAST)
+    (anchor_merkle ASC NULLS LAST)
     TABLESPACE pg_default;
 -- Index: idx_collection_timestamp
 CREATE INDEX idx_collection_timestamp

From 59c339a3cf5a7ea125b8f842051870ed429bd82f Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Thu, 13 Aug 2020 19:55:25 +0300
Subject: [PATCH 31/66] ErrorCode = backend.ErrorOK when digest found in Get

---
 dcrtimed/backend/postgres/postgres.go | 2 --
 dcrtimed/backend/postgres/sql.go      | 1 +
 dcrtimed/dcrtimed.go                  | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index cad4799..8828a8f 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -96,7 +96,6 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 		if !found {
 			gdme.ErrorCode = backend.ErrorNotFound
 		}
-		log.Infof("ffffffferererer: %v, %v", gdme.ErrorCode, gdme.Timestamp)
 		gdmes = append(gdmes, gdme)
 	}
 	return gdmes, nil
@@ -110,7 +109,6 @@ func (pg *Postgres) GetTimestamps([]int64) ([]backend.TimestampResult, error) {
 // Store hashes and return timestamp and associated errors.  Put is
 // allowed to return transient errors.
 func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult, error) {
-
 	// Two-phase commit.
 	pg.Lock()
 	commit := pg.commit
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index f08310b..e0828f8 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -39,6 +39,7 @@ WHERE digest = $1`
 		if chainTs.Valid {
 			(*r).AnchoredTimestamp = chainTs.Int64
 		}
+		(*r).ErrorCode = backend.ErrorOK
 		return true, nil
 	}
 
diff --git a/dcrtimed/dcrtimed.go b/dcrtimed/dcrtimed.go
index c0ef211..d54cd47 100644
--- a/dcrtimed/dcrtimed.go
+++ b/dcrtimed/dcrtimed.go
@@ -1119,7 +1119,7 @@ func (d *DcrtimeStore) verifyV2(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Translate digest results.
+	// Translate digest result.
 	var dReply v2.VerifyDigest
 	if len(drs) != 0 {
 		dr := drs[len(drs)-1]

From 5731cc04b0ff74f616ab3667684e815d62057fcc Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Fri, 14 Aug 2020 02:30:59 +0300
Subject: [PATCH 32/66] add flush logic - collect unflushed reocrds and calc
 merkle and broadcase anchoring tx

---
 dcrtimed/backend/postgres/postgres.go | 119 +++++++++++++++++++++++---
 dcrtimed/backend/postgres/sql.go      |  41 +++++++++
 2 files changed, 150 insertions(+), 10 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 8828a8f..1f58d59 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -7,6 +7,7 @@ package postgres
 import (
 	"crypto/sha256"
 	"database/sql"
+	"errors"
 	"fmt"
 	"net/url"
 	"os"
@@ -16,6 +17,7 @@ import (
 
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/dcrtimewallet"
+	"github.com/decred/dcrtime/merkle"
 	"github.com/lib/pq"
 	_ "github.com/lib/pq"
 	"github.com/robfig/cron"
@@ -40,6 +42,8 @@ var (
 	// Seconds Minutes Hours Days Months DayOfWeek
 	flushSchedule = "10 0 * * * *" // On the hour + 10 seconds
 	duration      = time.Hour      // Default how often we combine digests
+
+	errEmptySet = errors.New("empty set")
 )
 
 // Postgres is a postgreSQL implementation of a backend, it stores all uploaded
@@ -291,6 +295,99 @@ func internalNew(host, net, rootCert, cert, key string) (*Postgres, error) {
 	return pg, nil
 }
 
+// doFlush gets all timestamps which have unflushed records and flushes them.
+// It skips current timestamp.
+// It returns the number of directories that were flushed.
+//
+// This must be called with the WRITE lock held.  We may have to consider
+// errors out of this function terminal.
+func (pg *Postgres) doFlush() (int, error) {
+	current := pg.now().Unix()
+
+	// Get timestamps with unflushed records.
+	// Exclude current timestamp.
+	tss, err := pg.getUnflushedTimestamps(current)
+	if err != nil {
+		return 0, err
+	}
+	count := 0
+	// Flush timestamps' records
+	for _, ts := range tss {
+		err = pg.flush(ts)
+		if err != nil {
+			e := fmt.Sprintf("flush %v: %v", ts, err)
+			if pg.testing {
+				panic(e)
+			}
+			log.Error(e)
+		} else {
+			count++
+		}
+	}
+
+	return count, nil
+}
+
+// flusher is called periodically to flush the current timestamp.
+func (pg *Postgres) flusher() {
+	// From this point on the operation must be atomic.
+	pg.Lock()
+	defer pg.Unlock()
+	start := time.Now()
+	count, err := pg.doFlush()
+	end := time.Since(start)
+	if err != nil {
+		log.Errorf("flusher: %v", err)
+	}
+
+	log.Infof("Flusher: directories %v in %v", count, end)
+}
+
+// flush flushes all records associated with given timestamp.
+// returns nil iff ts records flushed successfully
+//
+// This function must be called with the WRITE lock held.
+func (pg *Postgres) flush(ts int64) error {
+	// Get timestamp's digests
+	digests, err := pg.getDigestsByTimestamp(ts)
+	if err != nil {
+		return err
+	}
+
+	if len(digests) == 0 {
+		// this really should not happen.
+		return errEmptySet
+	}
+
+	// Create merkle root and send to wallet
+	mt := merkle.Tree(digests)
+	root := *mt[len(mt)-1] // Last element is root
+	fr := backend.FlushRecord{
+		Root:           root,
+		Hashes:         mt[:len(digests)], // Only store hashes
+		FlushTimestamp: time.Now().Unix(),
+	}
+	if !pg.testing {
+		tx, err := pg.wallet.Construct(root)
+		if err != nil {
+			// XXX do something with unsufficient funds here.
+			return fmt.Errorf("flush Construct tx: %v", err)
+		}
+		log.Infof("Flush timestamp: %v digests %v merkle: %x tx: %v",
+			ts, len(digests), root, tx.String())
+		fr.Tx = *tx
+	}
+
+	// XXX Insert anchor data to db!
+
+	// XXX Update records merkle root
+
+	// Update commit.
+	pg.commit++
+
+	return nil
+}
+
 // New creates a new backend instance.  The caller should issue a Close once
 // the Postgres backend is no longer needed.
 func New(host, net, rootCert, cert, key, walletCert, walletHost string, enableCollections bool, walletPassphrase []byte) (*Postgres, error) {
@@ -309,20 +406,21 @@ func New(host, net, rootCert, cert, key, walletCert, walletHost string, enableCo
 		return nil, err
 	}
 
-	// Flushing backend reconciles uncommitted work to the global database.
-	//start := time.Now()
-	//flushed, err := pg.doFlush()
-	//end := time.Since(start)
-	//if err != nil {
-	//return nil, err
-	//}
+	// Flushing backend reconciles uncommitted work to the anchors table.
+	start := time.Now()
+	flushed, err := pg.doFlush()
+	end := time.Since(start)
+	if err != nil {
+		return nil, err
+	}
 
-	//if flushed != 0 {
-	//log.Infof("Startup flusher: directories %v in %v", flushed, end)
-	//}
+	if flushed != 0 {
+		log.Infof("Startup flusher: timestamps %v in %v", flushed, end)
+	}
 
 	// Launch cron.
 	err = pg.cron.AddFunc(flushSchedule, func() {
+		pg.flusher()
 	})
 	if err != nil {
 		return nil, err
@@ -331,4 +429,5 @@ func New(host, net, rootCert, cert, key, walletCert, walletHost string, enableCo
 	pg.cron.Start()
 
 	return pg, nil
+
 }
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index e0828f8..3837934 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -7,6 +7,47 @@ import (
 	"github.com/decred/dcrtime/dcrtimed/backend"
 )
 
+func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error) {
+	q := `SELECT digest from records WHERE collection_timestamp = $1`
+
+	rows, err := pg.db.Query(q, ts)
+	if err != nil {
+		return nil, err
+	}
+	var rawDigest []byte
+	var digest [sha256.Size]byte
+	digests := []*[sha256.Size]byte{}
+	for rows.Next() {
+		err = rows.Scan(&rawDigest)
+		if err != nil {
+			return nil, err
+		}
+		copy(digest[:], rawDigest[:])
+		digests = append(digests, &digest)
+	}
+	return digests, nil
+}
+
+func (pg *Postgres) getUnflushedTimestamps(current int64) ([]int64, error) {
+	q := `SELECT collection_timestamp FROM records 
+WHERE collection_timestamp != $1 AND anchor_merkle IS NULL`
+
+	rows, err := pg.db.Query(q, current)
+	if err != nil {
+		return nil, err
+	}
+	var ts int64
+	tss := []int64{}
+	for rows.Next() {
+		err = rows.Scan(&ts)
+		if err != nil {
+			return nil, err
+		}
+		tss = append(tss, ts)
+	}
+	return tss, nil
+}
+
 func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool, error) {
 	q := `SELECT r.anchor_merkle, r.collection_timestamp, an.tx_hash, 
 an.chain_timestamp

From c742744c8a8768eb03646a894c6ed1741db5ba7b Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Fri, 14 Aug 2020 14:54:10 +0300
Subject: [PATCH 33/66] insert anchor data into db

---
 dcrtimed/backend/postgres/postgres.go | 16 ++++++++++------
 dcrtimed/backend/postgres/schema.md   |  2 --
 dcrtimed/backend/postgres/sql.go      | 25 ++++++++++++++++---------
 3 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 1f58d59..6f19f1f 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -24,7 +24,6 @@ import (
 )
 
 const (
-	fStr         = "20060102.150405"
 	tableRecords = "records"
 	tableAnchors = "anchors"
 	dbUser       = "dcrtimed"
@@ -346,7 +345,7 @@ func (pg *Postgres) flusher() {
 // flush flushes all records associated with given timestamp.
 // returns nil iff ts records flushed successfully
 //
-// This function must be called with the WRITE lock held.
+// This function must be called with the WRITE lock held
 func (pg *Postgres) flush(ts int64) error {
 	// Get timestamp's digests
 	digests, err := pg.getDigestsByTimestamp(ts)
@@ -355,13 +354,14 @@ func (pg *Postgres) flush(ts int64) error {
 	}
 
 	if len(digests) == 0 {
-		// this really should not happen.
+		// This really should not happen
 		return errEmptySet
 	}
 
-	// Create merkle root and send to wallet
+	// Generate merkle
 	mt := merkle.Tree(digests)
-	root := *mt[len(mt)-1] // Last element is root
+	// Last element is root
+	root := *mt[len(mt)-1]
 	fr := backend.FlushRecord{
 		Root:           root,
 		Hashes:         mt[:len(digests)], // Only store hashes
@@ -378,7 +378,11 @@ func (pg *Postgres) flush(ts int64) error {
 		fr.Tx = *tx
 	}
 
-	// XXX Insert anchor data to db!
+	// Insert anchor data into db
+	err = pg.insertAnchor(fr)
+	if err != nil {
+		return err
+	}
 
 	// XXX Update records merkle root
 
diff --git a/dcrtimed/backend/postgres/schema.md b/dcrtimed/backend/postgres/schema.md
index 8fd7704..48286c0 100644
--- a/dcrtimed/backend/postgres/schema.md
+++ b/dcrtimed/backend/postgres/schema.md
@@ -10,7 +10,6 @@ in an `anchor` will be connected to the corresponding entry in the anchors
 table using the col `anchor_merkle` which defined as forgein key & indexed in 
 records table, below you find the detailed description of the two tables:
 
-
 ### Tables
 
 **Records:**
@@ -26,7 +25,6 @@ records table, below you find the detailed description of the two tables:
 | Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                     |
 |------------------|-------------------|----------|--------|--------|---------|--------|---------------------------------|
 | merkle           | char. varying(64) | x        | x      |        | x       | x      | Anchor merkle root              |
-| hashes           | text[]            | x        |        |        | x       | x      | Anchored hashes                 |
 | tx_hash          | text              |          |        |        | x       | x      | Anchor tx hash                  |
 | chain_timestamp  | bigint            |          |        |        |         |        | Anchor timestamp on blockchain  |
 | flush_timestamp  | bigint            |          |        |        |         |        | When anchor actually  flushed   |
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 3837934..000f8b6 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -7,6 +7,19 @@ import (
 	"github.com/decred/dcrtime/dcrtimed/backend"
 )
 
+func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
+	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp)
+VALUES($1, $2, $3)`
+
+	var root []byte
+	err := pg.db.QueryRow(q, copy(root[:], fr.Root[:]), fr.Tx.String(),
+		fr.FlushTimestamp).Scan()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error) {
 	q := `SELECT digest from records WHERE collection_timestamp = $1`
 
@@ -125,9 +138,8 @@ func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
 func (pg *Postgres) createAnchorsTable() error {
 	_, err := pg.db.Exec(`CREATE TABLE public.anchors
 (
-    merkle bytea NOT NULL,
-    hashes text[] COLLATE pg_catalog."default" NOT NULL,
-    tx_hash text COLLATE pg_catalog."default",
+    merkle bytea NOT NULL UNIQUE,
+    tx_hash text COLLATE pg_catalog."default" UNIQUE,
     chain_timestamp bigint,
     flush_timestamp bigint,
     CONSTRAINT anchors_pkey PRIMARY KEY (merkle)
@@ -142,11 +154,6 @@ CREATE INDEX idx_flush_timestamp
     ON public.anchors USING btree
     (flush_timestamp ASC NULLS LAST)
     TABLESPACE pg_default;
--- Index: idx_hashes
-CREATE UNIQUE INDEX idx_hashes
-    ON public.anchors USING btree
-    (hashes COLLATE pg_catalog."default" ASC NULLS LAST)
-    TABLESPACE pg_default;
 -- Index: idx_merkle
 CREATE UNIQUE INDEX idx_merkle
     ON public.anchors USING btree
@@ -168,7 +175,7 @@ CREATE UNIQUE INDEX idx_tx_hash
 func (pg *Postgres) createRecordsTable() error {
 	_, err := pg.db.Exec(`CREATE TABLE public.records
 (
-    digest bytea NOT NULL,
+    digest bytea NOT NULL UNIQUE,
     anchor_merkle bytea,
     collection_timestamp bigint NOT NULL,
     CONSTRAINT records_pkey PRIMARY KEY (digest),

From 0bac395188ca18095dbab29b6dd7bcf6ee74807b Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 15 Aug 2020 02:22:56 +0300
Subject: [PATCH 34/66] fix anchor merkle insert and update records' merkle col

---
 dcrtimed/backend/postgres/postgres.go |  3 ++-
 dcrtimed/backend/postgres/sql.go      | 19 +++++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 6f19f1f..66c3946 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -384,7 +384,8 @@ func (pg *Postgres) flush(ts int64) error {
 		return err
 	}
 
-	// XXX Update records merkle root
+	// Update timestamp's records merkle root
+	pg.updateRecordsAnchor(ts, fr.Root)
 
 	// Update commit.
 	pg.commit++
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 000f8b6..698d473 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -7,14 +7,29 @@ import (
 	"github.com/decred/dcrtime/dcrtimed/backend"
 )
 
+func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot [sha256.Size]byte) error {
+	q := `UPDATE records SET anchor_merkle = $1
+	WHERE collection_timestamp = $2`
+
+	err := pg.db.QueryRow(q, merkleRoot[:], ts).Scan()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
 	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp)
 VALUES($1, $2, $3)`
 
-	var root []byte
-	err := pg.db.QueryRow(q, copy(root[:], fr.Root[:]), fr.Tx.String(),
+	err := pg.db.QueryRow(q, fr.Root[:], fr.Tx.String(),
 		fr.FlushTimestamp).Scan()
 	if err != nil {
+		// The insert command won't return any value, the following error is
+		// expected and means anchor row inserted successfully
+		if err.Error() == "sql: no rows in result set" {
+			return nil
+		}
 		return err
 	}
 	return nil

From e8d5d7c7e7e55edab0327059e26fae620e6c69b6 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 16 Aug 2020 02:01:29 +0300
Subject: [PATCH 35/66] lazy flush anchor on digest get when chain timestamp
 isn't there yet / waiting for 6 confirmations

---
 dcrtimed/backend/postgres/postgres.go | 81 ++++++++++++++++++++++++---
 dcrtimed/backend/postgres/sql.go      | 69 ++++++++++++++++++++---
 2 files changed, 133 insertions(+), 17 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 66c3946..7386ff0 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -24,12 +24,14 @@ import (
 )
 
 const (
-	tableRecords = "records"
-	tableAnchors = "anchors"
-	dbUser       = "dcrtimed"
-
-	// errorFound is thrown if digest was found in records table
-	errorFound = 1001
+	tableRecords  = "records"
+	tableAnchors  = "anchors"
+	dbUser        = "dcrtimed"
+	confirmations = 6
+
+	// error codes that are overridden during tests only.
+	// digestFound is thrown if digest was found in records table
+	digestFound = 1001
 )
 
 var (
@@ -77,6 +79,49 @@ func (pg *Postgres) truncate(t time.Time, d time.Duration) time.Time {
 	return t.Truncate(d)
 }
 
+var (
+	errInvalidConfirmations  = errors.New("invalid confirmations")
+	errNotEnoughConfirmation = errors.New("not enough confirmations")
+)
+
+// lazyFlush takes a pointer to a flush record and updates the chain anchor
+// timestamp of said record and writes it back to the database and returns
+// the result of the wallet's Lookup function
+//
+// IMPORTANT NOTE: We *may* write to the anchors database in case of a lazy
+// timestamp update to the anchor timestamo while holding the READ lock.  This
+// is OK because at worst we are racing multiple atomic writes.
+// This is suboptimal but beats taking a write lock for all get* calls.
+func (pg *Postgres) lazyFlush(fr *backend.FlushRecord) (*dcrtimewallet.TxLookupResult, error) {
+	res, err := pg.wallet.Lookup((*fr).Tx)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf("lazyFlush confirmations: %v", res.Confirmations)
+
+	if res.Confirmations == -1 {
+		return nil, errInvalidConfirmations
+	} else if res.Confirmations < confirmations {
+		// Return error & wallet lookup res
+		// for error handling
+		return res, errNotEnoughConfirmation
+	}
+
+	fr.ChainTimestamp = res.Timestamp
+
+	// Update anchor row in database
+	err = pg.updateAnchorChainTs(fr)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Flushed anchor timestamp: %v %v", fr.Tx.String(),
+		res.Timestamp)
+
+	return res, nil
+}
+
 // Return timestamp information for given digests.
 func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error) {
 	gdmes := make([]backend.GetResult, 0, len(digests))
@@ -96,8 +141,26 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 		if err != nil {
 			return nil, err
 		}
+
 		if !found {
 			gdme.ErrorCode = backend.ErrorNotFound
+		} else {
+			// Override error code during testing
+			if pg.testing {
+				gdme.ErrorCode = digestFound
+				// Lazyflush records if was anchored but blockchain isn't
+				// avialable yet
+			} else if gdme.Tx.String() != "" && gdme.AnchoredTimestamp == 0 {
+				// Lazyflush records if was anchored but blockchain isn't
+				// avialable yet
+				_, err = pg.lazyFlush(&backend.FlushRecord{
+					Tx:   gdme.Tx,
+					Root: gdme.MerkleRoot,
+				})
+				if err != nil {
+					return nil, err
+				}
+			}
 		}
 		gdmes = append(gdmes, gdme)
 	}
@@ -149,7 +212,7 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 
 			// Override error code during testing
 			if pg.testing {
-				me[len(me)-1].ErrorCode = errorFound
+				me[len(me)-1].ErrorCode = digestFound
 			}
 			continue
 		}
@@ -296,7 +359,7 @@ func internalNew(host, net, rootCert, cert, key string) (*Postgres, error) {
 
 // doFlush gets all timestamps which have unflushed records and flushes them.
 // It skips current timestamp.
-// It returns the number of directories that were flushed.
+// It returns the number of timestamps that were flushed.
 //
 // This must be called with the WRITE lock held.  We may have to consider
 // errors out of this function terminal.
@@ -339,7 +402,7 @@ func (pg *Postgres) flusher() {
 		log.Errorf("flusher: %v", err)
 	}
 
-	log.Infof("Flusher: directories %v in %v", count, end)
+	log.Infof("Flusher: timestamps %v in %v", count, end)
 }
 
 // flush flushes all records associated with given timestamp.
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 698d473..201f208 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -4,9 +4,26 @@ import (
 	"crypto/sha256"
 	"database/sql"
 
+	"github.com/decred/dcrd/chaincfg/chainhash"
 	"github.com/decred/dcrtime/dcrtimed/backend"
+	"github.com/decred/dcrtime/merkle"
 )
 
+func (pg *Postgres) updateAnchorChainTs(fr *backend.FlushRecord) error {
+	q := `UPDATE anchors SET chain_timestamp = $1
+	WHERE merkle = $2`
+	err := pg.db.QueryRow(q, fr.ChainTimestamp, fr.Root[:]).Scan()
+	if err != nil {
+		// The insert command won't return any value, the following error is
+		// expected and means anchor row inserted successfully
+		if err.Error() == "sql: no rows in result set" {
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
 func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot [sha256.Size]byte) error {
 	q := `UPDATE records SET anchor_merkle = $1
 	WHERE collection_timestamp = $2`
@@ -20,7 +37,7 @@ func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot [sha256.Size]byte)
 
 func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
 	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp)
-VALUES($1, $2, $3)`
+	VALUES($1, $2, $3)`
 
 	err := pg.db.QueryRow(q, fr.Root[:], fr.Tx.String(),
 		fr.FlushTimestamp).Scan()
@@ -35,6 +52,28 @@ VALUES($1, $2, $3)`
 	return nil
 }
 
+func (pg *Postgres) getDigestsByMerkleRoot(merkle []byte) ([]*[sha256.Size]byte, error) {
+	q := `SELECT digest from records WHERE anchor_merkle = $1`
+
+	rows, err := pg.db.Query(q, merkle)
+	if err != nil {
+		return nil, err
+	}
+	var rawDigest []byte
+	var digest [sha256.Size]byte
+	digests := []*[sha256.Size]byte{}
+	for rows.Next() {
+		err = rows.Scan(&rawDigest)
+		if err != nil {
+			return nil, err
+		}
+		copy(digest[:], rawDigest[:])
+		digests = append(digests, &digest)
+	}
+	return digests, nil
+
+}
+
 func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error) {
 	q := `SELECT digest from records WHERE collection_timestamp = $1`
 
@@ -90,25 +129,39 @@ WHERE digest = $1`
 	}
 	defer rows.Close()
 
-	var merkle []byte
+	var mr []byte
 	var txHash sql.NullString
 	var chainTs sql.NullInt64
 	var serverTs int64
 	for rows.Next() {
-		err = rows.Scan(&merkle, &serverTs, &txHash, &chainTs)
+		err = rows.Scan(&mr, &serverTs, &txHash, &chainTs)
 		if err != nil {
 			return false, err
 		}
-		(*r).Timestamp = serverTs
-		copy(merkle[:], (*r).MerkleRoot[:sha256.Size])
+		r.Timestamp = serverTs
+		copy(r.MerkleRoot[:sha256.Size], mr[:])
 		// txHash & chainTs can be NULL - handle safely
 		if txHash.Valid {
-			copy((*r).Tx[:], []byte(txHash.String))
+			tx, err := chainhash.NewHashFromStr(txHash.String)
+			if err != nil {
+				return false, err
+			}
+			r.Tx = *tx
 		}
 		if chainTs.Valid {
-			(*r).AnchoredTimestamp = chainTs.Int64
+			r.AnchoredTimestamp = chainTs.Int64
+		}
+		if mr != nil {
+			hashes, err := pg.getDigestsByMerkleRoot(mr)
+			if err != nil {
+				return false, err
+			}
+			var digest [sha256.Size]byte
+			copy(digest[:], hash[:])
+			// That pointer better not be nil!
+			r.MerklePath = *merkle.AuthPath(hashes, &digest)
 		}
-		(*r).ErrorCode = backend.ErrorOK
+		r.ErrorCode = backend.ErrorOK
 		return true, nil
 	}
 

From aef715eacea9e16ab5fb4723a50de35dec48bfdf Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 16 Aug 2020 13:44:33 +0300
Subject: [PATCH 36/66] cleanup lazyflush

---
 dcrtimed/backend/postgres/postgres.go | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 7386ff0..be0a88d 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -19,7 +19,6 @@ import (
 	"github.com/decred/dcrtime/dcrtimed/dcrtimewallet"
 	"github.com/decred/dcrtime/merkle"
 	"github.com/lib/pq"
-	_ "github.com/lib/pq"
 	"github.com/robfig/cron"
 )
 
@@ -148,18 +147,27 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 			// Override error code during testing
 			if pg.testing {
 				gdme.ErrorCode = digestFound
-				// Lazyflush records if was anchored but blockchain isn't
-				// avialable yet
-			} else if gdme.Tx.String() != "" && gdme.AnchoredTimestamp == 0 {
-				// Lazyflush records if was anchored but blockchain isn't
-				// avialable yet
-				_, err = pg.lazyFlush(&backend.FlushRecord{
+			} else if gdme.MerkleRoot != [sha256.Size]byte{} && gdme.AnchoredTimestamp == 0 {
+				// Lazyflush record if it was anchored but blockchain timestamp
+				// isn't avialable yet
+				fr := backend.FlushRecord{
 					Tx:   gdme.Tx,
 					Root: gdme.MerkleRoot,
-				})
+				}
+				_, err = pg.lazyFlush(&fr)
 				if err != nil {
-					return nil, err
+					switch err {
+					case errNotEnoughConfirmation:
+						// All good, continue without blockchain timestamp
+					case errInvalidConfirmations:
+						log.Errorf("%v: Confirmations = -1",
+							gdme.Tx.String())
+						return nil, err
+					default:
+						return nil, err
+					}
 				}
+				gdme.AnchoredTimestamp = fr.ChainTimestamp
 			}
 		}
 		gdmes = append(gdmes, gdme)

From 6ea6e68e85c23be98f98f3b8acb0293566c1f3c2 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Mon, 17 Aug 2020 16:45:17 +0300
Subject: [PATCH 37/66] - Get DISTINCT timestamps when flushing, - Reverse
 hashes order - generating merkle root & path

---
 dcrtimed/backend/postgres/sql.go | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 201f208..067f892 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -59,19 +59,22 @@ func (pg *Postgres) getDigestsByMerkleRoot(merkle []byte) ([]*[sha256.Size]byte,
 	if err != nil {
 		return nil, err
 	}
-	var rawDigest []byte
-	var digest [sha256.Size]byte
-	digests := []*[sha256.Size]byte{}
+	var digests []*[sha256.Size]byte
 	for rows.Next() {
+		var rawDigest []byte
 		err = rows.Scan(&rawDigest)
 		if err != nil {
 			return nil, err
 		}
+		var digest [sha256.Size]byte
 		copy(digest[:], rawDigest[:])
 		digests = append(digests, &digest)
 	}
+	// Reverse hashes
+	for i, j := 0, len(digests)-1; i < j; i, j = i+1, j-1 {
+		digests[i], digests[j] = digests[j], digests[i]
+	}
 	return digests, nil
-
 }
 
 func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error) {
@@ -81,14 +84,14 @@ func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error
 	if err != nil {
 		return nil, err
 	}
-	var rawDigest []byte
-	var digest [sha256.Size]byte
-	digests := []*[sha256.Size]byte{}
+	var digests []*[sha256.Size]byte
 	for rows.Next() {
+		var rawDigest []byte
 		err = rows.Scan(&rawDigest)
 		if err != nil {
 			return nil, err
 		}
+		var digest [sha256.Size]byte
 		copy(digest[:], rawDigest[:])
 		digests = append(digests, &digest)
 	}
@@ -96,7 +99,7 @@ func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error
 }
 
 func (pg *Postgres) getUnflushedTimestamps(current int64) ([]int64, error) {
-	q := `SELECT collection_timestamp FROM records 
+	q := `SELECT DISTINCT collection_timestamp FROM records 
 WHERE collection_timestamp != $1 AND anchor_merkle IS NULL`
 
 	rows, err := pg.db.Query(q, current)
@@ -139,7 +142,7 @@ WHERE digest = $1`
 			return false, err
 		}
 		r.Timestamp = serverTs
-		copy(r.MerkleRoot[:sha256.Size], mr[:])
+		copy(r.MerkleRoot[:], mr[:sha256.Size])
 		// txHash & chainTs can be NULL - handle safely
 		if txHash.Valid {
 			tx, err := chainhash.NewHashFromStr(txHash.String)
@@ -158,6 +161,7 @@ WHERE digest = $1`
 			}
 			var digest [sha256.Size]byte
 			copy(digest[:], hash[:])
+
 			// That pointer better not be nil!
 			r.MerklePath = *merkle.AuthPath(hashes, &digest)
 		}

From 8abb29b11d0169a1b49f084c74839f293508d8aa Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 18 Aug 2020 03:01:20 +0300
Subject: [PATCH 38/66] implement GetTimestamps interface func

---
 dcrtimed/backend/postgres/postgres.go | 70 ++++++++++++++++++++++++--
 dcrtimed/backend/postgres/sql.go      | 71 +++++++++++++++++++++++----
 2 files changed, 129 insertions(+), 12 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index be0a88d..2a723e4 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -175,9 +175,73 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 	return gdmes, nil
 }
 
-// Return all hashes for given timestamps.
-func (pg *Postgres) GetTimestamps([]int64) ([]backend.TimestampResult, error) {
-	return nil, nil
+// GetTimestamps is a required interface function.  In our case it retrieves
+// the digests for a given timestamp.
+//
+// GetTimestamps satisfies the backend interface.
+func (pg *Postgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult, error) {
+	gtmes := make([]backend.TimestampResult, 0, len(timestamps))
+
+	// We need to be read locked from here on out.  Note that we are not
+	// locking/releasing.  This is by design in order to let all readers
+	// finish before a potential write occurs.
+	pg.RLock()
+	defer pg.RUnlock()
+
+	// Iterate over timestamps and translate results to backend interface.
+	for _, ts := range timestamps {
+		gtme := backend.TimestampResult{
+			Timestamp: ts,
+		}
+		if pg.enableCollections {
+			exists, tsDigests, err := pg.getRecordsByServerTs(ts)
+			if err != nil {
+				return nil, err
+			}
+			gtme = backend.TimestampResult{
+				ErrorCode: backend.ErrorOK,
+			}
+			if !exists {
+				gtme.ErrorCode = backend.ErrorNotFound
+			}
+			// copy ts digests
+			gtme.Digests = make([][sha256.Size]byte, 0, len(tsDigests))
+			for _, digest := range tsDigests {
+				gtme.Digests = append(gtme.Digests, (*digest).Digest)
+				gtme.Tx = (*digest).Tx
+				gtme.AnchoredTimestamp = (*digest).AnchoredTimestamp
+				gtme.MerkleRoot = (*digest).MerkleRoot
+			}
+			// Lazyflush record if it was anchored but blockchain timestamp
+			// isn't avialable yet
+			if gtme.MerkleRoot != [sha256.Size]byte{} && gtme.AnchoredTimestamp == 0 {
+				fr := backend.FlushRecord{
+					Tx:   gtme.Tx,
+					Root: gtme.MerkleRoot,
+				}
+				_, err = pg.lazyFlush(&fr)
+				if err != nil {
+					switch err {
+					case errNotEnoughConfirmation:
+						// All good, continue without blockchain timestamp
+					case errInvalidConfirmations:
+						log.Errorf("%v: Confirmations = -1",
+							gtme.Tx.String())
+						return nil, err
+					default:
+						return nil, err
+					}
+				}
+				gtme.AnchoredTimestamp = fr.ChainTimestamp
+			}
+		} else {
+			gtme = backend.TimestampResult{
+				ErrorCode: backend.ErrorNotAllowed,
+			}
+		}
+		gtmes = append(gtmes, gtme)
+	}
+	return gtmes, nil
 }
 
 // Store hashes and return timestamp and associated errors.  Put is
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 067f892..ef8bed2 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -11,7 +11,8 @@ import (
 
 func (pg *Postgres) updateAnchorChainTs(fr *backend.FlushRecord) error {
 	q := `UPDATE anchors SET chain_timestamp = $1
-	WHERE merkle = $2`
+WHERE merkle = $2`
+
 	err := pg.db.QueryRow(q, fr.ChainTimestamp, fr.Root[:]).Scan()
 	if err != nil {
 		// The insert command won't return any value, the following error is
@@ -26,7 +27,7 @@ func (pg *Postgres) updateAnchorChainTs(fr *backend.FlushRecord) error {
 
 func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot [sha256.Size]byte) error {
 	q := `UPDATE records SET anchor_merkle = $1
-	WHERE collection_timestamp = $2`
+WHERE collection_timestamp = $2`
 
 	err := pg.db.QueryRow(q, merkleRoot[:], ts).Scan()
 	if err != nil {
@@ -37,7 +38,7 @@ func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot [sha256.Size]byte)
 
 func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
 	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp)
-	VALUES($1, $2, $3)`
+VALUES($1, $2, $3)`
 
 	err := pg.db.QueryRow(q, fr.Root[:], fr.Tx.String(),
 		fr.FlushTimestamp).Scan()
@@ -118,13 +119,61 @@ WHERE collection_timestamp != $1 AND anchor_merkle IS NULL`
 	return tss, nil
 }
 
+func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult, error) {
+	q := `SELECT r.anchor_merkle, an.tx_hash, an.chain_timestamp, r.digest
+FROM records as r
+LEFT JOIN anchors as an
+on r.achor_merkle = an.merkle
+WHERE r.collection_timestamp = $1`
+
+	rows, err := pg.db.Query(q, ts)
+	if err != nil {
+		return false, nil, err
+	}
+	defer rows.Close()
+	var (
+		mr      []byte
+		digest  []byte
+		txHash  sql.NullString
+		chainTs sql.NullInt64
+	)
+	r := []*backend.GetResult{}
+	for rows.Next() {
+		rr := backend.GetResult{
+			Timestamp: ts,
+		}
+		err = rows.Scan(&mr, &txHash, &chainTs, &digest)
+		if err != nil {
+			return false, nil, err
+		}
+		rr.Timestamp = ts
+		copy(rr.MerkleRoot[:], mr[:sha256.Size])
+		// txHash & chainTs can be NULL - handle safely
+		if txHash.Valid {
+			tx, err := chainhash.NewHashFromStr(txHash.String)
+			if err != nil {
+				return false, nil, err
+			}
+			rr.Tx = *tx
+		}
+		if chainTs.Valid {
+			rr.AnchoredTimestamp = chainTs.Int64
+		}
+		copy(rr.Digest[:], digest[:])
+
+		r = append(r, &rr)
+	}
+
+	return len(r) > 0, r, nil
+}
+
 func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool, error) {
 	q := `SELECT r.anchor_merkle, r.collection_timestamp, an.tx_hash, 
 an.chain_timestamp
 FROM records as r
 LEFT JOIN anchors as an
 ON r.anchor_merkle = an.merkle
-WHERE digest = $1`
+WHERE r.digest = $1`
 
 	rows, err := pg.db.Query(q, hash)
 	if err != nil {
@@ -173,9 +222,11 @@ WHERE digest = $1`
 }
 
 func (pg *Postgres) hasTable(name string) (bool, error) {
-	rows, err := pg.db.Query(`SELECT EXISTS (SELECT 
-		FROM information_schema.tables 
-		WHERE table_schema = 'public' AND table_name  = $1)`, name)
+	q := `SELECT EXISTS (SELECT 
+FROM information_schema.tables 
+WHERE table_schema = 'public' AND table_name  = $1)`
+
+	rows, err := pg.db.Query(q, name)
 	if err != nil {
 		return false, err
 	}
@@ -191,8 +242,10 @@ func (pg *Postgres) hasTable(name string) (bool, error) {
 }
 
 func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
-	rows, err := pg.db.Query(`SELECT EXISTS (SELECT FROM records 
-		WHERE digest = $1)`, hash)
+	q := `SELECT EXISTS 
+(SELECT FROM records WHERE digest = $1)`
+
+	rows, err := pg.db.Query(q, hash)
 	if err != nil {
 		return false, err
 	}

From 1beed9ef67e81825a81fcf8e55344ff3fe533725 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Tue, 18 Aug 2020 19:48:46 +0300
Subject: [PATCH 39/66] cleanup GetTimestamps func

---
 dcrtimed/backend/postgres/postgres.go | 8 ++------
 dcrtimed/backend/postgres/sql.go      | 2 +-
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 2a723e4..9e59fdf 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -198,9 +198,7 @@ func (pg *Postgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult
 			if err != nil {
 				return nil, err
 			}
-			gtme = backend.TimestampResult{
-				ErrorCode: backend.ErrorOK,
-			}
+			gtme.ErrorCode = backend.ErrorOK
 			if !exists {
 				gtme.ErrorCode = backend.ErrorNotFound
 			}
@@ -235,9 +233,7 @@ func (pg *Postgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult
 				gtme.AnchoredTimestamp = fr.ChainTimestamp
 			}
 		} else {
-			gtme = backend.TimestampResult{
-				ErrorCode: backend.ErrorNotAllowed,
-			}
+			gtme.ErrorCode = backend.ErrorNotAllowed
 		}
 		gtmes = append(gtmes, gtme)
 	}
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index ef8bed2..0a92873 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -123,7 +123,7 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult,
 	q := `SELECT r.anchor_merkle, an.tx_hash, an.chain_timestamp, r.digest
 FROM records as r
 LEFT JOIN anchors as an
-on r.achor_merkle = an.merkle
+on r.anchor_merkle = an.merkle
 WHERE r.collection_timestamp = $1`
 
 	rows, err := pg.db.Query(q, ts)

From 74eb33692b32852d20fa149c332e193faa3b924c Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 19 Aug 2020 02:58:25 +0300
Subject: [PATCH 40/66] update schema col types

---
 dcrtimed/backend/postgres/schema.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dcrtimed/backend/postgres/schema.md b/dcrtimed/backend/postgres/schema.md
index 48286c0..8887123 100644
--- a/dcrtimed/backend/postgres/schema.md
+++ b/dcrtimed/backend/postgres/schema.md
@@ -17,14 +17,14 @@ records table, below you find the detailed description of the two tables:
 |----------------------|-------------------|----------|--------|--------|---------|--------|------------------------------|
 | collection_timestamp | bigint            | x        |        |        | x       |        | Unix timestamp of collection |
 | digest               | bytea             | x        | x      |        | x       | x      | Timestamped digest           |
-| anchor_merkle        | char. varying(64) |          |        | x      | x       |        | Anchor merkle root           |
+| anchor_merkle        | bytea           ) |          |        | x      | x       |        | Anchor merkle root           |
 
 **Note:** `anchor_merkle` linking to anchors table, nil if not anchored yet
 
 **Anchors:**
 | Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                     |
 |------------------|-------------------|----------|--------|--------|---------|--------|---------------------------------|
-| merkle           | char. varying(64) | x        | x      |        | x       | x      | Anchor merkle root              |
+| merkle           | bytea             | x        | x      |        | x       | x      | Anchor merkle root              |
 | tx_hash          | text              |          |        |        | x       | x      | Anchor tx hash                  |
 | chain_timestamp  | bigint            |          |        |        |         |        | Anchor timestamp on blockchain  |
 | flush_timestamp  | bigint            |          |        |        |         |        | When anchor actually  flushed   |

From f369711c9c4df1510ee76aeb72d9e334a718c824 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 19 Aug 2020 03:16:09 +0300
Subject: [PATCH 41/66] query string indentation & tx hash col type bytea

---
 dcrtimed/backend/postgres/schema.md |  2 +-
 dcrtimed/backend/postgres/sql.go    | 36 ++++++++++++++---------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/dcrtimed/backend/postgres/schema.md b/dcrtimed/backend/postgres/schema.md
index 8887123..b931171 100644
--- a/dcrtimed/backend/postgres/schema.md
+++ b/dcrtimed/backend/postgres/schema.md
@@ -25,7 +25,7 @@ records table, below you find the detailed description of the two tables:
 | Col Name         | Type              | Not Null | P. Key | F. Key | Indexed | Unique | Description                     |
 |------------------|-------------------|----------|--------|--------|---------|--------|---------------------------------|
 | merkle           | bytea             | x        | x      |        | x       | x      | Anchor merkle root              |
-| tx_hash          | text              |          |        |        | x       | x      | Anchor tx hash                  |
+| tx_hash          | bytea             |          |        |        | x       | x      | Anchor tx hash                  |
 | chain_timestamp  | bigint            |          |        |        |         |        | Anchor timestamp on blockchain  |
 | flush_timestamp  | bigint            |          |        |        |         |        | When anchor actually  flushed   |
 
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 0a92873..cbf4cf7 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -11,7 +11,7 @@ import (
 
 func (pg *Postgres) updateAnchorChainTs(fr *backend.FlushRecord) error {
 	q := `UPDATE anchors SET chain_timestamp = $1
-WHERE merkle = $2`
+				WHERE merkle = $2`
 
 	err := pg.db.QueryRow(q, fr.ChainTimestamp, fr.Root[:]).Scan()
 	if err != nil {
@@ -27,7 +27,7 @@ WHERE merkle = $2`
 
 func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot [sha256.Size]byte) error {
 	q := `UPDATE records SET anchor_merkle = $1
-WHERE collection_timestamp = $2`
+				WHERE collection_timestamp = $2`
 
 	err := pg.db.QueryRow(q, merkleRoot[:], ts).Scan()
 	if err != nil {
@@ -38,7 +38,7 @@ WHERE collection_timestamp = $2`
 
 func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
 	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp)
-VALUES($1, $2, $3)`
+				VALUES($1, $2, $3)`
 
 	err := pg.db.QueryRow(q, fr.Root[:], fr.Tx.String(),
 		fr.FlushTimestamp).Scan()
@@ -101,7 +101,7 @@ func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error
 
 func (pg *Postgres) getUnflushedTimestamps(current int64) ([]int64, error) {
 	q := `SELECT DISTINCT collection_timestamp FROM records 
-WHERE collection_timestamp != $1 AND anchor_merkle IS NULL`
+				WHERE collection_timestamp != $1 AND anchor_merkle IS NULL`
 
 	rows, err := pg.db.Query(q, current)
 	if err != nil {
@@ -121,10 +121,10 @@ WHERE collection_timestamp != $1 AND anchor_merkle IS NULL`
 
 func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult, error) {
 	q := `SELECT r.anchor_merkle, an.tx_hash, an.chain_timestamp, r.digest
-FROM records as r
-LEFT JOIN anchors as an
-on r.anchor_merkle = an.merkle
-WHERE r.collection_timestamp = $1`
+				FROM records as r
+				LEFT JOIN anchors as an
+				on r.anchor_merkle = an.merkle
+				WHERE r.collection_timestamp = $1`
 
 	rows, err := pg.db.Query(q, ts)
 	if err != nil {
@@ -169,11 +169,11 @@ WHERE r.collection_timestamp = $1`
 
 func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool, error) {
 	q := `SELECT r.anchor_merkle, r.collection_timestamp, an.tx_hash, 
-an.chain_timestamp
-FROM records as r
-LEFT JOIN anchors as an
-ON r.anchor_merkle = an.merkle
-WHERE r.digest = $1`
+				an.chain_timestamp
+				FROM records as r
+				LEFT JOIN anchors as an
+				ON r.anchor_merkle = an.merkle
+				WHERE r.digest = $1`
 
 	rows, err := pg.db.Query(q, hash)
 	if err != nil {
@@ -223,8 +223,8 @@ WHERE r.digest = $1`
 
 func (pg *Postgres) hasTable(name string) (bool, error) {
 	q := `SELECT EXISTS (SELECT 
-FROM information_schema.tables 
-WHERE table_schema = 'public' AND table_name  = $1)`
+				FROM information_schema.tables 
+				WHERE table_schema = 'public' AND table_name  = $1)`
 
 	rows, err := pg.db.Query(q, name)
 	if err != nil {
@@ -243,7 +243,7 @@ WHERE table_schema = 'public' AND table_name  = $1)`
 
 func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
 	q := `SELECT EXISTS 
-(SELECT FROM records WHERE digest = $1)`
+			  (SELECT FROM records WHERE digest = $1)`
 
 	rows, err := pg.db.Query(q, hash)
 	if err != nil {
@@ -264,7 +264,7 @@ func (pg *Postgres) createAnchorsTable() error {
 	_, err := pg.db.Exec(`CREATE TABLE public.anchors
 (
     merkle bytea NOT NULL UNIQUE,
-    tx_hash text COLLATE pg_catalog."default" UNIQUE,
+    tx_hash bytea UNIQUE,
     chain_timestamp bigint,
     flush_timestamp bigint,
     CONSTRAINT anchors_pkey PRIMARY KEY (merkle)
@@ -287,7 +287,7 @@ CREATE UNIQUE INDEX idx_merkle
 -- Index: idx_tx_hash
 CREATE UNIQUE INDEX idx_tx_hash
     ON public.anchors USING btree
-    (tx_hash COLLATE pg_catalog."default" ASC NULLS LAST)
+    (tx_hash ASC NULLS LAST)
     TABLESPACE pg_default;
 `)
 	if err != nil {

From c0ee982e8f14527c51c2052f261d2a1c52c790da Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 19 Aug 2020 03:37:28 +0300
Subject: [PATCH 42/66] adjust code to tx hash col type to bytea

---
 dcrtimed/backend/postgres/postgres.go |  4 +--
 dcrtimed/backend/postgres/sql.go      | 39 +++++++++++++--------------
 2 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 9e59fdf..290b624 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -40,8 +40,8 @@ var (
 	// matching we mean both are hourly or every so many minutes.
 	//
 	// Seconds Minutes Hours Days Months DayOfWeek
-	flushSchedule = "10 0 * * * *" // On the hour + 10 seconds
-	duration      = time.Hour      // Default how often we combine digests
+	flushSchedule = "10 * * * * *" // On the hour + 10 seconds
+	duration      = time.Minute    // Default how often we combine digests
 
 	errEmptySet = errors.New("empty set")
 )
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index cbf4cf7..963f926 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -40,7 +40,7 @@ func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
 	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp)
 				VALUES($1, $2, $3)`
 
-	err := pg.db.QueryRow(q, fr.Root[:], fr.Tx.String(),
+	err := pg.db.QueryRow(q, fr.Root[:], fr.Tx[:],
 		fr.FlushTimestamp).Scan()
 	if err != nil {
 		// The insert command won't return any value, the following error is
@@ -134,7 +134,7 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult,
 	var (
 		mr      []byte
 		digest  []byte
-		txHash  sql.NullString
+		txHash  []byte
 		chainTs sql.NullInt64
 	)
 	r := []*backend.GetResult{}
@@ -148,14 +148,12 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult,
 		}
 		rr.Timestamp = ts
 		copy(rr.MerkleRoot[:], mr[:sha256.Size])
-		// txHash & chainTs can be NULL - handle safely
-		if txHash.Valid {
-			tx, err := chainhash.NewHashFromStr(txHash.String)
-			if err != nil {
-				return false, nil, err
-			}
-			rr.Tx = *tx
+		tx, err := chainhash.NewHash(txHash[:])
+		if err != nil {
+			return false, nil, err
 		}
+		rr.Tx = *tx
+		// chainTs can be NULL - handle safely
 		if chainTs.Valid {
 			rr.AnchoredTimestamp = chainTs.Int64
 		}
@@ -180,11 +178,12 @@ func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool,
 		return false, err
 	}
 	defer rows.Close()
-
-	var mr []byte
-	var txHash sql.NullString
-	var chainTs sql.NullInt64
-	var serverTs int64
+	var (
+		mr       []byte
+		txHash   []byte
+		chainTs  sql.NullInt64
+		serverTs int64
+	)
 	for rows.Next() {
 		err = rows.Scan(&mr, &serverTs, &txHash, &chainTs)
 		if err != nil {
@@ -192,14 +191,12 @@ func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool,
 		}
 		r.Timestamp = serverTs
 		copy(r.MerkleRoot[:], mr[:sha256.Size])
-		// txHash & chainTs can be NULL - handle safely
-		if txHash.Valid {
-			tx, err := chainhash.NewHashFromStr(txHash.String)
-			if err != nil {
-				return false, err
-			}
-			r.Tx = *tx
+		tx, err := chainhash.NewHash(txHash[:])
+		if err != nil {
+			return false, err
 		}
+		r.Tx = *tx
+		// chainTs can be NULL - handle safely
 		if chainTs.Valid {
 			r.AnchoredTimestamp = chainTs.Int64
 		}

From 38f8adbe1c236d4a53fd1fedc501d4c1850534bc Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 19 Aug 2020 14:56:23 +0300
Subject: [PATCH 43/66] add LastAnchor interface func

---
 dcrtimed/backend/postgres/postgres.go | 31 ++++++++++++++++++++++-
 dcrtimed/backend/postgres/sql.go      | 36 +++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 290b624..97d4e33 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -378,7 +378,36 @@ func (pg *Postgres) GetBalance() (*backend.GetBalanceResult, error) {
 
 // LastAnchor retrieves last successful anchor details
 func (pg *Postgres) LastAnchor() (*backend.LastAnchorResult, error) {
-	return nil, nil
+	ts, merkle, tx, err := pg.getLatestAnchoredTimestamp()
+	if err != nil {
+		return nil, err
+	}
+
+	if ts == 0 {
+		return &backend.LastAnchorResult{}, nil
+	}
+
+	var me backend.LastAnchorResult
+	me.Tx = *tx
+
+	// Lookup anchored tx info,
+	// and update db if info changed.
+	fr := backend.FlushRecord{
+		Tx:   *tx,
+		Root: *merkle,
+	}
+	txWalletInfo, err := pg.lazyFlush(&fr)
+
+	// If no error, or no enough confirmations
+	// err continue, else return err.
+	if err != nil && err != errNotEnoughConfirmation {
+		return &backend.LastAnchorResult{}, err
+	}
+
+	me.ChainTimestamp = fr.ChainTimestamp
+	me.BlockHash = txWalletInfo.BlockHash.String()
+	me.BlockHeight = txWalletInfo.BlockHeight
+	return &me, nil
 }
 
 func buildQueryString(rootCert, cert, key string) string {
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 963f926..b39ee31 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -9,6 +9,42 @@ import (
 	"github.com/decred/dcrtime/merkle"
 )
 
+func (pg *Postgres) getLatestAnchoredTimestamp() (int64, *[sha256.Size]byte, *chainhash.Hash, error) {
+	q := `SELECT r.collection_timestamp, r.anchor_merkle, an.tx_hash
+				FROM records as r
+				LEFT JOIN anchors as an
+				on r.anchor_merkle = an.merkle
+				WHERE r.anchor_merkle IS NOT NULL
+				ORDER BY r.collection_timestamp DESC
+				LIMIT 1`
+
+	rows, err := pg.db.Query(q)
+	if err != nil {
+		return 0, nil, nil, err
+	}
+	defer rows.Close()
+
+	var (
+		serverTs int64
+		txHash   []byte
+		mr       []byte
+		merkle   [sha256.Size]byte
+		tx       *chainhash.Hash
+	)
+	for rows.Next() {
+		err = rows.Scan(&serverTs, &mr, &txHash)
+		if err != nil {
+			return 0, nil, nil, err
+		}
+		copy(merkle[:], mr[:sha256.Size])
+		tx, err = chainhash.NewHash(txHash)
+		if err != nil {
+			return 0, nil, nil, err
+		}
+	}
+	return serverTs, &merkle, tx, nil
+}
+
 func (pg *Postgres) updateAnchorChainTs(fr *backend.FlushRecord) error {
 	q := `UPDATE anchors SET chain_timestamp = $1
 				WHERE merkle = $2`

From eee5982c70687562bf1415ac17a4e911cd119d7c Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 26 Aug 2020 01:17:16 +0300
Subject: [PATCH 44/66] add Fsck :beach:

---
 cmd/dcrtime_fsck/config.go                 |  67 +++++
 cmd/dcrtime_fsck/dcrtime_fsck.go           |  31 ++-
 cmd/dcrtime_fsck/fsck.log                  |  36 +++
 cmd/dcrtime_timestamp/dcrtime_timestamp.go |   2 +-
 dcrtimed/backend/postgres/dump.go          |  35 +++
 dcrtimed/backend/postgres/fsck.go          | 286 +++++++++++++++++++++
 dcrtimed/backend/postgres/postgres.go      |   6 -
 dcrtimed/backend/postgres/sql.go           |  30 ++-
 fsck.log                                   | 123 +++++++++
 9 files changed, 601 insertions(+), 15 deletions(-)
 create mode 100644 cmd/dcrtime_fsck/config.go
 create mode 100644 cmd/dcrtime_fsck/fsck.log
 create mode 100644 dcrtimed/backend/postgres/dump.go
 create mode 100644 dcrtimed/backend/postgres/fsck.go
 create mode 100644 fsck.log

diff --git a/cmd/dcrtime_fsck/config.go b/cmd/dcrtime_fsck/config.go
new file mode 100644
index 0000000..d4a0065
--- /dev/null
+++ b/cmd/dcrtime_fsck/config.go
@@ -0,0 +1,67 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"path/filepath"
+
+	"github.com/jessevdk/go-flags"
+)
+
+const defaultConfigFilename = "dcrtimed.conf"
+
+var (
+	defaultConfigFile = filepath.Join(defaultHomeDir, defaultConfigFilename)
+	defaultBackend    = "filesystem"
+)
+
+// config defines the configuration options for dcrtime_fsck
+//
+// See loadConfig for details on the configuration load process.
+type config struct {
+	HomeDir           string   `short:"A" long:"appdata" description:"Path to application home directory"`
+	ShowVersion       bool     `short:"V" long:"version" description:"Display version information and exit"`
+	ConfigFile        string   `short:"C" long:"configfile" description:"Path to configuration file"`
+	DataDir           string   `short:"b" long:"datadir" description:"Directory to store data"`
+	LogDir            string   `long:"logdir" description:"Directory to log output."`
+	TestNet           bool     `long:"testnet" description:"Use the test network"`
+	SimNet            bool     `long:"simnet" description:"Use the simulation test network"`
+	Profile           string   `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`
+	CPUProfile        string   `long:"cpuprofile" description:"Write CPU profile to the specified file"`
+	MemProfile        string   `long:"memprofile" description:"Write mem profile to the specified file"`
+	DebugLevel        string   `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`
+	Listeners         []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 49152, testnet: 59152)"`
+	WalletHost        string   `long:"wallethost" description:"Hostname for wallet server"`
+	WalletCert        string   `long:"walletcert" description:"Certificate path for wallet server"`
+	WalletPassphrase  string   `long:"walletpassphrase" description:"Passphrase for wallet server"`
+	Version           string
+	HTTPSCert         string   `long:"httpscert" description:"File containing the https certificate file"`
+	HTTPSKey          string   `long:"httpskey" description:"File containing the https certificate key"`
+	StoreHost         string   `long:"storehost" description:"Enable proxy mode - send requests to the specified ip:port"`
+	StoreCert         string   `long:"storecert" description:"File containing the https certificate file for storehost"`
+	EnableCollections bool     `long:"enablecollections" description:"Allow clients to query collection timestamps."`
+	APITokens         []string `long:"apitoken" description:"Token used to grant access to privileged API resources"`
+	APIVersions       string   `long:"apiversions" description:"Enables API versions on the daemon"`
+	Backend           string   `long:"backend" description:"Sets the cache layer type 'filesystem'/'postgres'"`
+	PostgresHost      string   `long:"postgreshost" description:"Postgres ip:port"`
+	PostgresRootCert  string   `long:"postgresrootcert" description:"File containing the CA certificate for postgres"`
+	PostgresCert      string   `long:"postgrescert" description:"File containing the dcrtimed client certificate for postgres"`
+	PostgresKey       string   `long:"postgreskey" description:"File containing the dcrtimed client certificate key for postgres"`
+}
+
+// loadConfig initializes and parses the config using a config file
+func loadConfig() (*config, error) {
+	// Default config.
+	cfg := config{
+		Backend: defaultBackend,
+	}
+
+	err := flags.IniParse(defaultConfigFile, &cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}
diff --git a/cmd/dcrtime_fsck/dcrtime_fsck.go b/cmd/dcrtime_fsck/dcrtime_fsck.go
index 194b9fd..44a1ccf 100644
--- a/cmd/dcrtime_fsck/dcrtime_fsck.go
+++ b/cmd/dcrtime_fsck/dcrtime_fsck.go
@@ -11,6 +11,7 @@ import (
 	"github.com/decred/dcrd/dcrutil/v2"
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/backend/filesystem"
+	"github.com/decred/dcrtime/dcrtimed/backend/postgres"
 )
 
 var (
@@ -28,6 +29,11 @@ var (
 func _main() error {
 	flag.Parse()
 
+	loadedCfg, err := loadConfig()
+	if err != nil {
+		return fmt.Errorf("Could not load configuration file: %v", err)
+	}
+
 	root := *fsRoot
 	if root == "" {
 		root = filepath.Join(defaultHomeDir, "data")
@@ -52,13 +58,32 @@ func _main() error {
 
 	fmt.Printf("=== Root: %v\n", root)
 
-	fs, err := filesystem.NewDump(root)
+	var b backend.Backend
+	switch (*loadedCfg).Backend {
+	case "filesystem":
+		b, err = filesystem.NewDump(root)
+	case "postgres":
+		var net string
+		switch loadedCfg.TestNet {
+		case true:
+			net = "testnet"
+		default:
+			net = "mainnet"
+		}
+		b, err = postgres.NewDump(loadedCfg.PostgresHost,
+			net,
+			loadedCfg.PostgresRootCert,
+			loadedCfg.PostgresCert,
+			loadedCfg.PostgresKey)
+	default:
+		err = fmt.Errorf("Unsupported backend type: %v", (*loadedCfg).Backend)
+	}
 	if err != nil {
 		return err
 	}
-	defer fs.Close()
+	defer b.Close()
 
-	return fs.Fsck(&backend.FsckOptions{
+	return b.Fsck(&backend.FsckOptions{
 		Verbose:     *verbose,
 		PrintHashes: *printHashes,
 		Fix:         *fix,
diff --git a/cmd/dcrtime_fsck/fsck.log b/cmd/dcrtime_fsck/fsck.log
new file mode 100644
index 0000000..b159663
--- /dev/null
+++ b/cmd/dcrtime_fsck/fsck.log
@@ -0,0 +1,36 @@
+{"version":1,"timestamp":1598391272,"action":"header"}
+{"version":1,"start":1598391272,"dryrun":true}
+
+{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
+
+{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
+
+{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
+
+{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
+
+{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
+{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
+
+{"version":1,"timestamp":1598392394,"action":"header"}
+{"version":1,"start":1598392394,"dryrun":true}
+
+{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
+
+{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
+{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
+
+{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
+
+{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
+
+{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
+
diff --git a/cmd/dcrtime_timestamp/dcrtime_timestamp.go b/cmd/dcrtime_timestamp/dcrtime_timestamp.go
index c028cb9..f2c6a53 100644
--- a/cmd/dcrtime_timestamp/dcrtime_timestamp.go
+++ b/cmd/dcrtime_timestamp/dcrtime_timestamp.go
@@ -28,7 +28,7 @@ func _main() error {
 			continue
 		}
 
-		// Try timestam second
+		// Try timestamp second
 		timestamp, err := time.Parse(fStr, a)
 		if err == nil {
 			fmt.Printf("%v\n", timestamp.Unix())
diff --git a/dcrtimed/backend/postgres/dump.go b/dcrtimed/backend/postgres/dump.go
new file mode 100644
index 0000000..b73a567
--- /dev/null
+++ b/dcrtimed/backend/postgres/dump.go
@@ -0,0 +1,35 @@
+// Copyright (c) 2017-2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+	"net/url"
+)
+
+func NewDump(host, net, rootCert, cert, key string) (*Postgres, error) {
+	// Connect to database
+	dbName := net + "_dcrtime"
+	h := "postgresql://" + dbUser + "@" + host + "/" + dbName
+	u, err := url.Parse(h)
+	if err != nil {
+		return nil, fmt.Errorf("parse url '%v': %v", h, err)
+	}
+
+	qs := buildQueryString(rootCert, cert, key)
+	addr := u.String() + "?" + qs
+
+	db, err := sql.Open("postgres", addr)
+	if err != nil {
+		return nil, fmt.Errorf("connect to database '%v': %v", addr, err)
+	}
+
+	pg := &Postgres{
+		db: db,
+	}
+
+	return pg, nil
+}
diff --git a/dcrtimed/backend/postgres/fsck.go b/dcrtimed/backend/postgres/fsck.go
new file mode 100644
index 0000000..479387c
--- /dev/null
+++ b/dcrtimed/backend/postgres/fsck.go
@@ -0,0 +1,286 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package postgres
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/decred/dcrd/txscript/v2"
+	"github.com/decred/dcrdata/api/types/v4"
+	"github.com/decred/dcrtime/dcrtimed/backend"
+	"github.com/decred/dcrtime/merkle"
+)
+
+const (
+	PostgresActionVersion = 1 // All structure versions
+
+	PostgresActionHeader          = "header"
+	PostgresActionDeleteTimestamp = "deletetimestamp"
+	PostgresActionDeleteDigest    = "deletedigest"
+	PostgresActionDeleteDuplicate = "deleteduplicate"
+)
+
+type PostgresAction struct {
+	Version   uint64 `json:"version"`   // Version of structure
+	Timestamp int64  `json:"timestamp"` // Timestamp of action
+	Action    string `json:"action"`    // Following JSON command
+}
+
+type PostgresHeader struct {
+	Version uint64 `json:"version"` // Version of structure
+	Start   int64  `json:"start"`   // Start of fsck
+	DryRun  bool   `json:"dryrun"`  // Dry run
+}
+
+type PostgresDeleteTimestamp struct {
+	Version   uint64 `json:"version"`   // Version of structure
+	Timestamp int64  `json:"timestamp"` // Timestamp
+	Directory string `json:"directory"` // Directory name of Timestamp
+}
+
+type PostgresDeleteDigest struct {
+	Version         uint64 `json:"version"`         // Version of structure
+	Timestamp       int64  `json:"timestamp"`       // Timestamp of digest
+	GlobalTimestamp int64  `json:"globaltimestamp"` // Global timestamp of digest
+	Digest          string `json:"digest"`          // Digest that was deleted
+}
+
+type PostgresDeleteDuplicate struct {
+	Version            uint64 `json:"version"`            // Version of structure
+	Digest             string `json:"digest"`             // Duplicate digest
+	Found              int64  `json:"found"`              // Original timestamp
+	FoundDirectory     string `json:"founddirectory"`     // Original directory
+	Duplicate          int64  `json:"duplicate"`          // Duplicate timestamp
+	DuplicateDirectory string `json:"duplicatedirectory"` // Duplicate directory
+}
+
+// validJournalAction returns true if the action is a valid PostgresAction.
+func validJournalAction(action string) bool {
+	switch action {
+	case PostgresActionHeader:
+	case PostgresActionDeleteTimestamp:
+	case PostgresActionDeleteDigest:
+	case PostgresActionDeleteDuplicate:
+	default:
+		return false
+	}
+	return true
+}
+
+// journal records what fix occurred at what time if filename != "".
+func journal(filename, action string, payload interface{}) error {
+	// See if we are journaling
+	if filename == "" {
+		return nil
+	}
+
+	// Sanity
+	if !validJournalAction(action) {
+		return fmt.Errorf("invalid journal action: %v", action)
+	}
+
+	f, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0640)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// Write PostgresAction
+	e := json.NewEncoder(f)
+	rt := PostgresAction{
+		Version:   PostgresActionVersion,
+		Timestamp: time.Now().Unix(),
+		Action:    action,
+	}
+	err = e.Encode(rt)
+	if err != nil {
+		return err
+	}
+
+	// Write payload
+	err = e.Encode(payload)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(f, "\n")
+
+	return err
+}
+
+// fsckTimestamp verifies that a timestamp is coherent by doing the following:
+// 1.  Find timestamp' digests in records table & anchor db info if anchored
+// 2.  Ensure no duplicates in timestamp's digets
+// 3.  If timestamp was anchored:
+// 3.1 Generate merkle root using records table digests and verify against
+//     anchor's merkle root from db
+// 3.2 Verify that the anchor merkle root on db exists on the blockchain.
+func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error {
+	exists, records, err := pg.getRecordsByServerTs(ts)
+	if err != nil {
+		return err
+	}
+	// Non fatal error if there is nothing to do
+	if !exists {
+		return nil
+	}
+
+	var (
+		anchored bool
+		fr       backend.FlushRecord
+	)
+	digests := make(map[string]int64)
+	for _, r := range records {
+		k := hex.EncodeToString(r.Digest[:])
+		if _, ok := digests[k]; ok {
+			// This really can't happen but we check it so that we
+			// can equate lengths later to determine if the map and
+			// array are the same.
+			return fmt.Errorf("    *** ERROR duplicate key: %v", k)
+		}
+		digests[k] = ts
+		if r.MerkleRoot != [sha256.Size]byte{} {
+			anchored = true
+			fr.Root = r.MerkleRoot
+			fr.Tx = r.Tx
+		}
+		fr.Hashes = append(fr.Hashes, &r.Digest)
+	}
+
+	// If anchored generate merkle and compare against merkle in anchors
+	// table
+	if anchored {
+		// Generate merkle
+		mt := merkle.Tree(fr.Hashes)
+		// Last element is root
+		root := *mt[len(mt)-1]
+		if !bytes.Equal(root[:], fr.Root[:]) {
+			return fmt.Errorf("   *** ERROR mismatched merkle "+
+				"root: %x %x", root, fr.Root)
+		}
+	}
+
+	// 3.3 Verify merkle root in tx
+	u := options.URL + fr.Tx.String() + "/out"
+	r, err := http.Get(u)
+	if err != nil {
+		return fmt.Errorf("   *** ERROR HTTP Get: %v", err)
+	}
+	defer r.Body.Close()
+
+	if r.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			return fmt.Errorf("   *** ERROR invalid "+
+				"body: %v %v", r.StatusCode, body)
+		}
+		return fmt.Errorf("   *** ERROR invalid dcrdata "+
+			"answer: %v %s", r.StatusCode, body)
+	}
+
+	var txOuts []types.TxOut
+	d := json.NewDecoder(r.Body)
+	if err := d.Decode(&txOuts); err != nil {
+		return err
+	}
+
+	var done bool
+	for _, v := range txOuts {
+		if !types.IsNullDataScript(v.ScriptPubKeyDecoded.Type) {
+			continue
+		}
+		script, err := hex.DecodeString(v.ScriptPubKeyDecoded.Hex)
+		if err != nil {
+			return fmt.Errorf("   *** ERROR invalid "+
+				"dcrdata script: %v", err)
+		}
+		data, err := txscript.PushedData(script)
+		if err != nil {
+			return fmt.Errorf("   *** ERROR invalid "+
+				"script: %v", err)
+		}
+		if !bytes.Equal(data[0], fr.Root[:]) {
+			continue
+		}
+
+		// Everything is cool so mark it and break out
+		done = true
+		break
+	}
+
+	if !done {
+		return fmt.Errorf("   *** ERROR merkle root not "+
+			"found: tx %v merkle %x", fr.Tx, fr.Root)
+	}
+
+	return nil
+}
+
+func (pg *Postgres) fsckTimestamps(options *backend.FsckOptions) error {
+	tss, err := pg.getAllRecordsTimestamps()
+	if err != nil {
+		return err
+	}
+
+	for _, ts := range *tss {
+		if options.Verbose || options.PrintHashes {
+			fmt.Printf("--- Checking: %v \n", ts)
+		}
+		err = pg.fsckTimestamp(options, ts)
+		if err != nil {
+			return err
+		}
+		if options.Verbose || options.PrintHashes {
+			fmt.Printf("=== Verified: %v \n",
+				ts)
+		}
+	}
+	return nil
+}
+
+// Fsck walks all db records and verifies all that there is no apparent data
+// corruption and that the anchors indeed exist on the blockchain.
+func (pg *Postgres) Fsck(options *backend.FsckOptions) error {
+	ts := time.Now().Unix()
+	fmt.Printf("=== FSCK started %v\n", ts)
+	fmt.Printf("--- Phase 1: checking records table\n")
+
+	if options.File != "" {
+		// Create journal file
+		f, err := os.OpenFile(options.File, os.O_RDWR|os.O_CREATE, 0640)
+		if err != nil {
+			return err
+		}
+		f.Close()
+	}
+
+	err := journal(options.File, PostgresActionHeader,
+		PostgresHeader{
+			Version: PostgresActionVersion,
+			Start:   ts,
+			DryRun:  !options.Fix,
+		})
+	if err != nil {
+		return fmt.Errorf("   *** ERROR journal: %v",
+			err)
+	}
+
+	if options == nil {
+		options = &backend.FsckOptions{}
+	}
+
+	err = pg.fsckTimestamps(options)
+	if err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 97d4e33..cdef185 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -356,12 +356,6 @@ func (pg *Postgres) Restore(*os.File, bool, string) error {
 	return nil
 }
 
-// Fsck walks all data and verifies its integrity. In addition it
-// verifies anchored timestamps' existence on the blockchain.
-func (pg *Postgres) Fsck(*backend.FsckOptions) error {
-	return nil
-}
-
 // GetBalance retrieves balance information for the wallet
 // backing this instance
 func (pg *Postgres) GetBalance() (*backend.GetBalanceResult, error) {
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index b39ee31..80ce798 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -9,6 +9,27 @@ import (
 	"github.com/decred/dcrtime/merkle"
 )
 
+func (pg *Postgres) getAllRecordsTimestamps() (*[]int64, error) {
+	q := `SELECT collection_timestamp FROM records`
+
+	rows, err := pg.db.Query(q)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var tss []int64
+	var ts int64
+	for rows.Next() {
+		err = rows.Scan(&ts)
+		if err != nil {
+			return nil, err
+		}
+		tss = append(tss, ts)
+	}
+	return &tss, nil
+}
+
 func (pg *Postgres) getLatestAnchoredTimestamp() (int64, *[sha256.Size]byte, *chainhash.Hash, error) {
 	q := `SELECT r.collection_timestamp, r.anchor_merkle, an.tx_hash
 				FROM records as r
@@ -25,11 +46,10 @@ func (pg *Postgres) getLatestAnchoredTimestamp() (int64, *[sha256.Size]byte, *ch
 	defer rows.Close()
 
 	var (
-		serverTs int64
-		txHash   []byte
-		mr       []byte
-		merkle   [sha256.Size]byte
-		tx       *chainhash.Hash
+		serverTs   int64
+		txHash, mr []byte
+		merkle     [sha256.Size]byte
+		tx         *chainhash.Hash
 	)
 	for rows.Next() {
 		err = rows.Scan(&serverTs, &mr, &txHash)
diff --git a/fsck.log b/fsck.log
new file mode 100644
index 0000000..d6d5e8e
--- /dev/null
+++ b/fsck.log
@@ -0,0 +1,123 @@
+{"version":1,"timestamp":1598202410,"action":"header"}
+{"version":1,"start":1598202410,"dryrun":true}
+
+{"version":1,"timestamp":1598219261,"action":"header"}
+{"version":1,"start":1598219261,"dryrun":true}
+
+{"version":1,"timestamp":1598219280,"action":"header"}
+{"version":1,"start":1598219280,"dryrun":true}
+
+{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
+
+{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
+
+{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
+
+{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
+
+{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
+{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
+
+{"version":1,"timestamp":1598264220,"action":"header"}
+{"version":1,"start":1598264220,"dryrun":true}
+
+{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
+
+{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
+{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
+
+{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
+
+{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
+
+{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
+
+{"version":1,"timestamp":1598265824,"action":"header"}
+{"version":1,"start":1598265824,"dryrun":true}
+
+{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
+
+{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
+{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
+
+{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
+
+{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
+
+{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
+
+{"version":1,"timestamp":1598265914,"action":"header"}
+{"version":1,"start":1598265914,"dryrun":true}
+
+{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
+
+{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
+
+{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
+
+{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
+
+{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
+{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
+
+{"version":1,"timestamp":1598265981,"action":"header"}
+{"version":1,"start":1598265981,"dryrun":true}
+
+{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
+
+{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
+
+{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
+{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
+
+{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
+
+{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
+
+{"version":1,"timestamp":1598266009,"action":"header"}
+{"version":1,"start":1598266009,"dryrun":true}
+
+{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
+
+{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
+
+{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
+
+{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
+{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
+
+{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
+{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
+
+{"version":1,"timestamp":1598393477,"action":"header"}
+{"version":1,"start":1598393477,"dryrun":true}
+
+{"version":1,"timestamp":1598393737,"action":"header"}
+{"version":1,"start":1598393737,"dryrun":true}
+
+{"version":1,"timestamp":1598393749,"action":"header"}
+{"version":1,"start":1598393749,"dryrun":true}
+

From c52cd06f83218cbc4db2979891845bec2c7ca6fc Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Wed, 26 Aug 2020 14:58:46 +0300
Subject: [PATCH 45/66] cleanup Fsck

---
 dcrtimed/backend/postgres/fsck.go | 17 ++++++++++++++++-
 fsck.log                          | 18 ++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/dcrtimed/backend/postgres/fsck.go b/dcrtimed/backend/postgres/fsck.go
index 479387c..a1a20fd 100644
--- a/dcrtimed/backend/postgres/fsck.go
+++ b/dcrtimed/backend/postgres/fsck.go
@@ -153,9 +153,13 @@ func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error
 			fr.Root = r.MerkleRoot
 			fr.Tx = r.Tx
 		}
+		if options.PrintHashes {
+			fmt.Printf("Hash           : %v\n", hex.EncodeToString(r.Digest[:]))
+		}
 		fr.Hashes = append(fr.Hashes, &r.Digest)
 	}
-
+	fmt.Printf("No duplicates found\n")
+	fmt.Printf("Anchored       : %v\n", anchored)
 	// If anchored generate merkle and compare against merkle in anchors
 	// table
 	if anchored {
@@ -167,6 +171,11 @@ func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error
 			return fmt.Errorf("   *** ERROR mismatched merkle "+
 				"root: %x %x", root, fr.Root)
 		}
+		if options.PrintHashes {
+			fmt.Printf("Anchor's Calculated merkle: %x\n", root[:])
+			fmt.Printf("Anchor's merkle from db: %x\n", fr.Root[:])
+		}
+		fmt.Printf("Verfied anchor's merkle root\n")
 	}
 
 	// 3.3 Verify merkle root in tx
@@ -222,6 +231,12 @@ func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error
 			"found: tx %v merkle %x", fr.Tx, fr.Root)
 	}
 
+	if options.PrintHashes {
+		fmt.Printf("Anchor's tx: %v found on the blockchain\n", fr.Tx)
+	} else {
+		fmt.Printf("Anchor's tx found on the blockchain\n")
+	}
+
 	return nil
 }
 
diff --git a/fsck.log b/fsck.log
index d6d5e8e..fc6ee5c 100644
--- a/fsck.log
+++ b/fsck.log
@@ -121,3 +121,21 @@
 {"version":1,"timestamp":1598393749,"action":"header"}
 {"version":1,"start":1598393749,"dryrun":true}
 
+{"version":1,"timestamp":1598394696,"action":"header"}
+{"version":1,"start":1598394696,"dryrun":true}
+
+{"version":1,"timestamp":1598435066,"action":"header"}
+{"version":1,"start":1598435066,"dryrun":true}
+
+{"version":1,"timestamp":1598435253,"action":"header"}
+{"version":1,"start":1598435253,"dryrun":true}
+
+{"version":1,"timestamp":1598436054,"action":"header"}
+{"version":1,"start":1598436054,"dryrun":true}
+
+{"version":1,"timestamp":1598442896,"action":"header"}
+{"version":1,"start":1598442896,"dryrun":true}
+
+{"version":1,"timestamp":1598442901,"action":"header"}
+{"version":1,"start":1598442901,"dryrun":true}
+

From 7f271b12226b228edb847cef9efdf7e16cc34746 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Fri, 28 Aug 2020 01:25:12 +0300
Subject: [PATCH 46/66] add postgres backend support for dcrtime_dumpdb

---
 cmd/dcrtime_dumpdb/config.go          |  67 ++++++++++++
 cmd/dcrtime_dumpdb/dcrtime_dumpdb.go  |  32 +++++-
 cmd/dcrtime_fsck/README.md            |   2 +
 dcrtimed/backend/postgres/dump.go     | 143 ++++++++++++++++++++++++++
 dcrtimed/backend/postgres/fsck.go     |   5 +-
 dcrtimed/backend/postgres/postgres.go |   9 +-
 dcrtimed/backend/postgres/sql.go      |  16 +--
 7 files changed, 254 insertions(+), 20 deletions(-)
 create mode 100644 cmd/dcrtime_dumpdb/config.go

diff --git a/cmd/dcrtime_dumpdb/config.go b/cmd/dcrtime_dumpdb/config.go
new file mode 100644
index 0000000..d4a0065
--- /dev/null
+++ b/cmd/dcrtime_dumpdb/config.go
@@ -0,0 +1,67 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"path/filepath"
+
+	"github.com/jessevdk/go-flags"
+)
+
+const defaultConfigFilename = "dcrtimed.conf"
+
+var (
+	defaultConfigFile = filepath.Join(defaultHomeDir, defaultConfigFilename)
+	defaultBackend    = "filesystem"
+)
+
+// config defines the configuration options for dcrtime_fsck
+//
+// See loadConfig for details on the configuration load process.
+type config struct {
+	HomeDir           string   `short:"A" long:"appdata" description:"Path to application home directory"`
+	ShowVersion       bool     `short:"V" long:"version" description:"Display version information and exit"`
+	ConfigFile        string   `short:"C" long:"configfile" description:"Path to configuration file"`
+	DataDir           string   `short:"b" long:"datadir" description:"Directory to store data"`
+	LogDir            string   `long:"logdir" description:"Directory to log output."`
+	TestNet           bool     `long:"testnet" description:"Use the test network"`
+	SimNet            bool     `long:"simnet" description:"Use the simulation test network"`
+	Profile           string   `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65536"`
+	CPUProfile        string   `long:"cpuprofile" description:"Write CPU profile to the specified file"`
+	MemProfile        string   `long:"memprofile" description:"Write mem profile to the specified file"`
+	DebugLevel        string   `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`
+	Listeners         []string `long:"listen" description:"Add an interface/port to listen for connections (default all interfaces port: 49152, testnet: 59152)"`
+	WalletHost        string   `long:"wallethost" description:"Hostname for wallet server"`
+	WalletCert        string   `long:"walletcert" description:"Certificate path for wallet server"`
+	WalletPassphrase  string   `long:"walletpassphrase" description:"Passphrase for wallet server"`
+	Version           string
+	HTTPSCert         string   `long:"httpscert" description:"File containing the https certificate file"`
+	HTTPSKey          string   `long:"httpskey" description:"File containing the https certificate key"`
+	StoreHost         string   `long:"storehost" description:"Enable proxy mode - send requests to the specified ip:port"`
+	StoreCert         string   `long:"storecert" description:"File containing the https certificate file for storehost"`
+	EnableCollections bool     `long:"enablecollections" description:"Allow clients to query collection timestamps."`
+	APITokens         []string `long:"apitoken" description:"Token used to grant access to privileged API resources"`
+	APIVersions       string   `long:"apiversions" description:"Enables API versions on the daemon"`
+	Backend           string   `long:"backend" description:"Sets the cache layer type 'filesystem'/'postgres'"`
+	PostgresHost      string   `long:"postgreshost" description:"Postgres ip:port"`
+	PostgresRootCert  string   `long:"postgresrootcert" description:"File containing the CA certificate for postgres"`
+	PostgresCert      string   `long:"postgrescert" description:"File containing the dcrtimed client certificate for postgres"`
+	PostgresKey       string   `long:"postgreskey" description:"File containing the dcrtimed client certificate key for postgres"`
+}
+
+// loadConfig initializes and parses the config using a config file
+func loadConfig() (*config, error) {
+	// Default config.
+	cfg := config{
+		Backend: defaultBackend,
+	}
+
+	err := flags.IniParse(defaultConfigFile, &cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}
diff --git a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
index 103af2c..a407378 100644
--- a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
+++ b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
@@ -8,7 +8,9 @@ import (
 
 	"github.com/decred/dcrd/chaincfg/v2"
 	"github.com/decred/dcrd/dcrutil/v2"
+	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/backend/filesystem"
+	"github.com/decred/dcrtime/dcrtimed/backend/postgres"
 )
 
 var (
@@ -24,6 +26,11 @@ var (
 func _main() error {
 	flag.Parse()
 
+	loadedCfg, err := loadConfig()
+	if err != nil {
+		return fmt.Errorf("Could not load configuration file: %v", err)
+	}
+
 	if *restore {
 		if *destination == "" {
 			return fmt.Errorf("-destination must be set")
@@ -52,13 +59,32 @@ func _main() error {
 
 	fmt.Printf("=== Root: %v\n", root)
 
-	fs, err := filesystem.NewDump(root)
+	var b backend.Backend
+	switch (*loadedCfg).Backend {
+	case "filesystem":
+		b, err = filesystem.NewDump(root)
+	case "postgres":
+		var net string
+		switch loadedCfg.TestNet {
+		case true:
+			net = "testnet"
+		default:
+			net = "mainnet"
+		}
+		b, err = postgres.NewDump(loadedCfg.PostgresHost,
+			net,
+			loadedCfg.PostgresRootCert,
+			loadedCfg.PostgresCert,
+			loadedCfg.PostgresKey)
+	default:
+		err = fmt.Errorf("Unsupported backend type: %v", (*loadedCfg).Backend)
+	}
 	if err != nil {
 		return err
 	}
-	defer fs.Close()
+	defer b.Close()
 
-	return fs.Dump(os.Stdout, !*dumpJSON)
+	return b.Dump(os.Stdout, !*dumpJSON)
 }
 
 func main() {
diff --git a/cmd/dcrtime_fsck/README.md b/cmd/dcrtime_fsck/README.md
index 10814db..7464a84 100644
--- a/cmd/dcrtime_fsck/README.md
+++ b/cmd/dcrtime_fsck/README.md
@@ -1,6 +1,8 @@
 dcrtime_fsck
 ============
 
+// XXX: update docs and mention postgres
+
 The filesystem backend can under rare circumstances become incoherent. This
 tool iterates over all timestamp directories and corrects known failures.
 
diff --git a/dcrtimed/backend/postgres/dump.go b/dcrtimed/backend/postgres/dump.go
index b73a567..b30f727 100644
--- a/dcrtimed/backend/postgres/dump.go
+++ b/dcrtimed/backend/postgres/dump.go
@@ -5,9 +5,15 @@
 package postgres
 
 import (
+	"crypto/sha256"
 	"database/sql"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"net/url"
+	"os"
+
+	"github.com/decred/dcrtime/dcrtimed/backend"
 )
 
 func NewDump(host, net, rootCert, cert, key string) (*Postgres, error) {
@@ -33,3 +39,140 @@ func NewDump(host, net, rootCert, cert, key string) (*Postgres, error) {
 
 	return pg, nil
 }
+
+// Dump dumps database to the provided file descriptor. If the
+// human flag is set to true it pretty prints the database content
+// otherwise it dumps a JSON stream.
+func (pg *Postgres) Dump(f *os.File, verbose bool) error {
+	err := pg.dumpTimestamps(f, verbose)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (pg *Postgres) dumpTimestamps(f *os.File, verbose bool) error {
+	tss, err := pg.getAllRecordsTimestamps()
+	if err != nil {
+		return err
+	}
+
+	for _, ts := range *tss {
+		if verbose {
+			fmt.Fprintf(f, "--- Timestamp: %v\n", ts)
+		}
+		err = pg.dumpTimestamp(f, verbose, ts)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (pg *Postgres) dumpTimestamp(f *os.File, verbose bool, ts int64) error {
+	exists, records, flushTs, err := pg.getRecordsByServerTs(ts)
+	if err != nil {
+		return err
+	}
+	// Non fatal error if there is nothing to do
+	if !exists {
+		return nil
+	}
+
+	var (
+		anchored bool
+		fr       backend.FlushRecord
+		digests  = make([]backend.DigestReceived, 0, 10000)
+	)
+	for _, r := range records {
+		if r.MerkleRoot != [sha256.Size]byte{} && !anchored {
+			anchored = true
+			fr.Root = r.MerkleRoot
+			fr.Tx = r.Tx
+			fr.ChainTimestamp = r.AnchoredTimestamp
+			fr.FlushTimestamp = flushTs
+		}
+		fr.Hashes = append(fr.Hashes, &r.Digest)
+		digests = append(digests, backend.DigestReceived{
+			Digest:    hex.EncodeToString(r.Digest[:]),
+			Timestamp: r.Timestamp,
+		})
+	}
+
+	if anchored {
+		if verbose {
+			dumpFlushRecord(f, &fr)
+		} else {
+			e := json.NewEncoder(f)
+			rt := backend.RecordType{
+				Version: backend.RecordTypeVersion,
+				Type:    backend.RecordTypeFlushRecord,
+			}
+			err := e.Encode(rt)
+			if err != nil {
+				return err
+			}
+			frj := backend.FlushRecordJSON{
+				Root:           fr.Root,
+				Hashes:         fr.Hashes,
+				Tx:             fr.Tx,
+				ChainTimestamp: fr.ChainTimestamp,
+				FlushTimestamp: fr.FlushTimestamp,
+				Timestamp:      ts,
+			}
+			err = e.Encode(frj)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	for _, v := range digests {
+		err := dumpDigestTimestamp(f, verbose,
+			backend.RecordTypeDigestReceived, v)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func dumpDigestTimestamp(f *os.File, verbose bool, recordType string, dr backend.DigestReceived) error {
+	if verbose {
+		fmt.Fprintf(f, "Digest     : %v\n", dr.Digest)
+		fmt.Fprintf(f, "Timestamp  : %v\n", dr.Timestamp)
+	} else {
+		e := json.NewEncoder(f)
+		rt := backend.RecordType{
+			Version: backend.RecordTypeVersion,
+			Type:    recordType,
+		}
+		err := e.Encode(rt)
+		if err != nil {
+			return err
+		}
+		r := backend.DigestReceived{
+			Digest:    dr.Digest,
+			Timestamp: dr.Timestamp,
+		}
+		err = e.Encode(r)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func dumpFlushRecord(f *os.File, flushRecord *backend.FlushRecord) {
+	fmt.Fprintf(f, "Merkle root    : %x\n",
+		flushRecord.Root)
+	fmt.Fprintf(f, "Tx             : %v\n", flushRecord.Tx)
+	fmt.Fprintf(f, "Chain timestamp: %v\n",
+		flushRecord.ChainTimestamp)
+	fmt.Fprintf(f, "Flush timestamp: %v\n",
+		flushRecord.FlushTimestamp)
+	for _, v := range flushRecord.Hashes {
+		fmt.Fprintf(f, "  Flushed      : %x\n", *v)
+	}
+}
diff --git a/dcrtimed/backend/postgres/fsck.go b/dcrtimed/backend/postgres/fsck.go
index a1a20fd..dfebe98 100644
--- a/dcrtimed/backend/postgres/fsck.go
+++ b/dcrtimed/backend/postgres/fsck.go
@@ -125,7 +125,7 @@ func journal(filename, action string, payload interface{}) error {
 //     anchor's merkle root from db
 // 3.2 Verify that the anchor merkle root on db exists on the blockchain.
 func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error {
-	exists, records, err := pg.getRecordsByServerTs(ts)
+	exists, records, flushTs, err := pg.getRecordsByServerTs(ts)
 	if err != nil {
 		return err
 	}
@@ -148,10 +148,11 @@ func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error
 			return fmt.Errorf("    *** ERROR duplicate key: %v", k)
 		}
 		digests[k] = ts
-		if r.MerkleRoot != [sha256.Size]byte{} {
+		if r.MerkleRoot != [sha256.Size]byte{} && !anchored {
 			anchored = true
 			fr.Root = r.MerkleRoot
 			fr.Tx = r.Tx
+			fr.FlushTimestamp = flushTs
 		}
 		if options.PrintHashes {
 			fmt.Printf("Hash           : %v\n", hex.EncodeToString(r.Digest[:]))
diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index cdef185..171bfab 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -194,7 +194,7 @@ func (pg *Postgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult
 			Timestamp: ts,
 		}
 		if pg.enableCollections {
-			exists, tsDigests, err := pg.getRecordsByServerTs(ts)
+			exists, tsDigests, _, err := pg.getRecordsByServerTs(ts)
 			if err != nil {
 				return nil, err
 			}
@@ -341,13 +341,6 @@ func (pg *Postgres) Close() {
 	pg.db.Close()
 }
 
-// Dump dumps database to the provided file descriptor. If the
-// human flag is set to true it pretty prints the database content
-// otherwise it dumps a JSON stream.
-func (pg *Postgres) Dump(*os.File, bool) error {
-	return nil
-}
-
 // Restore recreates the the database from the provided file
 // descriptor. The verbose flag is set to true to indicate that this
 // call may parint to stdout. The provided string describes the target
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 80ce798..19ccc27 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -175,8 +175,9 @@ func (pg *Postgres) getUnflushedTimestamps(current int64) ([]int64, error) {
 	return tss, nil
 }
 
-func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult, error) {
-	q := `SELECT r.anchor_merkle, an.tx_hash, an.chain_timestamp, r.digest
+func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult, int64, error) {
+	q := `SELECT r.anchor_merkle, an.tx_hash, an.chain_timestamp, r.digest,
+        an.flush_timestamp
 				FROM records as r
 				LEFT JOIN anchors as an
 				on r.anchor_merkle = an.merkle
@@ -184,7 +185,7 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult,
 
 	rows, err := pg.db.Query(q, ts)
 	if err != nil {
-		return false, nil, err
+		return false, nil, 0, err
 	}
 	defer rows.Close()
 	var (
@@ -192,21 +193,22 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult,
 		digest  []byte
 		txHash  []byte
 		chainTs sql.NullInt64
+		flushTs int64
 	)
 	r := []*backend.GetResult{}
 	for rows.Next() {
 		rr := backend.GetResult{
 			Timestamp: ts,
 		}
-		err = rows.Scan(&mr, &txHash, &chainTs, &digest)
+		err = rows.Scan(&mr, &txHash, &chainTs, &digest, &flushTs)
 		if err != nil {
-			return false, nil, err
+			return false, nil, 0, err
 		}
 		rr.Timestamp = ts
 		copy(rr.MerkleRoot[:], mr[:sha256.Size])
 		tx, err := chainhash.NewHash(txHash[:])
 		if err != nil {
-			return false, nil, err
+			return false, nil, 0, err
 		}
 		rr.Tx = *tx
 		// chainTs can be NULL - handle safely
@@ -218,7 +220,7 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult,
 		r = append(r, &rr)
 	}
 
-	return len(r) > 0, r, nil
+	return len(r) > 0, r, flushTs, nil
 }
 
 func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool, error) {

From 9f88cf62b855ac5f750a70824af7f21cf7eb4e50 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 29 Aug 2020 13:07:27 +0300
Subject: [PATCH 47/66] fix duplicates in postgres Dump and preprare for
 Restore

---
 cmd/dcrtime_dumpdb/dcrtime_dumpdb.go | 31 +++++++++++++---------------
 dcrtimed/backend/postgres/sql.go     |  2 +-
 dump.dump                            | 27 ++++++++++++++++++++++++
 dump.dump.postgres                   | 11 ++++++++++
 4 files changed, 53 insertions(+), 18 deletions(-)
 create mode 100644 dump.dump
 create mode 100644 dump.dump.postgres

diff --git a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
index a407378..c3fa5db 100644
--- a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
+++ b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
@@ -26,23 +26,14 @@ var (
 func _main() error {
 	flag.Parse()
 
-	loadedCfg, err := loadConfig()
-	if err != nil {
-		return fmt.Errorf("Could not load configuration file: %v", err)
-	}
-
 	if *restore {
 		if *destination == "" {
 			return fmt.Errorf("-destination must be set")
 		}
-
-		fs, err := filesystem.NewRestore(*destination)
-		if err != nil {
-			return err
-		}
-		defer fs.Close()
-
-		return fs.Restore(os.Stdin, true, *destination)
+	}
+	loadedCfg, err := loadConfig()
+	if err != nil {
+		return fmt.Errorf("Could not load configuration file: %v", err)
 	}
 
 	root := *fsRoot
@@ -55,13 +46,12 @@ func _main() error {
 		}
 	}
 
-	// Dump
-
-	fmt.Printf("=== Root: %v\n", root)
-
 	var b backend.Backend
 	switch (*loadedCfg).Backend {
 	case "filesystem":
+		if *restore {
+			b, err = filesystem.NewRestore(*destination)
+		}
 		b, err = filesystem.NewDump(root)
 	case "postgres":
 		var net string
@@ -71,6 +61,9 @@ func _main() error {
 		default:
 			net = "mainnet"
 		}
+		if *restore {
+			b, err = filesystem.NewRestore(*destination)
+		}
 		b, err = postgres.NewDump(loadedCfg.PostgresHost,
 			net,
 			loadedCfg.PostgresRootCert,
@@ -84,6 +77,10 @@ func _main() error {
 	}
 	defer b.Close()
 
+	fmt.Printf("=== Root: %v\n", root)
+	if *restore {
+		return b.Restore(os.Stdin, true, *destination)
+	}
 	return b.Dump(os.Stdout, !*dumpJSON)
 }
 
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 19ccc27..c1d4a42 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -10,7 +10,7 @@ import (
 )
 
 func (pg *Postgres) getAllRecordsTimestamps() (*[]int64, error) {
-	q := `SELECT collection_timestamp FROM records`
+	q := `SELECT DISTINCT collection_timestamp FROM records`
 
 	rows, err := pg.db.Query(q)
 	if err != nil {
diff --git a/dump.dump b/dump.dump
new file mode 100644
index 0000000..f1d72bd
--- /dev/null
+++ b/dump.dump
@@ -0,0 +1,27 @@
+=== Root: /Users/amass/Library/Application Support/Dcrtimed/data/testnet3
+{"version":1,"type":"flush"}
+{"root":[240,144,86,149,193,246,241,152,209,125,58,82,140,27,205,108,72,181,218,108,46,34,68,11,154,246,89,10,216,193,97,90],"hashes":[[240,144,86,149,193,246,241,152,209,125,58,82,140,27,205,108,72,181,218,108,46,34,68,11,154,246,89,10,216,193,97,90]],"tx":[9,64,243,196,131,184,27,246,124,169,117,14,211,101,203,21,90,170,67,78,119,23,199,18,34,182,145,170,122,231,211,247],"chaintimestamp":1593018261,"flushtimestamp":1593018130,"timestamp":1593018060}
+{"version":1,"type":"digest"}
+{"digest":"f0905695c1f6f198d17d3a528c1bcd6c48b5da6c2e22440b9af6590ad8c1615a","timestamp":1593018060}
+{"version":1,"type":"flush"}
+{"root":[105,56,232,252,75,172,244,158,50,51,98,7,145,246,48,171,31,147,38,30,249,105,179,84,72,242,138,101,145,57,201,0],"hashes":[[105,56,232,252,75,172,244,158,50,51,98,7,145,246,48,171,31,147,38,30,249,105,179,84,72,242,138,101,145,57,201,0]],"tx":[150,212,112,179,197,252,59,42,14,250,95,93,167,209,181,52,16,167,187,30,101,16,243,81,115,97,197,249,73,106,119,52],"chaintimestamp":1593022396,"flushtimestamp":1593022390,"timestamp":1593022320}
+{"version":1,"type":"digest"}
+{"digest":"6938e8fc4bacf49e3233620791f630ab1f93261ef969b35448f28a659139c900","timestamp":1593022320}
+{"version":1,"type":"flush"}
+{"root":[164,222,239,177,139,88,233,240,13,185,175,128,14,69,116,113,226,102,141,200,222,154,42,89,163,136,194,123,180,15,95,224],"hashes":[[29,156,132,223,1,82,38,55,155,75,57,156,112,7,87,139,203,216,77,107,167,116,40,122,215,1,137,35,55,67,192,72],[149,241,7,50,205,132,25,185,242,244,3,128,133,255,125,129,143,86,203,92,223,133,194,96,130,121,218,2,135,77,87,209],[246,24,247,110,130,178,10,255,192,19,36,95,16,130,153,45,69,207,24,36,54,221,200,161,65,229,125,215,43,91,158,163]],"tx":[190,108,7,5,226,66,25,119,249,12,45,219,184,143,216,11,164,84,117,17,0,42,29,215,165,190,102,47,30,68,248,59],"chaintimestamp":1596542871,"flushtimestamp":1596542410,"timestamp":1596538800}
+{"version":1,"type":"digest"}
+{"digest":"1d9c84df015226379b4b399c7007578bcbd84d6ba774287ad70189233743c048","timestamp":1596538800}
+{"version":1,"type":"digest"}
+{"digest":"95f10732cd8419b9f2f4038085ff7d818f56cb5cdf85c2608279da02874d57d1","timestamp":1596538800}
+{"version":1,"type":"digest"}
+{"digest":"f618f76e82b20affc013245f1082992d45cf182436ddc8a141e57dd72b5b9ea3","timestamp":1596538800}
+{"version":1,"type":"digestglobal"}
+{"digest":"1d9c84df015226379b4b399c7007578bcbd84d6ba774287ad70189233743c048","timestamp":1596538800}
+{"version":1,"type":"digestglobal"}
+{"digest":"6938e8fc4bacf49e3233620791f630ab1f93261ef969b35448f28a659139c900","timestamp":1593022320}
+{"version":1,"type":"digestglobal"}
+{"digest":"95f10732cd8419b9f2f4038085ff7d818f56cb5cdf85c2608279da02874d57d1","timestamp":1596538800}
+{"version":1,"type":"digestglobal"}
+{"digest":"f0905695c1f6f198d17d3a528c1bcd6c48b5da6c2e22440b9af6590ad8c1615a","timestamp":1593018060}
+{"version":1,"type":"digestglobal"}
+{"digest":"f618f76e82b20affc013245f1082992d45cf182436ddc8a141e57dd72b5b9ea3","timestamp":1596538800}
diff --git a/dump.dump.postgres b/dump.dump.postgres
new file mode 100644
index 0000000..a50ebec
--- /dev/null
+++ b/dump.dump.postgres
@@ -0,0 +1,11 @@
+=== Root: /Users/amass/Library/Application Support/Dcrtimed/data/testnet3
+{"version":1,"type":"flush"}
+{"root":[149,241,7,50,205,132,25,185,242,244,3,128,133,255,125,129,143,86,203,92,223,133,194,96,130,121,218,2,135,77,87,209],"hashes":[[149,241,7,50,205,132,25,185,242,244,3,128,133,255,125,129,143,86,203,92,223,133,194,96,130,121,218,2,135,77,87,209]],"tx":[160,39,86,250,159,124,163,38,127,51,153,110,20,39,200,250,226,120,127,225,217,15,29,11,12,212,36,5,198,214,127,107],"chaintimestamp":1597797160,"flushtimestamp":1597797138,"timestamp":1597795200}
+{"version":1,"type":"digest"}
+{"digest":"95f10732cd8419b9f2f4038085ff7d818f56cb5cdf85c2608279da02874d57d1","timestamp":1597795200}
+{"version":1,"type":"flush"}
+{"root":[229,178,116,98,187,223,189,219,237,192,72,4,182,53,196,133,179,5,151,162,205,195,161,224,32,95,25,189,236,214,130,6],"hashes":[[205,159,24,66,7,91,140,76,54,113,158,186,130,98,144,241,192,229,87,139,221,223,53,32,38,243,122,99,178,17,6,160],[170,165,190,50,176,97,130,215,192,241,139,81,108,78,12,171,21,42,226,176,223,192,226,42,99,95,73,201,155,146,167,47]],"tx":[202,1,158,212,237,235,113,217,212,45,118,96,44,136,215,47,67,152,110,91,236,78,81,237,4,252,28,198,26,28,4,143],"chaintimestamp":1598435106,"flushtimestamp":1598435050,"timestamp":1598434980}
+{"version":1,"type":"digest"}
+{"digest":"cd9f1842075b8c4c36719eba826290f1c0e5578bdddf352026f37a63b21106a0","timestamp":1598434980}
+{"version":1,"type":"digest"}
+{"digest":"aaa5be32b06182d7c0f18b516c4e0cab152ae2b0dfc0e22a635f49c99b92a72f","timestamp":1598434980}

From 2e643a7af7c3d9292294271eef07b268a6ff9134 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 29 Aug 2020 13:14:08 +0300
Subject: [PATCH 48/66] cleanup logs

---
 dump.dump          |  27 ---------
 dump.dump.postgres |  11 ----
 fsck.log           | 141 ---------------------------------------------
 3 files changed, 179 deletions(-)
 delete mode 100644 dump.dump
 delete mode 100644 dump.dump.postgres
 delete mode 100644 fsck.log

diff --git a/dump.dump b/dump.dump
deleted file mode 100644
index f1d72bd..0000000
--- a/dump.dump
+++ /dev/null
@@ -1,27 +0,0 @@
-=== Root: /Users/amass/Library/Application Support/Dcrtimed/data/testnet3
-{"version":1,"type":"flush"}
-{"root":[240,144,86,149,193,246,241,152,209,125,58,82,140,27,205,108,72,181,218,108,46,34,68,11,154,246,89,10,216,193,97,90],"hashes":[[240,144,86,149,193,246,241,152,209,125,58,82,140,27,205,108,72,181,218,108,46,34,68,11,154,246,89,10,216,193,97,90]],"tx":[9,64,243,196,131,184,27,246,124,169,117,14,211,101,203,21,90,170,67,78,119,23,199,18,34,182,145,170,122,231,211,247],"chaintimestamp":1593018261,"flushtimestamp":1593018130,"timestamp":1593018060}
-{"version":1,"type":"digest"}
-{"digest":"f0905695c1f6f198d17d3a528c1bcd6c48b5da6c2e22440b9af6590ad8c1615a","timestamp":1593018060}
-{"version":1,"type":"flush"}
-{"root":[105,56,232,252,75,172,244,158,50,51,98,7,145,246,48,171,31,147,38,30,249,105,179,84,72,242,138,101,145,57,201,0],"hashes":[[105,56,232,252,75,172,244,158,50,51,98,7,145,246,48,171,31,147,38,30,249,105,179,84,72,242,138,101,145,57,201,0]],"tx":[150,212,112,179,197,252,59,42,14,250,95,93,167,209,181,52,16,167,187,30,101,16,243,81,115,97,197,249,73,106,119,52],"chaintimestamp":1593022396,"flushtimestamp":1593022390,"timestamp":1593022320}
-{"version":1,"type":"digest"}
-{"digest":"6938e8fc4bacf49e3233620791f630ab1f93261ef969b35448f28a659139c900","timestamp":1593022320}
-{"version":1,"type":"flush"}
-{"root":[164,222,239,177,139,88,233,240,13,185,175,128,14,69,116,113,226,102,141,200,222,154,42,89,163,136,194,123,180,15,95,224],"hashes":[[29,156,132,223,1,82,38,55,155,75,57,156,112,7,87,139,203,216,77,107,167,116,40,122,215,1,137,35,55,67,192,72],[149,241,7,50,205,132,25,185,242,244,3,128,133,255,125,129,143,86,203,92,223,133,194,96,130,121,218,2,135,77,87,209],[246,24,247,110,130,178,10,255,192,19,36,95,16,130,153,45,69,207,24,36,54,221,200,161,65,229,125,215,43,91,158,163]],"tx":[190,108,7,5,226,66,25,119,249,12,45,219,184,143,216,11,164,84,117,17,0,42,29,215,165,190,102,47,30,68,248,59],"chaintimestamp":1596542871,"flushtimestamp":1596542410,"timestamp":1596538800}
-{"version":1,"type":"digest"}
-{"digest":"1d9c84df015226379b4b399c7007578bcbd84d6ba774287ad70189233743c048","timestamp":1596538800}
-{"version":1,"type":"digest"}
-{"digest":"95f10732cd8419b9f2f4038085ff7d818f56cb5cdf85c2608279da02874d57d1","timestamp":1596538800}
-{"version":1,"type":"digest"}
-{"digest":"f618f76e82b20affc013245f1082992d45cf182436ddc8a141e57dd72b5b9ea3","timestamp":1596538800}
-{"version":1,"type":"digestglobal"}
-{"digest":"1d9c84df015226379b4b399c7007578bcbd84d6ba774287ad70189233743c048","timestamp":1596538800}
-{"version":1,"type":"digestglobal"}
-{"digest":"6938e8fc4bacf49e3233620791f630ab1f93261ef969b35448f28a659139c900","timestamp":1593022320}
-{"version":1,"type":"digestglobal"}
-{"digest":"95f10732cd8419b9f2f4038085ff7d818f56cb5cdf85c2608279da02874d57d1","timestamp":1596538800}
-{"version":1,"type":"digestglobal"}
-{"digest":"f0905695c1f6f198d17d3a528c1bcd6c48b5da6c2e22440b9af6590ad8c1615a","timestamp":1593018060}
-{"version":1,"type":"digestglobal"}
-{"digest":"f618f76e82b20affc013245f1082992d45cf182436ddc8a141e57dd72b5b9ea3","timestamp":1596538800}
diff --git a/dump.dump.postgres b/dump.dump.postgres
deleted file mode 100644
index a50ebec..0000000
--- a/dump.dump.postgres
+++ /dev/null
@@ -1,11 +0,0 @@
-=== Root: /Users/amass/Library/Application Support/Dcrtimed/data/testnet3
-{"version":1,"type":"flush"}
-{"root":[149,241,7,50,205,132,25,185,242,244,3,128,133,255,125,129,143,86,203,92,223,133,194,96,130,121,218,2,135,77,87,209],"hashes":[[149,241,7,50,205,132,25,185,242,244,3,128,133,255,125,129,143,86,203,92,223,133,194,96,130,121,218,2,135,77,87,209]],"tx":[160,39,86,250,159,124,163,38,127,51,153,110,20,39,200,250,226,120,127,225,217,15,29,11,12,212,36,5,198,214,127,107],"chaintimestamp":1597797160,"flushtimestamp":1597797138,"timestamp":1597795200}
-{"version":1,"type":"digest"}
-{"digest":"95f10732cd8419b9f2f4038085ff7d818f56cb5cdf85c2608279da02874d57d1","timestamp":1597795200}
-{"version":1,"type":"flush"}
-{"root":[229,178,116,98,187,223,189,219,237,192,72,4,182,53,196,133,179,5,151,162,205,195,161,224,32,95,25,189,236,214,130,6],"hashes":[[205,159,24,66,7,91,140,76,54,113,158,186,130,98,144,241,192,229,87,139,221,223,53,32,38,243,122,99,178,17,6,160],[170,165,190,50,176,97,130,215,192,241,139,81,108,78,12,171,21,42,226,176,223,192,226,42,99,95,73,201,155,146,167,47]],"tx":[202,1,158,212,237,235,113,217,212,45,118,96,44,136,215,47,67,152,110,91,236,78,81,237,4,252,28,198,26,28,4,143],"chaintimestamp":1598435106,"flushtimestamp":1598435050,"timestamp":1598434980}
-{"version":1,"type":"digest"}
-{"digest":"cd9f1842075b8c4c36719eba826290f1c0e5578bdddf352026f37a63b21106a0","timestamp":1598434980}
-{"version":1,"type":"digest"}
-{"digest":"aaa5be32b06182d7c0f18b516c4e0cab152ae2b0dfc0e22a635f49c99b92a72f","timestamp":1598434980}
diff --git a/fsck.log b/fsck.log
deleted file mode 100644
index fc6ee5c..0000000
--- a/fsck.log
+++ /dev/null
@@ -1,141 +0,0 @@
-{"version":1,"timestamp":1598202410,"action":"header"}
-{"version":1,"start":1598202410,"dryrun":true}
-
-{"version":1,"timestamp":1598219261,"action":"header"}
-{"version":1,"start":1598219261,"dryrun":true}
-
-{"version":1,"timestamp":1598219280,"action":"header"}
-{"version":1,"start":1598219280,"dryrun":true}
-
-{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
-
-{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
-
-{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
-
-{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
-
-{"version":1,"timestamp":1598219282,"action":"deletetimestamp"}
-{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
-
-{"version":1,"timestamp":1598264220,"action":"header"}
-{"version":1,"start":1598264220,"dryrun":true}
-
-{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
-
-{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
-{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
-
-{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
-
-{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
-
-{"version":1,"timestamp":1598264223,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
-
-{"version":1,"timestamp":1598265824,"action":"header"}
-{"version":1,"start":1598265824,"dryrun":true}
-
-{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
-
-{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
-{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
-
-{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
-
-{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
-
-{"version":1,"timestamp":1598265826,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
-
-{"version":1,"timestamp":1598265914,"action":"header"}
-{"version":1,"start":1598265914,"dryrun":true}
-
-{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
-
-{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
-
-{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
-
-{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
-
-{"version":1,"timestamp":1598265916,"action":"deletetimestamp"}
-{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
-
-{"version":1,"timestamp":1598265981,"action":"header"}
-{"version":1,"start":1598265981,"dryrun":true}
-
-{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
-
-{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
-
-{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
-{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
-
-{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
-
-{"version":1,"timestamp":1598265983,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
-
-{"version":1,"timestamp":1598266009,"action":"header"}
-{"version":1,"start":1598266009,"dryrun":true}
-
-{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
-
-{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
-
-{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
-
-{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
-
-{"version":1,"timestamp":1598266011,"action":"deletetimestamp"}
-{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
-
-{"version":1,"timestamp":1598393477,"action":"header"}
-{"version":1,"start":1598393477,"dryrun":true}
-
-{"version":1,"timestamp":1598393737,"action":"header"}
-{"version":1,"start":1598393737,"dryrun":true}
-
-{"version":1,"timestamp":1598393749,"action":"header"}
-{"version":1,"start":1598393749,"dryrun":true}
-
-{"version":1,"timestamp":1598394696,"action":"header"}
-{"version":1,"start":1598394696,"dryrun":true}
-
-{"version":1,"timestamp":1598435066,"action":"header"}
-{"version":1,"start":1598435066,"dryrun":true}
-
-{"version":1,"timestamp":1598435253,"action":"header"}
-{"version":1,"start":1598435253,"dryrun":true}
-
-{"version":1,"timestamp":1598436054,"action":"header"}
-{"version":1,"start":1598436054,"dryrun":true}
-
-{"version":1,"timestamp":1598442896,"action":"header"}
-{"version":1,"start":1598442896,"dryrun":true}
-
-{"version":1,"timestamp":1598442901,"action":"header"}
-{"version":1,"start":1598442901,"dryrun":true}
-

From 5f8956b4a623ddd4730320c5b7f1525c31ea20de Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 29 Aug 2020 16:33:27 +0300
Subject: [PATCH 49/66] fix cli

---
 cmd/dcrtime_dumpdb/dcrtime_dumpdb.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
index c3fa5db..85e15b0 100644
--- a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
+++ b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
@@ -51,6 +51,7 @@ func _main() error {
 	case "filesystem":
 		if *restore {
 			b, err = filesystem.NewRestore(*destination)
+			break
 		}
 		b, err = filesystem.NewDump(root)
 	case "postgres":
@@ -63,6 +64,7 @@ func _main() error {
 		}
 		if *restore {
 			b, err = filesystem.NewRestore(*destination)
+			break
 		}
 		b, err = postgres.NewDump(loadedCfg.PostgresHost,
 			net,

From 0c73d1fd74be4768aa225d5065d511bc60b5217a Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 30 Aug 2020 01:30:05 +0300
Subject: [PATCH 50/66] add postgres Restore implementation

---
 cmd/dcrtime_dumpdb/dcrtime_dumpdb.go  |  6 +-
 cmd/dcrtime_fsck/dcrtime_fsck.go      |  2 +-
 dcrtimed/backend/postgres/dump.go     | 97 ++++++++++++++++++++++++++-
 dcrtimed/backend/postgres/postgres.go |  9 ---
 dcrtimed/backend/postgres/sql.go      | 29 +++++---
 5 files changed, 119 insertions(+), 24 deletions(-)

diff --git a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
index 85e15b0..e3831ac 100644
--- a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
+++ b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
@@ -62,11 +62,7 @@ func _main() error {
 		default:
 			net = "mainnet"
 		}
-		if *restore {
-			b, err = filesystem.NewRestore(*destination)
-			break
-		}
-		b, err = postgres.NewDump(loadedCfg.PostgresHost,
+		b, err = postgres.NewDB(loadedCfg.PostgresHost,
 			net,
 			loadedCfg.PostgresRootCert,
 			loadedCfg.PostgresCert,
diff --git a/cmd/dcrtime_fsck/dcrtime_fsck.go b/cmd/dcrtime_fsck/dcrtime_fsck.go
index 44a1ccf..8a725bd 100644
--- a/cmd/dcrtime_fsck/dcrtime_fsck.go
+++ b/cmd/dcrtime_fsck/dcrtime_fsck.go
@@ -70,7 +70,7 @@ func _main() error {
 		default:
 			net = "mainnet"
 		}
-		b, err = postgres.NewDump(loadedCfg.PostgresHost,
+		b, err = postgres.NewDB(loadedCfg.PostgresHost,
 			net,
 			loadedCfg.PostgresRootCert,
 			loadedCfg.PostgresCert,
diff --git a/dcrtimed/backend/postgres/dump.go b/dcrtimed/backend/postgres/dump.go
index b30f727..d8c6821 100644
--- a/dcrtimed/backend/postgres/dump.go
+++ b/dcrtimed/backend/postgres/dump.go
@@ -16,7 +16,7 @@ import (
 	"github.com/decred/dcrtime/dcrtimed/backend"
 )
 
-func NewDump(host, net, rootCert, cert, key string) (*Postgres, error) {
+func NewDB(host, net, rootCert, cert, key string) (*Postgres, error) {
 	// Connect to database
 	dbName := net + "_dcrtime"
 	h := "postgresql://" + dbUser + "@" + host + "/" + dbName
@@ -176,3 +176,98 @@ func dumpFlushRecord(f *os.File, flushRecord *backend.FlushRecord) {
 		fmt.Fprintf(f, "  Flushed      : %x\n", *v)
 	}
 }
+
+func (pg *Postgres) restoreFlushRecord(verbose bool, fr backend.FlushRecordJSON) error {
+	frOld := backend.FlushRecord{
+		Root:           fr.Root,
+		Hashes:         fr.Hashes,
+		Tx:             fr.Tx,
+		ChainTimestamp: fr.ChainTimestamp,
+		FlushTimestamp: fr.FlushTimestamp,
+	}
+
+	err := pg.insertAnchor(frOld)
+	if err != nil {
+		return err
+	}
+	if verbose {
+		fmt.Printf("Restored flushed anchor: (merkle:%v)\n", hex.EncodeToString(
+			fr.Root[:]))
+	}
+	return nil
+}
+
+// Restore reads JSON encoded database contents and recreates the postgres
+// database.
+func (pg *Postgres) Restore(f *os.File, verbose bool, location string) error {
+	d := json.NewDecoder(f)
+
+	// we store each flushed timestamp merkle root in order to insert it
+	// when restoring digests to the records table
+	tssMerkles := make(map[int64][sha256.Size]byte)
+
+	state := 0
+	for {
+		switch state {
+		case 0:
+			// Type
+			var t backend.RecordType
+			err := d.Decode(&t)
+			if err != nil {
+				return err
+			}
+
+			// Check version we understand
+			if t.Version != backend.RecordTypeVersion {
+				return fmt.Errorf("unknown version %v",
+					t.Version)
+			}
+
+			// Determine record type
+			switch t.Type {
+			case backend.RecordTypeDigestReceived:
+				state = 1
+			case backend.RecordTypeFlushRecord:
+				state = 2
+			default:
+				return fmt.Errorf("invalid record type: %v",
+					t.Type)
+			}
+		case 1:
+			// DigestReceived
+			var dr backend.DigestReceived
+			err := d.Decode(&dr)
+			if err != nil {
+				return err
+			}
+			// if digest' timestamp was anchored, get anchor' merkle root
+			// to insert it into records table
+			anchorRoot := tssMerkles[dr.Timestamp]
+			err = pg.insertRestoredDigest(dr, anchorRoot)
+			if err != nil {
+				return err
+			}
+			state = 0
+		case 2:
+			// Flushrecord
+			var fr backend.FlushRecordJSON
+			err := d.Decode(&fr)
+			if err != nil {
+				return err
+			}
+			err = pg.restoreFlushRecord(verbose, fr)
+			if err != nil {
+				return err
+			}
+			_, ok := tssMerkles[fr.Timestamp]
+			if !ok {
+				tssMerkles[fr.Timestamp] = fr.Root
+			}
+			state = 0
+		default:
+			// Illegal
+			return fmt.Errorf("invalid state %v", state)
+		}
+	}
+	return nil
+}
diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 171bfab..3ecf3dd 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -10,7 +10,6 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"os"
 	"path/filepath"
 	"sync"
 	"time"
@@ -341,14 +340,6 @@ func (pg *Postgres) Close() {
 	pg.db.Close()
 }
 
-// Restore recreates the the database from the provided file
-// descriptor. The verbose flag is set to true to indicate that this
-// call may parint to stdout. The provided string describes the target
-// location and is implementation specific.
-func (pg *Postgres) Restore(*os.File, bool, string) error {
-	return nil
-}
-
 // GetBalance retrieves balance information for the wallet
 // backing this instance
 func (pg *Postgres) GetBalance() (*backend.GetBalanceResult, error) {
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index c1d4a42..82c0258 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -3,12 +3,30 @@ package postgres
 import (
 	"crypto/sha256"
 	"database/sql"
+	"encoding/hex"
 
 	"github.com/decred/dcrd/chaincfg/chainhash"
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/merkle"
 )
 
+func (pg *Postgres) insertRestoredDigest(dr backend.DigestReceived, merkle [sha256.Size]byte) error {
+	q := `INSERT INTO records (collection_timestamp, digest, anchor_merkle)
+				VALUES($1, $2, $3)`
+
+	digest, err := hex.DecodeString(dr.Digest)
+	err = pg.db.QueryRow(q, dr.Timestamp, digest, merkle[:]).Scan()
+	if err != nil {
+		// The insert command won't return any value, the following error is
+		// expected and means anchor row inserted successfully
+		if err.Error() == "sql: no rows in result set" {
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
 func (pg *Postgres) getAllRecordsTimestamps() (*[]int64, error) {
 	q := `SELECT DISTINCT collection_timestamp FROM records`
 
@@ -71,11 +89,6 @@ func (pg *Postgres) updateAnchorChainTs(fr *backend.FlushRecord) error {
 
 	err := pg.db.QueryRow(q, fr.ChainTimestamp, fr.Root[:]).Scan()
 	if err != nil {
-		// The insert command won't return any value, the following error is
-		// expected and means anchor row inserted successfully
-		if err.Error() == "sql: no rows in result set" {
-			return nil
-		}
 		return err
 	}
 	return nil
@@ -93,11 +106,11 @@ func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot [sha256.Size]byte)
 }
 
 func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
-	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp)
-				VALUES($1, $2, $3)`
+	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp, chain_timestamp)
+				VALUES($1, $2, $3, $4)`
 
 	err := pg.db.QueryRow(q, fr.Root[:], fr.Tx[:],
-		fr.FlushTimestamp).Scan()
+		fr.FlushTimestamp, fr.ChainTimestamp).Scan()
 	if err != nil {
 		// The insert command won't return any value, the following error is
 		// expected and means anchor row inserted successfully

From 8fd25b45456871ef6e6d1b4ff2313da0becb93f7 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 30 Aug 2020 01:36:13 +0300
Subject: [PATCH 51/66] cleanup

---
 dcrtimed/backend/postgres/dump.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dcrtimed/backend/postgres/dump.go b/dcrtimed/backend/postgres/dump.go
index d8c6821..0ca0af5 100644
--- a/dcrtimed/backend/postgres/dump.go
+++ b/dcrtimed/backend/postgres/dump.go
@@ -269,5 +269,4 @@ func (pg *Postgres) Restore(f *os.File, verbose bool, location string) error {
 			return fmt.Errorf("invalid state %v", state)
 		}
 	}
-	return nil
 }

From 90f3674b874788fbe4aafc77edee3649603e7313 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 30 Aug 2020 01:38:42 +0300
Subject: [PATCH 52/66] delete commited log file

---
 cmd/dcrtime_fsck/fsck.log | 36 ------------------------------------
 1 file changed, 36 deletions(-)
 delete mode 100644 cmd/dcrtime_fsck/fsck.log

diff --git a/cmd/dcrtime_fsck/fsck.log b/cmd/dcrtime_fsck/fsck.log
deleted file mode 100644
index b159663..0000000
--- a/cmd/dcrtime_fsck/fsck.log
+++ /dev/null
@@ -1,36 +0,0 @@
-{"version":1,"timestamp":1598391272,"action":"header"}
-{"version":1,"start":1598391272,"dryrun":true}
-
-{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
-
-{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
-
-{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
-
-{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
-
-{"version":1,"timestamp":1598391274,"action":"deletetimestamp"}
-{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
-
-{"version":1,"timestamp":1598392394,"action":"header"}
-{"version":1,"start":1598392394,"dryrun":true}
-
-{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593024000,"directory":"20200624.184000"}
-
-{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
-{"version":1,"timestamp":1596553200,"directory":"20200804.150000"}
-
-{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593018120,"directory":"20200624.170200"}
-
-{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023880,"directory":"20200624.183800"}
-
-{"version":1,"timestamp":1598392395,"action":"deletetimestamp"}
-{"version":1,"timestamp":1593023940,"directory":"20200624.183900"}
-

From 2bbdd9913fe7837ee0ec74d13d820732c113dd8d Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 30 Aug 2020 21:29:13 +0300
Subject: [PATCH 53/66] log root only when filesystem is used

---
 cmd/dcrtime_dumpdb/dcrtime_dumpdb.go | 4 +++-
 cmd/dcrtime_fsck/README.md           | 6 ++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
index e3831ac..466d4b2 100644
--- a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
+++ b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
@@ -54,6 +54,9 @@ func _main() error {
 			break
 		}
 		b, err = filesystem.NewDump(root)
+		if !*dumpJSON {
+			fmt.Printf("=== Root: %v\n", root)
+		}
 	case "postgres":
 		var net string
 		switch loadedCfg.TestNet {
@@ -75,7 +78,6 @@ func _main() error {
 	}
 	defer b.Close()
 
-	fmt.Printf("=== Root: %v\n", root)
 	if *restore {
 		return b.Restore(os.Stdin, true, *destination)
 	}
diff --git a/cmd/dcrtime_fsck/README.md b/cmd/dcrtime_fsck/README.md
index 7464a84..bfa52ff 100644
--- a/cmd/dcrtime_fsck/README.md
+++ b/cmd/dcrtime_fsck/README.md
@@ -1,11 +1,13 @@
 dcrtime_fsck
 ============
 
-// XXX: update docs and mention postgres
-
 The filesystem backend can under rare circumstances become incoherent. This
 tool iterates over all timestamp directories and corrects known failures.
 
+PostgreSQL backend also supported, `dcrtime_fsck` digs in `dcrtimed.conf`
+file to determine which backend to run, and uses the provided db params 
+in case postgres backend is running.
+
 ## Flags
 
 ```

From f5ea7228fe5176195e6ec4bffd178a914e81f685 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 30 Aug 2020 21:46:30 +0300
Subject: [PATCH 54/66] `dcrtime_fsck`: require -destination flag only for
 filesystem backend

---
 cmd/dcrtime_dumpdb/dcrtime_dumpdb.go | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
index 466d4b2..326ae76 100644
--- a/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
+++ b/cmd/dcrtime_dumpdb/dcrtime_dumpdb.go
@@ -26,11 +26,6 @@ var (
 func _main() error {
 	flag.Parse()
 
-	if *restore {
-		if *destination == "" {
-			return fmt.Errorf("-destination must be set")
-		}
-	}
 	loadedCfg, err := loadConfig()
 	if err != nil {
 		return fmt.Errorf("Could not load configuration file: %v", err)
@@ -50,6 +45,9 @@ func _main() error {
 	switch (*loadedCfg).Backend {
 	case "filesystem":
 		if *restore {
+			if *destination == "" {
+				return fmt.Errorf("-destination must be set")
+			}
 			b, err = filesystem.NewRestore(*destination)
 			break
 		}

From 67fb76bdc21f1fa7cf72572d0d196f34f696ab99 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Fri, 11 Sep 2020 14:22:31 +0300
Subject: [PATCH 55/66] add sql models

---
 dcrtimed/backend/postgres/dump.go         |  50 +++++---
 dcrtimed/backend/postgres/fsck.go         |  24 ++--
 dcrtimed/backend/postgres/models.go       |  35 ++++++
 dcrtimed/backend/postgres/postgres.go     |  76 ++++++++----
 dcrtimed/backend/postgres/sql.go          | 141 ++++++++++++----------
 dcrtimed/backend/postgres/testpostgres.go |   5 +
 6 files changed, 221 insertions(+), 110 deletions(-)
 create mode 100644 dcrtimed/backend/postgres/models.go
 create mode 100644 dcrtimed/backend/postgres/testpostgres.go

diff --git a/dcrtimed/backend/postgres/dump.go b/dcrtimed/backend/postgres/dump.go
index 0ca0af5..4a21164 100644
--- a/dcrtimed/backend/postgres/dump.go
+++ b/dcrtimed/backend/postgres/dump.go
@@ -5,6 +5,7 @@
 package postgres
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"database/sql"
 	"encoding/hex"
@@ -13,6 +14,7 @@ import (
 	"net/url"
 	"os"
 
+	"github.com/decred/dcrd/chaincfg/chainhash"
 	"github.com/decred/dcrtime/dcrtimed/backend"
 )
 
@@ -70,7 +72,7 @@ func (pg *Postgres) dumpTimestamps(f *os.File, verbose bool) error {
 }
 
 func (pg *Postgres) dumpTimestamp(f *os.File, verbose bool, ts int64) error {
-	exists, records, flushTs, err := pg.getRecordsByServerTs(ts)
+	exists, records, err := pg.getRecordsByServerTs(ts)
 	if err != nil {
 		return err
 	}
@@ -84,18 +86,24 @@ func (pg *Postgres) dumpTimestamp(f *os.File, verbose bool, ts int64) error {
 		fr       backend.FlushRecord
 		digests  = make([]backend.DigestReceived, 0, 10000)
 	)
-	for _, r := range records {
-		if r.MerkleRoot != [sha256.Size]byte{} && !anchored {
+	for _, ar := range records {
+		if !bytes.Equal(ar.Anchor.Merkle, []byte{}) && !anchored {
 			anchored = true
-			fr.Root = r.MerkleRoot
-			fr.Tx = r.Tx
-			fr.ChainTimestamp = r.AnchoredTimestamp
-			fr.FlushTimestamp = flushTs
+			copy(fr.Root[:], ar.Anchor.Merkle[:sha256.Size])
+			tx, err := chainhash.NewHash(ar.Anchor.TxHash[:])
+			if err != nil {
+				return err
+			}
+			fr.Tx = *tx
+			fr.ChainTimestamp = ar.Anchor.ChainTimestamp
+			fr.FlushTimestamp = ar.Anchor.FlushTimestamp
 		}
-		fr.Hashes = append(fr.Hashes, &r.Digest)
+		var digest [sha256.Size]byte
+		copy(digest[:], ar.Record.Digest[:])
+		fr.Hashes = append(fr.Hashes, &digest)
 		digests = append(digests, backend.DigestReceived{
-			Digest:    hex.EncodeToString(r.Digest[:]),
-			Timestamp: r.Timestamp,
+			Digest:    hex.EncodeToString(ar.Record.Digest[:]),
+			Timestamp: ar.Record.CollectionTimestamp,
 		})
 	}
 
@@ -178,21 +186,20 @@ func dumpFlushRecord(f *os.File, flushRecord *backend.FlushRecord) {
 }
 
 func (pg *Postgres) restoreFlushRecord(verbose bool, fr backend.FlushRecordJSON) error {
-	frOld := backend.FlushRecord{
-		Root:           fr.Root,
-		Hashes:         fr.Hashes,
-		Tx:             fr.Tx,
+	a := Anchor{
+		Merkle:         fr.Root[:],
+		TxHash:         fr.Tx[:],
 		ChainTimestamp: fr.ChainTimestamp,
 		FlushTimestamp: fr.FlushTimestamp,
 	}
 
-	err := pg.insertAnchor(frOld)
+	err := pg.insertAnchor(a)
 	if err != nil {
 		return err
 	}
 	if verbose {
 		fmt.Printf("Restored flushed anchor: (merkle:%v)\n", hex.EncodeToString(
-			fr.Root[:]))
+			a.Merkle))
 	}
 	return nil
 }
@@ -243,7 +250,16 @@ func (pg *Postgres) Restore(f *os.File, verbose bool, location string) error {
 			// if digest' timestamp was anchored, get anchor' merkle root
 			// to insert it into records table
 			anchorRoot := tssMerkles[dr.Timestamp]
-			err = pg.insertRestoredDigest(dr, anchorRoot)
+			digest, err := hex.DecodeString(dr.Digest)
+			if err != nil {
+				return err
+			}
+			r := Record{
+				CollectionTimestamp: dr.Timestamp,
+				Digest:              digest,
+				AnchorMerkle:        anchorRoot[:],
+			}
+			err = pg.insertRestoredDigest(r)
 			if err != nil {
 				return err
 			}
diff --git a/dcrtimed/backend/postgres/fsck.go b/dcrtimed/backend/postgres/fsck.go
index dfebe98..7adb8a4 100644
--- a/dcrtimed/backend/postgres/fsck.go
+++ b/dcrtimed/backend/postgres/fsck.go
@@ -15,6 +15,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/decred/dcrd/chaincfg/chainhash"
 	"github.com/decred/dcrd/txscript/v2"
 	"github.com/decred/dcrdata/api/types/v4"
 	"github.com/decred/dcrtime/dcrtimed/backend"
@@ -125,7 +126,7 @@ func journal(filename, action string, payload interface{}) error {
 //     anchor's merkle root from db
 // 3.2 Verify that the anchor merkle root on db exists on the blockchain.
 func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error {
-	exists, records, flushTs, err := pg.getRecordsByServerTs(ts)
+	exists, records, err := pg.getRecordsByServerTs(ts)
 	if err != nil {
 		return err
 	}
@@ -140,7 +141,7 @@ func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error
 	)
 	digests := make(map[string]int64)
 	for _, r := range records {
-		k := hex.EncodeToString(r.Digest[:])
+		k := hex.EncodeToString(r.Record.Digest[:])
 		if _, ok := digests[k]; ok {
 			// This really can't happen but we check it so that we
 			// can equate lengths later to determine if the map and
@@ -148,16 +149,23 @@ func (pg *Postgres) fsckTimestamp(options *backend.FsckOptions, ts int64) error
 			return fmt.Errorf("    *** ERROR duplicate key: %v", k)
 		}
 		digests[k] = ts
-		if r.MerkleRoot != [sha256.Size]byte{} && !anchored {
+		if !bytes.Equal(r.Anchor.Merkle, []byte{}) && !anchored {
 			anchored = true
-			fr.Root = r.MerkleRoot
-			fr.Tx = r.Tx
-			fr.FlushTimestamp = flushTs
+			copy(fr.Root[:], r.Anchor.Merkle[:sha256.Size])
+			tx, err := chainhash.NewHash(r.Anchor.TxHash[:])
+			if err != nil {
+				return err
+			}
+			fr.Tx = *tx
+			fr.FlushTimestamp = r.Anchor.FlushTimestamp
 		}
 		if options.PrintHashes {
-			fmt.Printf("Hash           : %v\n", hex.EncodeToString(r.Digest[:]))
+			fmt.Printf("Hash           : %v\n",
+				hex.EncodeToString(r.Record.Digest[:]))
 		}
-		fr.Hashes = append(fr.Hashes, &r.Digest)
+		var digest [sha256.Size]byte
+		copy(digest[:], r.Record.Digest[:])
+		fr.Hashes = append(fr.Hashes, &digest)
 	}
 	fmt.Printf("No duplicates found\n")
 	fmt.Printf("Anchored       : %v\n", anchored)
diff --git a/dcrtimed/backend/postgres/models.go b/dcrtimed/backend/postgres/models.go
new file mode 100644
index 0000000..94223a4
--- /dev/null
+++ b/dcrtimed/backend/postgres/models.go
@@ -0,0 +1,35 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package postgres
+
+import "github.com/decred/dcrtime/merkle"
+
+// Record records collected digests, it includes the collection timestamp
+// and the anchor merkle root if the digest was anchored
+type Record struct {
+	Digest              []byte
+	CollectionTimestamp int64
+	AnchorMerkle        []byte
+}
+
+// Anchor records anchors information, it includes the merkle root of all
+// digests included in a given anchor, the anchor's transaction hash, the flush
+// timestamp and the chain timestamp if the transaction has enough
+// confirmations.
+type Anchor struct {
+	Merkle         []byte
+	TxHash         []byte
+	ChainTimestamp int64
+	FlushTimestamp int64
+}
+
+// AnchoredRecord is a wrapper struct, used to return record's and it's
+// corresponding anchor information if digest was anchored.
+type AnchoredRecord struct {
+	Record Record
+	Anchor Anchor
+	// Not stored on db, calculated and returned to client for verification
+	MerklePath merkle.Branch
+}
diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 3ecf3dd..ce3f785 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -5,6 +5,7 @@
 package postgres
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"database/sql"
 	"errors"
@@ -14,6 +15,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/decred/dcrd/chaincfg/chainhash"
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/dcrtimewallet"
 	"github.com/decred/dcrtime/merkle"
@@ -109,7 +111,10 @@ func (pg *Postgres) lazyFlush(fr *backend.FlushRecord) (*dcrtimewallet.TxLookupR
 	fr.ChainTimestamp = res.Timestamp
 
 	// Update anchor row in database
-	err = pg.updateAnchorChainTs(fr)
+	err = pg.updateAnchorChainTs(Anchor{
+		ChainTimestamp: fr.ChainTimestamp,
+		Merkle:         fr.Root[:],
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -120,7 +125,7 @@ func (pg *Postgres) lazyFlush(fr *backend.FlushRecord) (*dcrtimewallet.TxLookupR
 	return res, nil
 }
 
-// Return timestamp information for given digests.
+// Get returns timestamp information for given digests.
 func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error) {
 	gdmes := make([]backend.GetResult, 0, len(digests))
 
@@ -135,7 +140,12 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 		gdme := backend.GetResult{
 			Digest: d,
 		}
-		found, err := pg.getRecordByDigest(d[:], &gdme)
+		ar := AnchoredRecord{
+			Record: Record{
+				Digest: d[:],
+			},
+		}
+		found, err := pg.getRecordByDigest(&ar)
 		if err != nil {
 			return nil, err
 		}
@@ -144,9 +154,18 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 			gdme.ErrorCode = backend.ErrorNotFound
 		} else {
 			// Override error code during testing
+			gdme.ErrorCode = backend.ErrorOK
+			gdme.MerklePath = ar.MerklePath
+			copy(gdme.MerkleRoot[:], ar.Anchor.Merkle[:])
+			tx, err := chainhash.NewHash(ar.Anchor.TxHash[:])
+			if err != nil {
+				return nil, err
+			}
+			gdme.Tx = *tx
 			if pg.testing {
 				gdme.ErrorCode = digestFound
-			} else if gdme.MerkleRoot != [sha256.Size]byte{} && gdme.AnchoredTimestamp == 0 {
+			} else if !bytes.Equal(ar.Anchor.Merkle, []byte{}) &&
+				gdme.AnchoredTimestamp == 0 {
 				// Lazyflush record if it was anchored but blockchain timestamp
 				// isn't avialable yet
 				fr := backend.FlushRecord{
@@ -193,7 +212,7 @@ func (pg *Postgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult
 			Timestamp: ts,
 		}
 		if pg.enableCollections {
-			exists, tsDigests, _, err := pg.getRecordsByServerTs(ts)
+			exists, records, err := pg.getRecordsByServerTs(ts)
 			if err != nil {
 				return nil, err
 			}
@@ -202,12 +221,18 @@ func (pg *Postgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult
 				gtme.ErrorCode = backend.ErrorNotFound
 			}
 			// copy ts digests
-			gtme.Digests = make([][sha256.Size]byte, 0, len(tsDigests))
-			for _, digest := range tsDigests {
-				gtme.Digests = append(gtme.Digests, (*digest).Digest)
-				gtme.Tx = (*digest).Tx
-				gtme.AnchoredTimestamp = (*digest).AnchoredTimestamp
-				gtme.MerkleRoot = (*digest).MerkleRoot
+			gtme.Digests = make([][sha256.Size]byte, 0, len(records))
+			for _, r := range records {
+				var d [sha256.Size]byte
+				copy(d[:], r.Record.Digest[:])
+				gtme.Digests = append(gtme.Digests, d)
+				tx, err := chainhash.NewHash(r.Anchor.TxHash)
+				if err != nil {
+					return nil, err
+				}
+				gtme.Tx = *tx
+				gtme.AnchoredTimestamp = r.Anchor.ChainTimestamp
+				copy(gtme.MerkleRoot[:], r.Anchor.Merkle[:sha256.Size])
 			}
 			// Lazyflush record if it was anchored but blockchain timestamp
 			// isn't avialable yet
@@ -239,7 +264,7 @@ func (pg *Postgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult
 	return gtmes, nil
 }
 
-// Store hashes and return timestamp and associated errors.  Put is
+// Put stores hashes and return timestamp and associated errors.  Put is
 // allowed to return transient errors.
 func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult, error) {
 	// Two-phase commit.
@@ -267,7 +292,7 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 
 	for _, hash := range hashes {
 		// Check if digest exists
-		exists, err := pg.checkIfDigestExists(hash[:])
+		exists, err := pg.isDigestExists(hash[:])
 		if err != nil {
 			return 0, []backend.PutResult{}, err
 		}
@@ -356,15 +381,23 @@ func (pg *Postgres) GetBalance() (*backend.GetBalanceResult, error) {
 
 // LastAnchor retrieves last successful anchor details
 func (pg *Postgres) LastAnchor() (*backend.LastAnchorResult, error) {
-	ts, merkle, tx, err := pg.getLatestAnchoredTimestamp()
+	ts, a, err := pg.getLatestAnchoredTimestamp()
 	if err != nil {
 		return nil, err
 	}
-
 	if ts == 0 {
 		return &backend.LastAnchorResult{}, nil
 	}
 
+	var (
+		merkle [sha256.Size]byte
+		tx     *chainhash.Hash
+	)
+	copy(merkle[:], a.Merkle[:sha256.Size])
+	tx, err = chainhash.NewHash(a.TxHash)
+	if err != nil {
+		return nil, err
+	}
 	var me backend.LastAnchorResult
 	me.Tx = *tx
 
@@ -372,7 +405,7 @@ func (pg *Postgres) LastAnchor() (*backend.LastAnchorResult, error) {
 	// and update db if info changed.
 	fr := backend.FlushRecord{
 		Tx:   *tx,
-		Root: *merkle,
+		Root: merkle,
 	}
 	txWalletInfo, err := pg.lazyFlush(&fr)
 
@@ -500,9 +533,8 @@ func (pg *Postgres) flush(ts int64) error {
 	mt := merkle.Tree(digests)
 	// Last element is root
 	root := *mt[len(mt)-1]
-	fr := backend.FlushRecord{
-		Root:           root,
-		Hashes:         mt[:len(digests)], // Only store hashes
+	a := Anchor{
+		Merkle:         root[:],
 		FlushTimestamp: time.Now().Unix(),
 	}
 	if !pg.testing {
@@ -513,17 +545,17 @@ func (pg *Postgres) flush(ts int64) error {
 		}
 		log.Infof("Flush timestamp: %v digests %v merkle: %x tx: %v",
 			ts, len(digests), root, tx.String())
-		fr.Tx = *tx
+		a.TxHash = (*tx)[:]
 	}
 
 	// Insert anchor data into db
-	err = pg.insertAnchor(fr)
+	err = pg.insertAnchor(a)
 	if err != nil {
 		return err
 	}
 
 	// Update timestamp's records merkle root
-	pg.updateRecordsAnchor(ts, fr.Root)
+	pg.updateRecordsAnchor(ts, a.Merkle)
 
 	// Update commit.
 	pg.commit++
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 82c0258..c355d30 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -1,21 +1,21 @@
 package postgres
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"database/sql"
-	"encoding/hex"
 
-	"github.com/decred/dcrd/chaincfg/chainhash"
-	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/merkle"
 )
 
-func (pg *Postgres) insertRestoredDigest(dr backend.DigestReceived, merkle [sha256.Size]byte) error {
+// insertRestoredDigest accepts a Record model and inserts it to the db
+//
+// this func used when restoring a backup
+func (pg *Postgres) insertRestoredDigest(r Record) error {
 	q := `INSERT INTO records (collection_timestamp, digest, anchor_merkle)
 				VALUES($1, $2, $3)`
 
-	digest, err := hex.DecodeString(dr.Digest)
-	err = pg.db.QueryRow(q, dr.Timestamp, digest, merkle[:]).Scan()
+	err := pg.db.QueryRow(q, r.CollectionTimestamp, r.Digest, r.AnchorMerkle).Scan()
 	if err != nil {
 		// The insert command won't return any value, the following error is
 		// expected and means anchor row inserted successfully
@@ -27,6 +27,7 @@ func (pg *Postgres) insertRestoredDigest(dr backend.DigestReceived, merkle [sha2
 	return nil
 }
 
+// getAllRecordsTimestamps returns all timestamps found in records table
 func (pg *Postgres) getAllRecordsTimestamps() (*[]int64, error) {
 	q := `SELECT DISTINCT collection_timestamp FROM records`
 
@@ -48,7 +49,9 @@ func (pg *Postgres) getAllRecordsTimestamps() (*[]int64, error) {
 	return &tss, nil
 }
 
-func (pg *Postgres) getLatestAnchoredTimestamp() (int64, *[sha256.Size]byte, *chainhash.Hash, error) {
+// getLatestAnchoredTimestamp returns latest anchor information - tx hash and
+// merkle root, additionally it returns anchor's collection timestamp
+func (pg *Postgres) getLatestAnchoredTimestamp() (int64, Anchor, error) {
 	q := `SELECT r.collection_timestamp, r.anchor_merkle, an.tx_hash
 				FROM records as r
 				LEFT JOIN anchors as an
@@ -58,59 +61,60 @@ func (pg *Postgres) getLatestAnchoredTimestamp() (int64, *[sha256.Size]byte, *ch
 				LIMIT 1`
 
 	rows, err := pg.db.Query(q)
+	a := Anchor{}
 	if err != nil {
-		return 0, nil, nil, err
+		return 0, a, err
 	}
 	defer rows.Close()
 
 	var (
 		serverTs   int64
 		txHash, mr []byte
-		merkle     [sha256.Size]byte
-		tx         *chainhash.Hash
 	)
 	for rows.Next() {
 		err = rows.Scan(&serverTs, &mr, &txHash)
 		if err != nil {
-			return 0, nil, nil, err
-		}
-		copy(merkle[:], mr[:sha256.Size])
-		tx, err = chainhash.NewHash(txHash)
-		if err != nil {
-			return 0, nil, nil, err
+			return 0, a, err
 		}
+		a.Merkle = mr
+		a.TxHash = txHash
 	}
-	return serverTs, &merkle, tx, nil
+	return serverTs, a, nil
 }
 
-func (pg *Postgres) updateAnchorChainTs(fr *backend.FlushRecord) error {
+// updateAnchorChainTs accepts an anchor and updates it's chain timestamp
+// on db
+func (pg *Postgres) updateAnchorChainTs(a Anchor) error {
 	q := `UPDATE anchors SET chain_timestamp = $1
 				WHERE merkle = $2`
 
-	err := pg.db.QueryRow(q, fr.ChainTimestamp, fr.Root[:]).Scan()
+	err := pg.db.QueryRow(q, a.ChainTimestamp, a.Merkle).Scan()
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot [sha256.Size]byte) error {
+// updateRecordsAnchor accepts a timestamp and anchor's merkle root and
+// updates all digests in records table with given merkle
+func (pg *Postgres) updateRecordsAnchor(ts int64, merkleRoot []byte) error {
 	q := `UPDATE records SET anchor_merkle = $1
 				WHERE collection_timestamp = $2`
 
-	err := pg.db.QueryRow(q, merkleRoot[:], ts).Scan()
+	err := pg.db.QueryRow(q, merkleRoot, ts).Scan()
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
+// insertAnchor accepts an anchor and inserts it to db
+func (pg *Postgres) insertAnchor(a Anchor) error {
 	q := `INSERT INTO anchors (merkle, tx_hash, flush_timestamp, chain_timestamp)
 				VALUES($1, $2, $3, $4)`
 
-	err := pg.db.QueryRow(q, fr.Root[:], fr.Tx[:],
-		fr.FlushTimestamp, fr.ChainTimestamp).Scan()
+	err := pg.db.QueryRow(q, a.Merkle, a.TxHash,
+		a.FlushTimestamp, a.ChainTimestamp).Scan()
 	if err != nil {
 		// The insert command won't return any value, the following error is
 		// expected and means anchor row inserted successfully
@@ -122,6 +126,9 @@ func (pg *Postgres) insertAnchor(fr backend.FlushRecord) error {
 	return nil
 }
 
+// getDigestsByMerkleRoot accepts a merkle root, selects all digests from
+// records table using given merkle, converts them to arrays([sha256.Size])
+// and then finally returns the result as array of pointers
 func (pg *Postgres) getDigestsByMerkleRoot(merkle []byte) ([]*[sha256.Size]byte, error) {
 	q := `SELECT digest from records WHERE anchor_merkle = $1`
 
@@ -147,6 +154,9 @@ func (pg *Postgres) getDigestsByMerkleRoot(merkle []byte) ([]*[sha256.Size]byte,
 	return digests, nil
 }
 
+// getDigestsByTimestamp accepts a timestamp, selects all digests from
+// records table using given timestamp, converts them to arrays([sha256.Size])
+// and then finally returns the result as array of pointers
 func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error) {
 	q := `SELECT digest from records WHERE collection_timestamp = $1`
 
@@ -168,6 +178,9 @@ func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error
 	return digests, nil
 }
 
+// getUnflushedTimestamps accepts current server timestamp and queries records
+// table to find all timestamps which aren't flushed yet - has no anchoring
+// information
 func (pg *Postgres) getUnflushedTimestamps(current int64) ([]int64, error) {
 	q := `SELECT DISTINCT collection_timestamp FROM records 
 				WHERE collection_timestamp != $1 AND anchor_merkle IS NULL`
@@ -188,7 +201,11 @@ func (pg *Postgres) getUnflushedTimestamps(current int64) ([]int64, error) {
 	return tss, nil
 }
 
-func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult, int64, error) {
+// getRecordsByServerTs accepts a server collection timestamps and returns
+// all records timetamped during that timestamp cycle, additionally it returns
+// the anchor information in case the timestamp's digests anchored on the
+// blockchain
+func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*AnchoredRecord, error) {
 	q := `SELECT r.anchor_merkle, an.tx_hash, an.chain_timestamp, r.digest,
         an.flush_timestamp
 				FROM records as r
@@ -198,7 +215,7 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult,
 
 	rows, err := pg.db.Query(q, ts)
 	if err != nil {
-		return false, nil, 0, err
+		return false, nil, err
 	}
 	defer rows.Close()
 	var (
@@ -208,35 +225,39 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*backend.GetResult,
 		chainTs sql.NullInt64
 		flushTs int64
 	)
-	r := []*backend.GetResult{}
+	r := []*AnchoredRecord{}
 	for rows.Next() {
-		rr := backend.GetResult{
-			Timestamp: ts,
+		ar := AnchoredRecord{
+			Record: Record{
+				CollectionTimestamp: ts,
+			},
 		}
 		err = rows.Scan(&mr, &txHash, &chainTs, &digest, &flushTs)
 		if err != nil {
-			return false, nil, 0, err
+			return false, nil, err
 		}
-		rr.Timestamp = ts
-		copy(rr.MerkleRoot[:], mr[:sha256.Size])
-		tx, err := chainhash.NewHash(txHash[:])
-		if err != nil {
-			return false, nil, 0, err
+		ar.Record.Digest = digest
+		ar.Anchor = Anchor{
+			Merkle:         mr,
+			TxHash:         txHash,
+			FlushTimestamp: flushTs,
 		}
-		rr.Tx = *tx
 		// chainTs can be NULL - handle safely
 		if chainTs.Valid {
-			rr.AnchoredTimestamp = chainTs.Int64
+			ar.Anchor.ChainTimestamp = chainTs.Int64
 		}
-		copy(rr.Digest[:], digest[:])
 
-		r = append(r, &rr)
+		r = append(r, &ar)
 	}
 
-	return len(r) > 0, r, flushTs, nil
+	return len(r) > 0, r, nil
 }
 
-func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool, error) {
+// getRecordByDigest accepts apointer to an AnchoredRecord which initially
+// includes only the record hash, it queries the db to get digest's data
+// including anchor's data if hash is anchored, it returns a bool to indicate
+// wether digest was found on db or not
+func (pg *Postgres) getRecordByDigest(ar *AnchoredRecord) (bool, error) {
 	q := `SELECT r.anchor_merkle, r.collection_timestamp, an.tx_hash, 
 				an.chain_timestamp
 				FROM records as r
@@ -244,51 +265,39 @@ func (pg *Postgres) getRecordByDigest(hash []byte, r *backend.GetResult) (bool,
 				ON r.anchor_merkle = an.merkle
 				WHERE r.digest = $1`
 
-	rows, err := pg.db.Query(q, hash)
+	rows, err := pg.db.Query(q, ar.Record.Digest)
 	if err != nil {
 		return false, err
 	}
 	defer rows.Close()
-	var (
-		mr       []byte
-		txHash   []byte
-		chainTs  sql.NullInt64
-		serverTs int64
-	)
+	var chainTs sql.NullInt64
 	for rows.Next() {
-		err = rows.Scan(&mr, &serverTs, &txHash, &chainTs)
-		if err != nil {
-			return false, err
-		}
-		r.Timestamp = serverTs
-		copy(r.MerkleRoot[:], mr[:sha256.Size])
-		tx, err := chainhash.NewHash(txHash[:])
+		err = rows.Scan(&ar.Anchor.Merkle, &ar.Record.CollectionTimestamp, &ar.Anchor.TxHash, &chainTs)
 		if err != nil {
 			return false, err
 		}
-		r.Tx = *tx
 		// chainTs can be NULL - handle safely
 		if chainTs.Valid {
-			r.AnchoredTimestamp = chainTs.Int64
+			ar.Anchor.ChainTimestamp = chainTs.Int64
 		}
-		if mr != nil {
-			hashes, err := pg.getDigestsByMerkleRoot(mr)
+		if !bytes.Equal(ar.Anchor.Merkle, []byte{}) {
+			hashes, err := pg.getDigestsByMerkleRoot(ar.Anchor.Merkle)
 			if err != nil {
 				return false, err
 			}
 			var digest [sha256.Size]byte
-			copy(digest[:], hash[:])
+			copy(digest[:], ar.Record.Digest[:])
 
 			// That pointer better not be nil!
-			r.MerklePath = *merkle.AuthPath(hashes, &digest)
+			ar.MerklePath = *merkle.AuthPath(hashes, &digest)
 		}
-		r.ErrorCode = backend.ErrorOK
 		return true, nil
 	}
 
 	return false, nil
 }
 
+// hasTable accepts a table name and checks if it was created
 func (pg *Postgres) hasTable(name string) (bool, error) {
 	q := `SELECT EXISTS (SELECT 
 				FROM information_schema.tables 
@@ -309,7 +318,9 @@ func (pg *Postgres) hasTable(name string) (bool, error) {
 	return exists, nil
 }
 
-func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
+// isDigestExists accept a digest and checks if it's already exists in
+// records table
+func (pg *Postgres) isDigestExists(hash []byte) (bool, error) {
 	q := `SELECT EXISTS 
 			  (SELECT FROM records WHERE digest = $1)`
 
@@ -328,6 +339,7 @@ func (pg *Postgres) checkIfDigestExists(hash []byte) (bool, error) {
 	return exists, nil
 }
 
+// createAnchorsTable creates anchors table
 func (pg *Postgres) createAnchorsTable() error {
 	_, err := pg.db.Exec(`CREATE TABLE public.anchors
 (
@@ -365,6 +377,7 @@ CREATE UNIQUE INDEX idx_tx_hash
 	return nil
 }
 
+// createRecordsTable creates records table
 func (pg *Postgres) createRecordsTable() error {
 	_, err := pg.db.Exec(`CREATE TABLE public.records
 (
@@ -402,6 +415,8 @@ CREATE UNIQUE INDEX idx_digest
 	return nil
 }
 
+// createsTables creates db tables needed for our postgres backend
+// implementation
 func (pg *Postgres) createTables() error {
 	exists, err := pg.hasTable(tableAnchors)
 	if err != nil {
diff --git a/dcrtimed/backend/postgres/testpostgres.go b/dcrtimed/backend/postgres/testpostgres.go
new file mode 100644
index 0000000..03d1090
--- /dev/null
+++ b/dcrtimed/backend/postgres/testpostgres.go
@@ -0,0 +1,5 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package postgres

From efaa1de79e7d7a3a7091f675c6f4aaa73e91138a Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 12 Sep 2020 01:13:54 +0300
Subject: [PATCH 56/66] fix get timestamp

---
 dcrtimed/backend/postgres/postgres.go | 4 +++-
 dcrtimed/backend/postgres/sql.go      | 4 ----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index ce3f785..f1a88e2 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -41,6 +41,7 @@ var (
 	// matching we mean both are hourly or every so many minutes.
 	//
 	// Seconds Minutes Hours Days Months DayOfWeek
+	// XXX XXX XXX revert minute cycles local dev change XXX XXX XXX
 	flushSchedule = "10 * * * * *" // On the hour + 10 seconds
 	duration      = time.Minute    // Default how often we combine digests
 
@@ -153,15 +154,16 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 		if !found {
 			gdme.ErrorCode = backend.ErrorNotFound
 		} else {
-			// Override error code during testing
 			gdme.ErrorCode = backend.ErrorOK
 			gdme.MerklePath = ar.MerklePath
 			copy(gdme.MerkleRoot[:], ar.Anchor.Merkle[:])
 			tx, err := chainhash.NewHash(ar.Anchor.TxHash[:])
+			gdme.AnchoredTimestamp = ar.Anchor.ChainTimestamp
 			if err != nil {
 				return nil, err
 			}
 			gdme.Tx = *tx
+			// Override error code during testing
 			if pg.testing {
 				gdme.ErrorCode = digestFound
 			} else if !bytes.Equal(ar.Anchor.Merkle, []byte{}) &&
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index c355d30..515dba6 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -147,10 +147,6 @@ func (pg *Postgres) getDigestsByMerkleRoot(merkle []byte) ([]*[sha256.Size]byte,
 		copy(digest[:], rawDigest[:])
 		digests = append(digests, &digest)
 	}
-	// Reverse hashes
-	for i, j := 0, len(digests)-1; i < j; i, j = i+1, j-1 {
-		digests[i], digests[j] = digests[j], digests[i]
-	}
 	return digests, nil
 }
 

From 86248ab169bcbd89c015b8697c413fcc1783ee44 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 12 Sep 2020 12:29:22 +0300
Subject: [PATCH 57/66] dump hashes in reverse to keep insertion order

---
 dcrtimed/backend/postgres/dump.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/dcrtimed/backend/postgres/dump.go b/dcrtimed/backend/postgres/dump.go
index 4a21164..a811f6f 100644
--- a/dcrtimed/backend/postgres/dump.go
+++ b/dcrtimed/backend/postgres/dump.go
@@ -86,7 +86,9 @@ func (pg *Postgres) dumpTimestamp(f *os.File, verbose bool, ts int64) error {
 		fr       backend.FlushRecord
 		digests  = make([]backend.DigestReceived, 0, 10000)
 	)
-	for _, ar := range records {
+	// Iterate over records in reverse to keep the order they were inserted in
+	for i := len(records) - 1; i >= 0; i-- {
+		ar := records[i]
 		if !bytes.Equal(ar.Anchor.Merkle, []byte{}) && !anchored {
 			anchored = true
 			copy(fr.Root[:], ar.Anchor.Merkle[:sha256.Size])

From 14cb20bf7c4cce4849e41274a715bbdffc068fa5 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sat, 12 Sep 2020 13:53:36 +0300
Subject: [PATCH 58/66] swallow "no rows in result" error when updating
 anchor's chain timestamp

---
 dcrtimed/backend/postgres/sql.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 515dba6..4814d8b 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -90,6 +90,11 @@ func (pg *Postgres) updateAnchorChainTs(a Anchor) error {
 
 	err := pg.db.QueryRow(q, a.ChainTimestamp, a.Merkle).Scan()
 	if err != nil {
+		// The update command won't return any value, the following error is
+		// expected and means anchor row updated successfully
+		if err.Error() == "sql: no rows in result set" {
+			return nil
+		}
 		return err
 	}
 	return nil

From 7f78482d43e52210835d0749fe9ce47ff379beb4 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 13 Sep 2020 00:35:26 +0300
Subject: [PATCH 59/66] ditch testing flag from Postgres struct & add
 TestPostgres

---
 dcrtimed/backend/postgres/postgres.go     | 42 +++++++----------------
 dcrtimed/backend/postgres/testpostgres.go | 19 ++++++++++
 2 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index f1a88e2..94a3006 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -28,10 +28,6 @@ const (
 	tableAnchors  = "anchors"
 	dbUser        = "dcrtimed"
 	confirmations = 6
-
-	// error codes that are overridden during tests only.
-	// digestFound is thrown if digest was found in records table
-	digestFound = 1001
 )
 
 var (
@@ -63,9 +59,7 @@ type Postgres struct {
 
 	wallet *dcrtimewallet.DcrtimeWallet // Wallet context.
 
-	// testing only entries
-	myNow   func() time.Time // Override time.Now()
-	testing bool             // Enabled during test
+	myNow func() time.Time // Override time.Now()
 }
 
 // now returns current time stamp rounded down to 1 hour.  All timestamps are
@@ -163,13 +157,11 @@ func (pg *Postgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error
 				return nil, err
 			}
 			gdme.Tx = *tx
-			// Override error code during testing
-			if pg.testing {
-				gdme.ErrorCode = digestFound
-			} else if !bytes.Equal(ar.Anchor.Merkle, []byte{}) &&
+
+			// Lazyflush record if it was anchored but blockchain timestamp
+			// isn't avialable yet
+			if !bytes.Equal(ar.Anchor.Merkle, []byte{}) &&
 				gdme.AnchoredTimestamp == 0 {
-				// Lazyflush record if it was anchored but blockchain timestamp
-				// isn't avialable yet
 				fr := backend.FlushRecord{
 					Tx:   gdme.Tx,
 					Root: gdme.MerkleRoot,
@@ -303,11 +295,6 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 				Digest:    hash,
 				ErrorCode: backend.ErrorExists,
 			})
-
-			// Override error code during testing
-			if pg.testing {
-				me[len(me)-1].ErrorCode = digestFound
-			}
 			continue
 		}
 		// Insert record
@@ -488,9 +475,6 @@ func (pg *Postgres) doFlush() (int, error) {
 		err = pg.flush(ts)
 		if err != nil {
 			e := fmt.Sprintf("flush %v: %v", ts, err)
-			if pg.testing {
-				panic(e)
-			}
 			log.Error(e)
 		} else {
 			count++
@@ -539,16 +523,14 @@ func (pg *Postgres) flush(ts int64) error {
 		Merkle:         root[:],
 		FlushTimestamp: time.Now().Unix(),
 	}
-	if !pg.testing {
-		tx, err := pg.wallet.Construct(root)
-		if err != nil {
-			// XXX do something with unsufficient funds here.
-			return fmt.Errorf("flush Construct tx: %v", err)
-		}
-		log.Infof("Flush timestamp: %v digests %v merkle: %x tx: %v",
-			ts, len(digests), root, tx.String())
-		a.TxHash = (*tx)[:]
+	tx, err := pg.wallet.Construct(root)
+	if err != nil {
+		// XXX do something with unsufficient funds here.
+		return fmt.Errorf("flush Construct tx: %v", err)
 	}
+	log.Infof("Flush timestamp: %v digests %v merkle: %x tx: %v",
+		ts, len(digests), root, tx.String())
+	a.TxHash = (*tx)[:]
 
 	// Insert anchor data into db
 	err = pg.insertAnchor(a)
diff --git a/dcrtimed/backend/postgres/testpostgres.go b/dcrtimed/backend/postgres/testpostgres.go
index 03d1090..b85245c 100644
--- a/dcrtimed/backend/postgres/testpostgres.go
+++ b/dcrtimed/backend/postgres/testpostgres.go
@@ -3,3 +3,22 @@
 // license that can be found in the LICENSE file.
 
 package postgres
+
+import (
+	"sync"
+	"time"
+)
+
+// TestPostgres provides a implementation of the backend interface that stores
+// records in memory and that can be used for testing.
+type TestPostgres struct {
+	sync.RWMutex
+
+	commit uint // Current version, incremented during flush
+
+	myNow func() time.Time // Override time.Now()
+
+	// in memory data
+	records []Record
+	anchors []Anchor
+}

From 45613db0a364048bd978d22dc438f78e68a20a1a Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 13 Sep 2020 00:58:38 +0300
Subject: [PATCH 60/66] check rows.Err()

---
 dcrtimed/backend/postgres/sql.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 4814d8b..63dc859 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -46,7 +46,7 @@ func (pg *Postgres) getAllRecordsTimestamps() (*[]int64, error) {
 		}
 		tss = append(tss, ts)
 	}
-	return &tss, nil
+	return &tss, rows.Err()
 }
 
 // getLatestAnchoredTimestamp returns latest anchor information - tx hash and
@@ -79,7 +79,7 @@ func (pg *Postgres) getLatestAnchoredTimestamp() (int64, Anchor, error) {
 		a.Merkle = mr
 		a.TxHash = txHash
 	}
-	return serverTs, a, nil
+	return serverTs, a, rows.Err()
 }
 
 // updateAnchorChainTs accepts an anchor and updates it's chain timestamp
@@ -152,7 +152,7 @@ func (pg *Postgres) getDigestsByMerkleRoot(merkle []byte) ([]*[sha256.Size]byte,
 		copy(digest[:], rawDigest[:])
 		digests = append(digests, &digest)
 	}
-	return digests, nil
+	return digests, rows.Err()
 }
 
 // getDigestsByTimestamp accepts a timestamp, selects all digests from

From 6e8ff5d3b247d9863ca2d3b02b57da8208299799 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 13 Sep 2020 01:16:03 +0300
Subject: [PATCH 61/66] satisfy linter

---
 dcrtimed/backend/postgres/postgres.go |  2 +-
 dcrtimed/backend/postgres/sql.go      | 22 ++++++++++++++++------
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 94a3006..5411e05 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -324,7 +324,7 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 	if err != nil {
 		return 0, []backend.PutResult{}, err
 	}
-	err = stmt.Close()
+	defer stmt.Close()
 	if err != nil {
 		return 0, []backend.PutResult{}, err
 	}
diff --git a/dcrtimed/backend/postgres/sql.go b/dcrtimed/backend/postgres/sql.go
index 63dc859..beeb4cd 100644
--- a/dcrtimed/backend/postgres/sql.go
+++ b/dcrtimed/backend/postgres/sql.go
@@ -141,6 +141,8 @@ func (pg *Postgres) getDigestsByMerkleRoot(merkle []byte) ([]*[sha256.Size]byte,
 	if err != nil {
 		return nil, err
 	}
+	defer rows.Close()
+
 	var digests []*[sha256.Size]byte
 	for rows.Next() {
 		var rawDigest []byte
@@ -165,6 +167,8 @@ func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error
 	if err != nil {
 		return nil, err
 	}
+	defer rows.Close()
+
 	var digests []*[sha256.Size]byte
 	for rows.Next() {
 		var rawDigest []byte
@@ -176,7 +180,7 @@ func (pg *Postgres) getDigestsByTimestamp(ts int64) ([]*[sha256.Size]byte, error
 		copy(digest[:], rawDigest[:])
 		digests = append(digests, &digest)
 	}
-	return digests, nil
+	return digests, rows.Err()
 }
 
 // getUnflushedTimestamps accepts current server timestamp and queries records
@@ -190,6 +194,8 @@ func (pg *Postgres) getUnflushedTimestamps(current int64) ([]int64, error) {
 	if err != nil {
 		return nil, err
 	}
+	defer rows.Close()
+
 	var ts int64
 	tss := []int64{}
 	for rows.Next() {
@@ -199,7 +205,7 @@ func (pg *Postgres) getUnflushedTimestamps(current int64) ([]int64, error) {
 		}
 		tss = append(tss, ts)
 	}
-	return tss, nil
+	return tss, rows.Err()
 }
 
 // getRecordsByServerTs accepts a server collection timestamps and returns
@@ -219,6 +225,7 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*AnchoredRecord, err
 		return false, nil, err
 	}
 	defer rows.Close()
+
 	var (
 		mr      []byte
 		digest  []byte
@@ -251,7 +258,7 @@ func (pg *Postgres) getRecordsByServerTs(ts int64) (bool, []*AnchoredRecord, err
 		r = append(r, &ar)
 	}
 
-	return len(r) > 0, r, nil
+	return len(r) > 0, r, rows.Err()
 }
 
 // getRecordByDigest accepts apointer to an AnchoredRecord which initially
@@ -271,6 +278,7 @@ func (pg *Postgres) getRecordByDigest(ar *AnchoredRecord) (bool, error) {
 		return false, err
 	}
 	defer rows.Close()
+
 	var chainTs sql.NullInt64
 	for rows.Next() {
 		err = rows.Scan(&ar.Anchor.Merkle, &ar.Record.CollectionTimestamp, &ar.Anchor.TxHash, &chainTs)
@@ -295,7 +303,7 @@ func (pg *Postgres) getRecordByDigest(ar *AnchoredRecord) (bool, error) {
 		return true, nil
 	}
 
-	return false, nil
+	return false, rows.Err()
 }
 
 // hasTable accepts a table name and checks if it was created
@@ -309,6 +317,7 @@ func (pg *Postgres) hasTable(name string) (bool, error) {
 		return false, err
 	}
 	defer rows.Close()
+
 	var exists bool
 	for rows.Next() {
 		err = rows.Scan(&exists)
@@ -316,7 +325,7 @@ func (pg *Postgres) hasTable(name string) (bool, error) {
 			return false, err
 		}
 	}
-	return exists, nil
+	return exists, rows.Err()
 }
 
 // isDigestExists accept a digest and checks if it's already exists in
@@ -330,6 +339,7 @@ func (pg *Postgres) isDigestExists(hash []byte) (bool, error) {
 		return false, err
 	}
 	defer rows.Close()
+
 	var exists bool
 	for rows.Next() {
 		err = rows.Scan(&exists)
@@ -337,7 +347,7 @@ func (pg *Postgres) isDigestExists(hash []byte) (bool, error) {
 			return false, err
 		}
 	}
-	return exists, nil
+	return exists, rows.Err()
 }
 
 // createAnchorsTable creates anchors table

From 364216841a1b2d594dce207016d3daddf4b05689 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 13 Sep 2020 12:33:26 +0300
Subject: [PATCH 62/66] cleanup

---
 dcrtimed/backend/postgres/postgres.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 5411e05..699b18b 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -324,10 +324,6 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 	if err != nil {
 		return 0, []backend.PutResult{}, err
 	}
-	defer stmt.Close()
-	if err != nil {
-		return 0, []backend.PutResult{}, err
-	}
 	err = txn.Commit()
 	if err != nil {
 		return 0, []backend.PutResult{}, err

From 1aebba6842eeb08c76420ddd3989b87cc6d677e4 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Sun, 13 Sep 2020 12:49:47 +0300
Subject: [PATCH 63/66] opps

---
 dcrtimed/backend/postgres/postgres.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 699b18b..470cd7c 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -283,6 +283,7 @@ func (pg *Postgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult,
 	if err != nil {
 		return 0, []backend.PutResult{}, err
 	}
+	defer stmt.Close()
 
 	for _, hash := range hashes {
 		// Check if digest exists

From 522cf11e18336c005c15f6c9774310d35e379d25 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Mon, 14 Sep 2020 01:14:23 +0300
Subject: [PATCH 64/66] add failing tests - chunk 1

---
 dcrtimed/backend/postgres/postgres.go         |  70 +++---
 dcrtimed/backend/postgres/testpostgres.go     |  24 --
 .../postgres/testpostgres/postgres_test.go    | 211 ++++++++++++++++++
 .../postgres/testpostgres/testpostgres.go     | 168 ++++++++++++++
 4 files changed, 415 insertions(+), 58 deletions(-)
 delete mode 100644 dcrtimed/backend/postgres/testpostgres.go
 create mode 100644 dcrtimed/backend/postgres/testpostgres/postgres_test.go
 create mode 100644 dcrtimed/backend/postgres/testpostgres/testpostgres.go

diff --git a/dcrtimed/backend/postgres/postgres.go b/dcrtimed/backend/postgres/postgres.go
index 470cd7c..0ea41e9 100644
--- a/dcrtimed/backend/postgres/postgres.go
+++ b/dcrtimed/backend/postgres/postgres.go
@@ -210,45 +210,47 @@ func (pg *Postgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult
 			if err != nil {
 				return nil, err
 			}
-			gtme.ErrorCode = backend.ErrorOK
 			if !exists {
 				gtme.ErrorCode = backend.ErrorNotFound
-			}
-			// copy ts digests
-			gtme.Digests = make([][sha256.Size]byte, 0, len(records))
-			for _, r := range records {
-				var d [sha256.Size]byte
-				copy(d[:], r.Record.Digest[:])
-				gtme.Digests = append(gtme.Digests, d)
-				tx, err := chainhash.NewHash(r.Anchor.TxHash)
-				if err != nil {
-					return nil, err
-				}
-				gtme.Tx = *tx
-				gtme.AnchoredTimestamp = r.Anchor.ChainTimestamp
-				copy(gtme.MerkleRoot[:], r.Anchor.Merkle[:sha256.Size])
-			}
-			// Lazyflush record if it was anchored but blockchain timestamp
-			// isn't avialable yet
-			if gtme.MerkleRoot != [sha256.Size]byte{} && gtme.AnchoredTimestamp == 0 {
-				fr := backend.FlushRecord{
-					Tx:   gtme.Tx,
-					Root: gtme.MerkleRoot,
-				}
-				_, err = pg.lazyFlush(&fr)
-				if err != nil {
-					switch err {
-					case errNotEnoughConfirmation:
-						// All good, continue without blockchain timestamp
-					case errInvalidConfirmations:
-						log.Errorf("%v: Confirmations = -1",
-							gtme.Tx.String())
-						return nil, err
-					default:
+			} else {
+				gtme.ErrorCode = backend.ErrorOK
+				// copy ts digests
+				gtme.Digests = make([][sha256.Size]byte, 0, len(records))
+				for _, r := range records {
+					var d [sha256.Size]byte
+					copy(d[:], r.Record.Digest[:])
+					gtme.Digests = append(gtme.Digests, d)
+					tx, err := chainhash.NewHash(r.Anchor.TxHash)
+					if err != nil {
 						return nil, err
 					}
+					gtme.Tx = *tx
+					gtme.AnchoredTimestamp = r.Anchor.ChainTimestamp
+					copy(gtme.MerkleRoot[:], r.Anchor.Merkle[:sha256.Size])
+				}
+				// Lazyflush record if it was anchored but blockchain timestamp
+				// isn't avialable yet
+				if gtme.MerkleRoot != [sha256.Size]byte{} &&
+					gtme.AnchoredTimestamp == 0 {
+					fr := backend.FlushRecord{
+						Tx:   gtme.Tx,
+						Root: gtme.MerkleRoot,
+					}
+					_, err = pg.lazyFlush(&fr)
+					if err != nil {
+						switch err {
+						case errNotEnoughConfirmation:
+							// All good, continue without blockchain timestamp
+						case errInvalidConfirmations:
+							log.Errorf("%v: Confirmations = -1",
+								gtme.Tx.String())
+							return nil, err
+						default:
+							return nil, err
+						}
+					}
+					gtme.AnchoredTimestamp = fr.ChainTimestamp
 				}
-				gtme.AnchoredTimestamp = fr.ChainTimestamp
 			}
 		} else {
 			gtme.ErrorCode = backend.ErrorNotAllowed
diff --git a/dcrtimed/backend/postgres/testpostgres.go b/dcrtimed/backend/postgres/testpostgres.go
deleted file mode 100644
index b85245c..0000000
--- a/dcrtimed/backend/postgres/testpostgres.go
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) 2020 The Decred developers
-// Use of this source code is governed by an ISC
-// license that can be found in the LICENSE file.
-
-package postgres
-
-import (
-	"sync"
-	"time"
-)
-
-// TestPostgres provides a implementation of the backend interface that stores
-// records in memory and that can be used for testing.
-type TestPostgres struct {
-	sync.RWMutex
-
-	commit uint // Current version, incremented during flush
-
-	myNow func() time.Time // Override time.Now()
-
-	// in memory data
-	records []Record
-	anchors []Anchor
-}
diff --git a/dcrtimed/backend/postgres/testpostgres/postgres_test.go b/dcrtimed/backend/postgres/testpostgres/postgres_test.go
new file mode 100644
index 0000000..6a91161
--- /dev/null
+++ b/dcrtimed/backend/postgres/testpostgres/postgres_test.go
@@ -0,0 +1,211 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package testpostgres
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"testing"
+	"time"
+
+	"github.com/decred/dcrtime/dcrtimed/backend"
+)
+
+func TestGetTimestamp(t *testing.T) {
+	tp := New()
+
+	// we want to verify collections as well
+	tp.enableCollections = true
+
+	// Put batch success in current time
+	var hashes [][sha256.Size]byte
+	count := 10
+	for i := 0; i < count; i++ {
+		hash := [sha256.Size]byte{}
+		hash[0] = byte(i)
+		hashes = append(hashes, hash)
+	}
+	timestamp, me, err := tp.Put(hashes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(me) != count {
+		t.Fatalf("expected %v multi error", count)
+	}
+
+	// Get invalid timestamp+1, timestamp+2, timestamp+3
+	gtmes, err := tp.GetTimestamps([]int64{timestamp + 1, timestamp + 2,
+		timestamp + 3})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gtmes) != 3 {
+		t.Fatalf("expected 3 gtmes got %v", len(gtmes))
+	}
+	for _, gtme := range gtmes {
+		if gtme.ErrorCode != backend.ErrorNotFound {
+			t.Fatalf("expected ErrorNotFound got %v",
+				gtme.ErrorCode)
+		}
+	}
+
+	// Get invalid timestamp+1, timestamp+2, timestamp+3 and valid timestamp
+	gtmes, err = tp.GetTimestamps([]int64{timestamp + 1, timestamp + 2,
+		timestamp + 3, timestamp})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gtmes) != 4 {
+		t.Fatalf("expected 4 gtmes got %v", len(gtmes))
+	}
+	for i, gtme := range gtmes {
+		if i < len(gtmes)-1 && gtme.ErrorCode != backend.ErrorNotFound {
+			t.Fatalf("expected ErrorNotFound got %v",
+				gtme.ErrorCode)
+		}
+		if i == len(gtmes)-1 && gtme.ErrorCode != backend.ErrorOK {
+			t.Fatalf("expected ErrorOK got %v", gtme.ErrorCode)
+		}
+	}
+
+	// Get with timestamp
+	gtmes, err = tp.GetTimestamps([]int64{timestamp})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gtmes) != 1 {
+		t.Fatalf("expected 1 gtmes got %v", len(gtmes))
+	}
+	gtme := gtmes[0]
+	// Verify we got all the bits back.
+	if len(gtme.Digests) != count {
+		t.Fatalf("expected %v digests got %v", count, len(gtme.Digests))
+	}
+	exists := make(map[byte]struct{})
+	for _, digest := range gtme.Digests {
+		if _, ok := exists[digest[0]]; ok {
+			t.Fatalf("dup %v", digest[0])
+		}
+		exists[digest[0]] = struct{}{}
+	}
+	if len(exists) != count {
+		t.Fatalf("expected %v exists got %v", count, len(exists))
+	}
+}
+
+func TestGetDigests(t *testing.T) {
+	tp := New()
+
+	timestamp := tp.now().Unix()
+	tp.myNow = func() time.Time {
+		return time.Unix(timestamp, 0)
+	}
+
+	// Put batch success in current time
+	var hashes [][sha256.Size]byte
+	count := 10
+	for i := 0; i < count; i++ {
+		hash := [sha256.Size]byte{}
+		hash[0] = byte(i)
+		hashes = append(hashes, hash)
+	}
+
+	_, me, err := tp.Put(hashes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(me) != count {
+		t.Fatalf("expected %v multi error", count)
+	}
+
+	grs, err := tp.Get(hashes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(grs) != count {
+		t.Fatalf("expected %v GetResult, got %v", count, len(grs))
+	}
+
+	for i, gr := range grs {
+		if !bytes.Equal(gr.Digest[:], hashes[i][:]) {
+			t.Fatalf("invalid digest got %x want %x",
+				gr.Digest[:], hashes[i][:])
+		}
+	}
+
+	// Get mixed success and failure
+	for i := count; i < count*2; i++ {
+		hash := [sha256.Size]byte{}
+		hash[0] = byte(i)
+		hashes = append(hashes, hash)
+	}
+
+	grs, err = tp.Get(hashes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(grs) != count*2 {
+		t.Fatalf("expected %v GetResult", count*2)
+	}
+
+	for i, gr := range grs {
+		if i < count-1 && (!bytes.Equal(gr.Digest[:], hashes[i][:]) ||
+			gr.ErrorCode != backend.ErrorOK) {
+			t.Fatalf("invalid digest got %x want %x ErrorCode "+
+				"got %v want %v", gr.Digest[:], hashes[i][:],
+				gr.ErrorCode, backend.ErrorOK)
+		}
+		if i >= count && gr.ErrorCode != backend.ErrorNotFound {
+			t.Fatalf("invalid ErrorCode got %x want %x",
+				gr.ErrorCode, backend.ErrorNotFound)
+		}
+	}
+}
+
+func TestPut(t *testing.T) {
+	tp := New()
+
+	// Put batch success in current time
+	var hashes [][sha256.Size]byte
+	count := 10
+	for i := 0; i < count; i++ {
+		hash := [sha256.Size]byte{}
+		hash[0] = byte(i)
+		hashes = append(hashes, hash)
+	}
+
+	_, me, err := tp.Put(hashes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(me) != count {
+		t.Fatalf("expected %v multi error", count)
+	}
+
+	// Verify all return codes
+	for _, m := range me {
+		if m.ErrorCode != backend.ErrorOK {
+			t.Fatalf("expected ErrorCode %v got %v",
+				backend.ErrorOK, m.ErrorCode)
+		}
+	}
+
+	// Try again, now we expect count ErrorExists.
+	_, me, err = tp.Put(hashes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(me) != count {
+		t.Fatalf("expected %v multi error", count)
+	}
+
+	// Verify all return codes
+	for _, m := range me {
+		if m.ErrorCode != backend.ErrorExists {
+			t.Fatalf("expected ErrorCode %v got %v",
+				backend.ErrorExists, m.ErrorCode)
+		}
+	}
+}
diff --git a/dcrtimed/backend/postgres/testpostgres/testpostgres.go b/dcrtimed/backend/postgres/testpostgres/testpostgres.go
new file mode 100644
index 0000000..58b818a
--- /dev/null
+++ b/dcrtimed/backend/postgres/testpostgres/testpostgres.go
@@ -0,0 +1,168 @@
+// Copyright (c) 2020 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package testpostgres
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/decred/dcrtime/dcrtimed/backend"
+	"github.com/decred/dcrtime/dcrtimed/backend/postgres"
+)
+
+var duration = time.Minute // Default how often we combine digests
+
+// TestPostgres provides a implementation of the backend interface that stores
+// records in memory and that can be used for testing.
+type TestPostgres struct {
+	sync.RWMutex
+
+	myNow    func() time.Time // Override time.Now()
+	duration time.Duration    // How often we combine digests
+
+	enableCollections bool // Set to true to enable collection query
+
+	// in memory data
+	records map[string]postgres.Record //[hash]Record
+	anchors map[string]postgres.Anchor //[merkle]Anchor
+}
+
+func (tp *TestPostgres) getRecordsByServerTs(ts int64) (bool, []postgres.Record) {
+	rs := []postgres.Record{}
+
+	for _, r := range tp.records {
+		if r.CollectionTimestamp == ts {
+			rs = append(rs, r)
+		}
+	}
+	return len(rs) > 0, rs
+}
+
+func (tp *TestPostgres) getRecordByDigest(hash []byte) (postgres.Record, bool) {
+	r, exists := tp.records[hex.EncodeToString(hash)]
+	return r, exists
+}
+
+// GetTimestamps retrieves the digests for a given timestamp.
+func (tp *TestPostgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult, error) {
+	gtmes := make([]backend.TimestampResult, 0, len(timestamps))
+
+	tp.RLock()
+	defer tp.RUnlock()
+
+	for _, ts := range timestamps {
+		gtme := backend.TimestampResult{
+			Timestamp: ts,
+		}
+		if tp.enableCollections {
+			exists, records := tp.getRecordsByServerTs(ts)
+			if !exists {
+				gtme.ErrorCode = backend.ErrorNotFound
+			} else {
+				gtme.ErrorCode = backend.ErrorOK
+				// copy ts digests
+				gtme.Digests = make([][sha256.Size]byte, 0, len(records))
+				for _, r := range records {
+					var d [sha256.Size]byte
+					copy(d[:], r.Digest[:])
+					gtme.Digests = append(gtme.Digests, d)
+				}
+			}
+		} else {
+			gtme.ErrorCode = backend.ErrorNotAllowed
+		}
+		gtmes = append(gtmes, gtme)
+	}
+	return gtmes, nil
+}
+
+// Get returns timestamp information for given digests.
+func (tp *TestPostgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, error) {
+	gdmes := make([]backend.GetResult, 0, len(digests))
+
+	tp.RLock()
+	defer tp.RUnlock()
+
+	for _, digest := range digests {
+		gdme := backend.GetResult{
+			Digest: digest,
+		}
+		r, exists := tp.getRecordByDigest(digest[:])
+
+		if !exists {
+			gdme.ErrorCode = backend.ErrorNotFound
+		} else {
+			gdme.ErrorCode = backend.ErrorOK
+			gdme.Timestamp = r.CollectionTimestamp
+		}
+		gdmes = append(gdmes, gdme)
+	}
+
+	return gdmes, nil
+}
+
+// Put stores hashes and returns timestamp and associated errors.
+func (tp *TestPostgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutResult, error) {
+	// Get current time rounded down.
+	ts := tp.now().Unix()
+	// Prep return
+	me := make([]backend.PutResult, 0, len(hashes))
+
+	tp.Lock()
+	defer tp.Unlock()
+
+	for _, hash := range hashes {
+		// Check if digest exists
+		_, exists := tp.getRecordByDigest(hash[:])
+		if exists {
+			me = append(me, backend.PutResult{
+				Digest:    hash,
+				ErrorCode: backend.ErrorExists,
+			})
+			continue
+		}
+		// Add record to map
+		r := postgres.Record{
+			CollectionTimestamp: ts,
+			Digest:              hash[:],
+		}
+		tp.records[hex.EncodeToString(hash[:])] = r
+		fmt.Println(r)
+		// Mark as successful
+		me = append(me, backend.PutResult{
+			Digest:    hash,
+			ErrorCode: backend.ErrorOK,
+		})
+	}
+	fmt.Println("recordssss", tp.records)
+	return ts, me, nil
+}
+
+// now returns current time stamp rounded down to 1 hour.  All timestamps are
+// UTC.
+func (tp *TestPostgres) now() time.Time {
+	return tp.truncate(tp.myNow().UTC(), tp.duration)
+}
+
+// truncate rounds time down to the provided duration.
+func (tp *TestPostgres) truncate(t time.Time, d time.Duration) time.Time {
+	return t.Truncate(d)
+}
+
+// Close is a stub to satisfy the backend interface.
+func (tp *TestPostgres) Close() {}
+
+// New returns a new testcache context.
+func New() *TestPostgres {
+	return &TestPostgres{
+		records:  make(map[string]postgres.Record),
+		anchors:  make(map[string]postgres.Anchor),
+		duration: duration,
+		myNow:    time.Now,
+	}
+}

From ca27cd166b3d0eb85a6b97f403b9bf014c438fc7 Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Mon, 14 Sep 2020 13:04:09 +0300
Subject: [PATCH 65/66] fix duplicated digest in tests

---
 .../backend/postgres/testpostgres/testpostgres.go | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/dcrtimed/backend/postgres/testpostgres/testpostgres.go b/dcrtimed/backend/postgres/testpostgres/testpostgres.go
index 58b818a..f69a1f8 100644
--- a/dcrtimed/backend/postgres/testpostgres/testpostgres.go
+++ b/dcrtimed/backend/postgres/testpostgres/testpostgres.go
@@ -7,7 +7,6 @@ package testpostgres
 import (
 	"crypto/sha256"
 	"encoding/hex"
-	"fmt"
 	"sync"
 	"time"
 
@@ -117,6 +116,7 @@ func (tp *TestPostgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutRes
 	defer tp.Unlock()
 
 	for _, hash := range hashes {
+		d := hex.EncodeToString(hash[:])
 		// Check if digest exists
 		_, exists := tp.getRecordByDigest(hash[:])
 		if exists {
@@ -126,20 +126,23 @@ func (tp *TestPostgres) Put(hashes [][sha256.Size]byte) (int64, []backend.PutRes
 			})
 			continue
 		}
+
+		dslice, err := hex.DecodeString(d)
+		if err != nil {
+			return 0, nil, err
+		}
 		// Add record to map
-		r := postgres.Record{
+		tp.records[d] = postgres.Record{
 			CollectionTimestamp: ts,
-			Digest:              hash[:],
+			Digest:              dslice,
 		}
-		tp.records[hex.EncodeToString(hash[:])] = r
-		fmt.Println(r)
+
 		// Mark as successful
 		me = append(me, backend.PutResult{
 			Digest:    hash,
 			ErrorCode: backend.ErrorOK,
 		})
 	}
-	fmt.Println("recordssss", tp.records)
 	return ts, me, nil
 }
 

From d3b8aaf26262b6e4bcaabe1aa7d8ab590dd6840f Mon Sep 17 00:00:00 2001
From: amass <amassarwi@gmail.com>
Date: Mon, 14 Sep 2020 15:58:40 +0300
Subject: [PATCH 66/66] finish tests

---
 .../backend/filesystem/filesystem_test.go     |  2 +-
 .../postgres/testpostgres/postgres_test.go    | 68 ++++++++++++++++
 .../postgres/testpostgres/testpostgres.go     | 79 +++++++++++++++++++
 3 files changed, 148 insertions(+), 1 deletion(-)

diff --git a/dcrtimed/backend/filesystem/filesystem_test.go b/dcrtimed/backend/filesystem/filesystem_test.go
index 46de9e2..697293e 100644
--- a/dcrtimed/backend/filesystem/filesystem_test.go
+++ b/dcrtimed/backend/filesystem/filesystem_test.go
@@ -178,7 +178,7 @@ func TestGetDigests(t *testing.T) {
 			gr.ErrorCode != foundGlobal) {
 			t.Fatalf("invalid digest got %x want %x ErrorCode "+
 				"got %v want %v", gr.Digest[:], hashes[i][:],
-				gr.ErrorCode, foundLocal)
+				gr.ErrorCode, foundGlobal)
 		}
 		if i >= count && gr.ErrorCode != backend.ErrorNotFound {
 			t.Fatalf("invalid ErrorCode got %x want %x",
diff --git a/dcrtimed/backend/postgres/testpostgres/postgres_test.go b/dcrtimed/backend/postgres/testpostgres/postgres_test.go
index 6a91161..e3db0d8 100644
--- a/dcrtimed/backend/postgres/testpostgres/postgres_test.go
+++ b/dcrtimed/backend/postgres/testpostgres/postgres_test.go
@@ -93,6 +93,45 @@ func TestGetTimestamp(t *testing.T) {
 	if len(exists) != count {
 		t.Fatalf("expected %v exists got %v", count, len(exists))
 	}
+
+	// Move time forward and flush
+	tp.myNow = func() time.Time {
+		return time.Unix(timestamp, 0).Add(tp.duration)
+	}
+
+	// Flush current container to global database.
+	err = tp.flush(timestamp)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Get timestamp again despite not being current
+	gtmes, err = tp.GetTimestamps([]int64{timestamp})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(gtmes) != 1 {
+		t.Fatalf("expected 1 gtmes got %v", len(gtmes))
+	}
+	gtme = gtmes[0]
+
+	// Verify we got all the bits back.
+	if len(gtme.Digests) != count {
+		t.Fatalf("expected %v digests got %v", count, len(gtme.Digests))
+	}
+	if bytes.Equal(gtme.MerkleRoot[:], []byte{}) {
+		t.Fatalf("expected non empty merkle root got %x", gtme.MerkleRoot)
+	}
+	exists = make(map[byte]struct{})
+	for _, digest := range gtme.Digests {
+		if _, ok := exists[digest[0]]; ok {
+			t.Fatalf("dup %v", digest[0])
+		}
+		exists[digest[0]] = struct{}{}
+	}
+	if len(exists) != count {
+		t.Fatalf("expected %v exists got %v", count, len(exists))
+	}
 }
 
 func TestGetDigests(t *testing.T) {
@@ -162,6 +201,35 @@ func TestGetDigests(t *testing.T) {
 				gr.ErrorCode, backend.ErrorNotFound)
 		}
 	}
+
+	// Flush and repeat mixed success and failure
+
+	// Flush current container
+	err = tp.flush(timestamp)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	grs, err = tp.Get(hashes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(grs) != count*2 {
+		t.Fatalf("expected %v GetResult", count*2)
+	}
+
+	// Validate returned merkle root
+	for i, gr := range grs {
+		if i < count-1 && (!bytes.Equal(gr.Digest[:], hashes[i][:]) ||
+			bytes.Equal(gr.MerkleRoot[:], []byte{})) {
+			t.Fatalf("invalid digest got %x want %x Merkle %x", gr.Digest[:],
+				hashes[i][:], gr.MerkleRoot[:])
+		}
+		if i >= count && gr.ErrorCode != backend.ErrorNotFound {
+			t.Fatalf("invalid ErrorCode got %x want %x",
+				gr.ErrorCode, backend.ErrorNotFound)
+		}
+	}
 }
 
 func TestPut(t *testing.T) {
diff --git a/dcrtimed/backend/postgres/testpostgres/testpostgres.go b/dcrtimed/backend/postgres/testpostgres/testpostgres.go
index f69a1f8..9e4105b 100644
--- a/dcrtimed/backend/postgres/testpostgres/testpostgres.go
+++ b/dcrtimed/backend/postgres/testpostgres/testpostgres.go
@@ -7,11 +7,13 @@ package testpostgres
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"os"
 	"sync"
 	"time"
 
 	"github.com/decred/dcrtime/dcrtimed/backend"
 	"github.com/decred/dcrtime/dcrtimed/backend/postgres"
+	"github.com/decred/dcrtime/merkle"
 )
 
 var duration = time.Minute // Default how often we combine digests
@@ -47,6 +49,56 @@ func (tp *TestPostgres) getRecordByDigest(hash []byte) (postgres.Record, bool) {
 	return r, exists
 }
 
+func (tp *TestPostgres) insertAnchor(a postgres.Anchor) {
+	tp.anchors[hex.EncodeToString(a.Merkle)] = a
+}
+
+func (tp *TestPostgres) updateRecordsAnchor(ts int64, merkle []byte) {
+	for _, r := range tp.records {
+		if r.CollectionTimestamp == ts {
+			r.AnchorMerkle = merkle
+		}
+	}
+}
+
+// flush flushes all records associated with given timestamp.
+// returns nil iff ts records flushed successfully
+//
+// This function must be called with the WRITE lock held
+func (tp *TestPostgres) flush(ts int64) error {
+	// Get timestamp's digests
+	_, rs := tp.getRecordsByServerTs(ts)
+
+	if len(rs) == 0 {
+		// This really should not happen
+	}
+
+	// Convert bytes slices to sha256 arrays
+	var digests []*[sha256.Size]byte
+	for _, r := range rs {
+		var digest [sha256.Size]byte
+		copy(digest[:], r.Digest[:])
+		digests = append(digests, &digest)
+	}
+
+	// Generate merkle
+	mt := merkle.Tree(digests)
+	// Last element is root
+	root := *mt[len(mt)-1]
+	a := postgres.Anchor{
+		Merkle:         root[:],
+		FlushTimestamp: time.Now().Unix(),
+	}
+
+	// Insert anchor data into db
+	tp.insertAnchor(a)
+
+	// Update timestamp's records merkle root
+	tp.updateRecordsAnchor(ts, a.Merkle)
+
+	return nil
+}
+
 // GetTimestamps retrieves the digests for a given timestamp.
 func (tp *TestPostgres) GetTimestamps(timestamps []int64) ([]backend.TimestampResult, error) {
 	gtmes := make([]backend.TimestampResult, 0, len(timestamps))
@@ -70,6 +122,7 @@ func (tp *TestPostgres) GetTimestamps(timestamps []int64) ([]backend.TimestampRe
 					var d [sha256.Size]byte
 					copy(d[:], r.Digest[:])
 					gtme.Digests = append(gtme.Digests, d)
+					copy(gtme.MerkleRoot[:], r.AnchorMerkle[:])
 				}
 			}
 		} else {
@@ -98,6 +151,7 @@ func (tp *TestPostgres) Get(digests [][sha256.Size]byte) ([]backend.GetResult, e
 		} else {
 			gdme.ErrorCode = backend.ErrorOK
 			gdme.Timestamp = r.CollectionTimestamp
+			copy(gdme.MerkleRoot[:], r.AnchorMerkle[:])
 		}
 		gdmes = append(gdmes, gdme)
 	}
@@ -160,6 +214,31 @@ func (tp *TestPostgres) truncate(t time.Time, d time.Duration) time.Time {
 // Close is a stub to satisfy the backend interface.
 func (tp *TestPostgres) Close() {}
 
+// Dump is a stub to satisfy the backend interface.
+func (tp *TestPostgres) Dump(f *os.File, verbose bool) error {
+	return nil
+}
+
+// Restore is a stub to satisfy the backend interface.
+func Restore(f *os.File, verbose bool, location string) error {
+	return nil
+}
+
+// Fsck is a stub to satisfy the backend interface.
+func Fsck(options *backend.FsckOptions) error {
+	return nil
+}
+
+// GetBalance is a stub to satisfy the backend interface.
+func GetBalance() (*backend.GetBalanceResult, error) {
+	return nil, nil
+}
+
+// LastAnchor is a stub to satisfy the backend interface.
+func LastAnchor() (*backend.LastAnchorResult, error) {
+	return nil, nil
+}
+
 // New returns a new testcache context.
 func New() *TestPostgres {
 	return &TestPostgres{