Implement SQLite

etc/settings.json

@@ -8,6 +8,7 @@
   "LocalRoot": "uploadTest",
   "DriveRoot": "root",
   "DbPath": "queue.db",
+  "SQLitePath": ":memory:",
   "LocalTime": "America/Los_Angeles",
   "UploadMaxWorkers": 3
 }

main.go

@@ -1,12 +1,42 @@
 package main
 
 import (
+	"flag"
 	"log"
 	"net/http"
 	"sync"
 )
 
 func main() {
+
+	flag.Parse()
+
+	switch mode := flag.Arg(0) ; mode {
+	case "daemonV1":
+		daemonV1()
+	case "migrateToV2":
+		migrateToV2()
+	case "scanGDrive":
+		scanGDrive()
+	default:
+		flag.PrintDefaults()
+	}
+
+}
+
+func scanGDrive() {
+	loadSettings()
+	getLocTime()
+	setupSQL()
+	getDrive()
+
+}
+
+func migrateToV2() {
+
+}
+
+func daemonV1() {
 	loadSettings()
 	getLocTime()
 	setupBolt()
@@ -31,5 +61,4 @@ func main() {
 	}()
 
 	wg.Wait()
-
 }

migration/20190219200159_initial_tables.go

@@ -0,0 +1,79 @@
+package migration
+
+import (
+	"database/sql"
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20190219200159, Down20190219200159)
+}
+
+func Up20190219200159(tx *sql.Tx) error {
+	// This code is executed when the migration is applied.
+	_, err := tx.Exec(`
+CREATE TABLE localFile (
+  id INTEGER PRIMARY KEY,
+  parentID INTEGER,
+  fileType TEXT NOT NULL,
+  name TEXT NOT NULL,
+  md5 TEXT NOT NULL,
+  gDriveFileID TEXT,
+  uploadStatus TEXT NOT NULL DEFAULT "unknown",
+  updatedAt TEXT NOT NULL,
+  deleted INTEGER NOT NULL DEFAULT 0
+);
+`)
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(`
+CREATE TABLE localFileMetric (
+  localFileID INTEGER NOT NULL,
+  foundAt TEXT,
+  queuedAt TEXT,
+  uploadStartedAt TEXT,
+  uploadEndedAt TEXT,
+  uploadRetries INTEGER NOT NULL DEFAULT 0
+);
+`)
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(`
+CREATE TABLE gDriveFile (
+  id TEXT PRIMARY KEY,
+  parentID TEXT,
+  fileType TEXT NOT NULL,
+  name TEXT NOT NULL,
+  md5 TEXT NOT NULL,
+  localFileID INTEGER,
+  updatedAt TEXT NOT NULL,
+  deleted INTEGER NOT NULL DEFAULT 0
+);
+`)
+	if err != nil {
+		return err
+	}
+	_, err = tx.Exec(`
+CREATE TABLE gDriveFileMetric (
+  gDriveFileID TEXT,
+  foundAt TEXT
+);
+`)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func Down20190219200159(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	tx.Exec(`
+DROP TABLE localFile;
+DROP TABLE localFileMetric;
+DROP TABLE gDriveFile;
+DROP TABLE gDriveFileMetric;
+`)
+	return nil
+}

paths.go

@@ -112,16 +112,4 @@ func makeSortKey(path *Path) string {
 	return path.ModTime.Format("20060102")
 }
 
-var locTime *time.Location
 
-func getLocTime() *time.Location {
-	if locTime != nil {
-		return locTime
-	}
-	loc, err := time.LoadLocation(settings.LocalTime)
-	if err != nil {
-		log.Fatalf("Unable to local time location: %v", err)
-	}
-	locTime = loc
-	return locTime
-}

schema.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"github.com/jmoiron/sqlx"
+	_ "github.com/mattn/go-sqlite3"
+	_ "github.com/pjankovsky/gdriver-go/migration"
+	"github.com/pressly/goose"
+	"log"
+)
+
+func setupSQL() {
+	db := getDB()
+
+	err :=goose.SetDialect("sqlite3")
+	if err != nil {
+		log.Fatalf("Unable to set goose dialect: %v", err)
+	}
+
+	err = goose.Up(db.DB, "migration")
+	if err != nil {
+		log.Fatalf("Unable to run goose migrations: %v", err)
+	}
+}
+
+var cachedDB *sqlx.DB
+
+func getDB() *sqlx.DB {
+	if cachedDB == nil {
+		db, err := sqlx.Connect("sqlite3", settings.SQLitePath)
+		if err != nil {
+			log.Fatalf("Unable to connect to sqlite3 db: %v", err)
+		}
+		cachedDB = db
+	}
+	return cachedDB
+}

schema/common.go

@@ -0,0 +1,32 @@
+package schema
+
+// LocalFileID : DB autoinc ID
+type LocalFileID uint64
+
+// GDriveFileID : string ID from Google Drive
+type GDriveFileID string
+
+// MD5 : byte slice for MD5 hash
+type MD5 [32]byte
+
+// FileType : designate if the file is a Directory or File
+type FileType string
+
+// enum options for FileType
+const (
+	FileTypeFile FileType = "F"
+	FileTypeDir  FileType = "D"
+)
+
+// UploadStatus : the status of the upload
+type UploadStatus string
+
+// enum options for UploadStatus
+const (
+	UploadStatusUnknown    UploadStatus = "unknown"
+	UploadStatusError      UploadStatus = "error"
+	UploadStatusReady      UploadStatus = "ready"
+	UploadStatusPending    UploadStatus = "pending"
+	UploadStatusInProgress UploadStatus = "inprogress"
+	UploadStatusDone       UploadStatus = "done"
+)

schema/gdrive.go

@@ -0,0 +1,21 @@
+package schema
+
+import "time"
+
+// GDriveFileRow is for db table gDriveFile
+type GDriveFileRow struct {
+	id          GDriveFileID
+	parentID    GDriveFileID
+	fileType    FileType
+	name        string
+	md5         MD5
+	localFileID LocalFileID
+	updatedAt   time.Time
+	deleted     bool
+}
+
+// GDriveFileMetricRow is for db table gDriveFileMetric
+type GDriveFileMetricRow struct {
+	gDriveFileID GDriveFileID
+	foundAt      time.Time
+}

schema/local.go

@@ -0,0 +1,26 @@
+package schema
+
+import "time"
+
+// LocalFileRow is for db table localFile
+type LocalFileRow struct {
+	id           LocalFileID
+	parentID     LocalFileID
+	fileType     FileType
+	name         string
+	md5          MD5
+	gDriveFileID GDriveFileID
+	uploadStatus UploadStatus
+	updatedAt    time.Time
+	deleted      bool
+}
+
+// LocalFileMetricRow is for db table localFileMetric
+type LocalFileMetricRow struct {
+	localFileID     LocalFileID
+	foundAt         time.Time
+	queuedAt        time.Time
+	uploadStartedAt time.Time
+	uploadEndedAt   time.Time
+	uploadRetries   int
+}

settings.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"os"
 	"path"
+	"time"
 )
 
 type Settings struct {
@@ -19,6 +20,7 @@ type Settings struct {
 	DbPath           string
 	LocalTime        string
 	UploadMaxWorkers int
+	SQLitePath       string
 }
 
 var settings Settings
@@ -48,6 +50,21 @@ func loadSettings() {
 	log.Printf(" -- LocalRoot:         %v", settings.LocalRoot)
 	log.Printf(" -- DriveRoot:         %v", settings.DriveRoot)
 	log.Printf(" -- DbPath:            %v", settings.DbPath)
+	log.Printf(" -- SQLitePath:        %v", settings.SQLitePath)
 	log.Printf(" -- LocalTime:         %v", settings.LocalTime)
 	log.Printf(" -- UploadMaxWorkers:  %v", settings.UploadMaxWorkers)
 }
+
+var locTime *time.Location
+
+func getLocTime() *time.Location {
+	if locTime != nil {
+		return locTime
+	}
+	loc, err := time.LoadLocation(settings.LocalTime)
+	if err != nil {
+		log.Fatalf("Unable to local time location: %v", err)
+	}
+	locTime = loc
+	return locTime
+}