From 83a4f89a93c46459f219890b72c6b35d9b6632d7 Mon Sep 17 00:00:00 2001 From: Peter Jankovsky Date: Mon, 18 Feb 2019 20:12:45 -0800 Subject: [PATCH 1/5] basic structs --- schema/common.go | 25 +++++++++++++++++++++++++ schema/gdrive.go | 19 +++++++++++++++++++ schema/local.go | 24 ++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 schema/common.go create mode 100644 schema/gdrive.go create mode 100644 schema/local.go diff --git a/schema/common.go b/schema/common.go new file mode 100644 index 0000000..77034d5 --- /dev/null +++ b/schema/common.go @@ -0,0 +1,25 @@ +package schema + +type LocalFileId int + +type GDriveFileId string + +type MD5 string + +type FileType string + +const ( + FileTypeFile FileType = "F" + FileTypeDir FileType = "D" +) + +type Status string + +const ( + StatusUnknown Status = "unknown" + StatusError Status = "error" + StatusReady Status = "ready" + StatusPending Status = "pending" + StatusInProgress Status = "inprogress" + StatusDone Status = "done" +) diff --git a/schema/gdrive.go b/schema/gdrive.go new file mode 100644 index 0000000..8c5694c --- /dev/null +++ b/schema/gdrive.go @@ -0,0 +1,19 @@ +package schema + +import "time" + +type GDriveFile struct { + id GDriveFileId + parentId GDriveFileId + fileType FileType + name string + md5 MD5 + localFileId LocalFileId + updatedAt time.Time + deleted bool +} + +type GDriveFileMetric struct { + gDriveFileId GDriveFileId + foundAt time.Time +} diff --git a/schema/local.go b/schema/local.go new file mode 100644 index 0000000..04d7506 --- /dev/null +++ b/schema/local.go @@ -0,0 +1,24 @@ +package schema + +import "time" + +type LocalFile struct { + id LocalFileId + parentId LocalFileId + fileType FileType + name string + md5 MD5 + gDriveFileId GDriveFileId + uploadStatus Status + updatedAt time.Time + deleted bool +} + +type LocalFileMetric struct { + localFileId LocalFileId + foundAt time.Time + queuedAt time.Time + uploadStartedAt time.Time + uploadEndedAt time.Time + uploadRetries int +} From a8dbfdb5633e63f396b4b53b4778da402e28deff Mon Sep 17 00:00:00 2001 From: Peter Jankovsky Date: Mon, 18 Feb 2019 20:52:19 -0800 Subject: [PATCH 2/5] cleanup and comments --- schema/common.go | 27 +++++++++++++++++---------- schema/gdrive.go | 10 ++++++---- schema/local.go | 12 +++++++----- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/schema/common.go b/schema/common.go index 77034d5..95d66c9 100644 --- a/schema/common.go +++ b/schema/common.go @@ -1,25 +1,32 @@ package schema -type LocalFileId int +// LocalFileID : DB autoinc ID +type LocalFileID uint64 -type GDriveFileId string +// GDriveFileID : string ID from Google Drive +type GDriveFileID string -type MD5 string +// MD5 : byte slice for MD5 hash +type MD5 [32]byte +// FileType : designate if the file is a Directory or File type FileType string +// enum options for FileType const ( FileTypeFile FileType = "F" FileTypeDir FileType = "D" ) -type Status string +// UploadStatus : the status of the upload +type UploadStatus string +// enum options for UploadStatus const ( - StatusUnknown Status = "unknown" - StatusError Status = "error" - StatusReady Status = "ready" - StatusPending Status = "pending" - StatusInProgress Status = "inprogress" - StatusDone Status = "done" + UploadStatusUnknown UploadStatus = "unknown" + UploadStatusError UploadStatus = "error" + UploadStatusReady UploadStatus = "ready" + UploadStatusPending UploadStatus = "pending" + UploadStatusInProgress UploadStatus = "inprogress" + UploadStatusDone UploadStatus = "done" ) diff --git a/schema/gdrive.go b/schema/gdrive.go index 8c5694c..39947a0 100644 --- a/schema/gdrive.go +++ b/schema/gdrive.go @@ -2,18 +2,20 @@ package schema import "time" +// GDriveFile is for db table gdrive_file type GDriveFile struct { - id GDriveFileId - parentId GDriveFileId + id GDriveFileID + parentID GDriveFileID fileType FileType name string md5 MD5 - localFileId LocalFileId + localFileID LocalFileID updatedAt time.Time deleted bool } +// GDriveFileMetric is for db table gdrive_file_metric type GDriveFileMetric struct { - gDriveFileId GDriveFileId + gDriveFileID GDriveFileID foundAt time.Time } diff --git a/schema/local.go b/schema/local.go index 04d7506..f48796a 100644 --- a/schema/local.go +++ b/schema/local.go @@ -2,20 +2,22 @@ package schema import "time" +// LocalFile is for db table local_file type LocalFile struct { - id LocalFileId - parentId LocalFileId + id LocalFileID + parentID LocalFileID fileType FileType name string md5 MD5 - gDriveFileId GDriveFileId - uploadStatus Status + gDriveFileID GDriveFileID + uploadStatus UploadStatus updatedAt time.Time deleted bool } +// LocalFileMetric is for db table local_file_metric type LocalFileMetric struct { - localFileId LocalFileId + localFileID LocalFileID foundAt time.Time queuedAt time.Time uploadStartedAt time.Time From e76d98bd8cb9827541c0b1dabbb7ed889344d1bb Mon Sep 17 00:00:00 2001 From: Peter Jankovsky Date: Tue, 19 Feb 2019 20:45:03 -0800 Subject: [PATCH 3/5] update table row struct names --- .idea/sqldialects.xml | 6 ++++++ schema/gdrive.go | 8 ++++---- schema/local.go | 8 ++++---- 3 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 .idea/sqldialects.xml diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml new file mode 100644 index 0000000..c0e01ca --- /dev/null +++ b/.idea/sqldialects.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/schema/gdrive.go b/schema/gdrive.go index 39947a0..3b548d7 100644 --- a/schema/gdrive.go +++ b/schema/gdrive.go @@ -2,8 +2,8 @@ package schema import "time" -// GDriveFile is for db table gdrive_file -type GDriveFile struct { +// GDriveFileRow is for db table gDriveFile +type GDriveFileRow struct { id GDriveFileID parentID GDriveFileID fileType FileType @@ -14,8 +14,8 @@ type GDriveFile struct { deleted bool } -// GDriveFileMetric is for db table gdrive_file_metric -type GDriveFileMetric struct { +// GDriveFileMetricRow is for db table gDriveFileMetric +type GDriveFileMetricRow struct { gDriveFileID GDriveFileID foundAt time.Time } diff --git a/schema/local.go b/schema/local.go index f48796a..2e4ca49 100644 --- a/schema/local.go +++ b/schema/local.go @@ -2,8 +2,8 @@ package schema import "time" -// LocalFile is for db table local_file -type LocalFile struct { +// LocalFileRow is for db table localFile +type LocalFileRow struct { id LocalFileID parentID LocalFileID fileType FileType @@ -15,8 +15,8 @@ type LocalFile struct { deleted bool } -// LocalFileMetric is for db table local_file_metric -type LocalFileMetric struct { +// LocalFileMetricRow is for db table localFileMetric +type LocalFileMetricRow struct { localFileID LocalFileID foundAt time.Time queuedAt time.Time From c28a4ad48926d03b170fbae2dfe55f1651449064 Mon Sep 17 00:00:00 2001 From: Peter Jankovsky Date: Tue, 19 Feb 2019 20:45:20 -0800 Subject: [PATCH 4/5] clean up code --- etc/settings.json | 1 + paths.go | 12 ------------ settings.go | 17 +++++++++++++++++ 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/etc/settings.json b/etc/settings.json index 1e76a94..fd9ad6e 100644 --- a/etc/settings.json +++ b/etc/settings.json @@ -8,6 +8,7 @@ "LocalRoot": "uploadTest", "DriveRoot": "root", "DbPath": "queue.db", + "SQLitePath": ":memory:", "LocalTime": "America/Los_Angeles", "UploadMaxWorkers": 3 } \ No newline at end of file diff --git a/paths.go b/paths.go index a0ddda6..0d3939d 100644 --- a/paths.go +++ b/paths.go @@ -112,16 +112,4 @@ func makeSortKey(path *Path) string { return path.ModTime.Format("20060102") } -var locTime *time.Location -func getLocTime() *time.Location { - if locTime != nil { - return locTime - } - loc, err := time.LoadLocation(settings.LocalTime) - if err != nil { - log.Fatalf("Unable to local time location: %v", err) - } - locTime = loc - return locTime -} diff --git a/settings.go b/settings.go index e95aa53..5624dcb 100644 --- a/settings.go +++ b/settings.go @@ -5,6 +5,7 @@ import ( "log" "os" "path" + "time" ) type Settings struct { @@ -19,6 +20,7 @@ type Settings struct { DbPath string LocalTime string UploadMaxWorkers int + SQLitePath string } var settings Settings @@ -48,6 +50,21 @@ func loadSettings() { log.Printf(" -- LocalRoot: %v", settings.LocalRoot) log.Printf(" -- DriveRoot: %v", settings.DriveRoot) log.Printf(" -- DbPath: %v", settings.DbPath) + log.Printf(" -- SQLitePath: %v", settings.SQLitePath) log.Printf(" -- LocalTime: %v", settings.LocalTime) log.Printf(" -- UploadMaxWorkers: %v", settings.UploadMaxWorkers) } + +var locTime *time.Location + +func getLocTime() *time.Location { + if locTime != nil { + return locTime + } + loc, err := time.LoadLocation(settings.LocalTime) + if err != nil { + log.Fatalf("Unable to local time location: %v", err) + } + locTime = loc + return locTime +} From 76bf00fb59b8f29b81d821551768a97f3aaf652e Mon Sep 17 00:00:00 2001 From: Peter Jankovsky Date: Tue, 19 Feb 2019 20:45:53 -0800 Subject: [PATCH 5/5] Setup argument based execution + db migration --- main.go | 31 ++++++++- migration/20190219200159_initial_tables.go | 79 ++++++++++++++++++++++ schema.go | 36 ++++++++++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 migration/20190219200159_initial_tables.go create mode 100644 schema.go diff --git a/main.go b/main.go index 9750788..b8d965d 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,42 @@ package main import ( + "flag" "log" "net/http" "sync" ) func main() { + + flag.Parse() + + switch mode := flag.Arg(0) ; mode { + case "daemonV1": + daemonV1() + case "migrateToV2": + migrateToV2() + case "scanGDrive": + scanGDrive() + default: + flag.PrintDefaults() + } + +} + +func scanGDrive() { + loadSettings() + getLocTime() + setupSQL() + getDrive() + +} + +func migrateToV2() { + +} + +func daemonV1() { loadSettings() getLocTime() setupBolt() @@ -31,5 +61,4 @@ func main() { }() wg.Wait() - } diff --git a/migration/20190219200159_initial_tables.go b/migration/20190219200159_initial_tables.go new file mode 100644 index 0000000..22a6bf7 --- /dev/null +++ b/migration/20190219200159_initial_tables.go @@ -0,0 +1,79 @@ +package migration + +import ( + "database/sql" + "github.com/pressly/goose" +) + +func init() { + goose.AddMigration(Up20190219200159, Down20190219200159) +} + +func Up20190219200159(tx *sql.Tx) error { + // This code is executed when the migration is applied. + _, err := tx.Exec(` +CREATE TABLE localFile ( + id INTEGER PRIMARY KEY, + parentID INTEGER, + fileType TEXT NOT NULL, + name TEXT NOT NULL, + md5 TEXT NOT NULL, + gDriveFileID TEXT, + uploadStatus TEXT NOT NULL DEFAULT "unknown", + updatedAt TEXT NOT NULL, + deleted INTEGER NOT NULL DEFAULT 0 +); +`) + if err != nil { + return err + } + _, err = tx.Exec(` +CREATE TABLE localFileMetric ( + localFileID INTEGER NOT NULL, + foundAt TEXT, + queuedAt TEXT, + uploadStartedAt TEXT, + uploadEndedAt TEXT, + uploadRetries INTEGER NOT NULL DEFAULT 0 +); +`) + if err != nil { + return err + } + _, err = tx.Exec(` +CREATE TABLE gDriveFile ( + id TEXT PRIMARY KEY, + parentID TEXT, + fileType TEXT NOT NULL, + name TEXT NOT NULL, + md5 TEXT NOT NULL, + localFileID INTEGER, + updatedAt TEXT NOT NULL, + deleted INTEGER NOT NULL DEFAULT 0 +); +`) + if err != nil { + return err + } + _, err = tx.Exec(` +CREATE TABLE gDriveFileMetric ( + gDriveFileID TEXT, + foundAt TEXT +); +`) + if err != nil { + return err + } + return nil +} + +func Down20190219200159(tx *sql.Tx) error { + // This code is executed when the migration is rolled back. + tx.Exec(` +DROP TABLE localFile; +DROP TABLE localFileMetric; +DROP TABLE gDriveFile; +DROP TABLE gDriveFileMetric; +`) + return nil +} diff --git a/schema.go b/schema.go new file mode 100644 index 0000000..84bf37f --- /dev/null +++ b/schema.go @@ -0,0 +1,36 @@ +package main + +import ( + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + _ "github.com/pjankovsky/gdriver-go/migration" + "github.com/pressly/goose" + "log" +) + +func setupSQL() { + db := getDB() + + err :=goose.SetDialect("sqlite3") + if err != nil { + log.Fatalf("Unable to set goose dialect: %v", err) + } + + err = goose.Up(db.DB, "migration") + if err != nil { + log.Fatalf("Unable to run goose migrations: %v", err) + } +} + +var cachedDB *sqlx.DB + +func getDB() *sqlx.DB { + if cachedDB == nil { + db, err := sqlx.Connect("sqlite3", settings.SQLitePath) + if err != nil { + log.Fatalf("Unable to connect to sqlite3 db: %v", err) + } + cachedDB = db + } + return cachedDB +}