diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml new file mode 100644 index 0000000..c0e01ca --- /dev/null +++ b/.idea/sqldialects.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/etc/settings.json b/etc/settings.json index 1e76a94..fd9ad6e 100644 --- a/etc/settings.json +++ b/etc/settings.json @@ -8,6 +8,7 @@ "LocalRoot": "uploadTest", "DriveRoot": "root", "DbPath": "queue.db", + "SQLitePath": ":memory:", "LocalTime": "America/Los_Angeles", "UploadMaxWorkers": 3 } \ No newline at end of file diff --git a/main.go b/main.go index 9750788..b8d965d 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,42 @@ package main import ( + "flag" "log" "net/http" "sync" ) func main() { + + flag.Parse() + + switch mode := flag.Arg(0) ; mode { + case "daemonV1": + daemonV1() + case "migrateToV2": + migrateToV2() + case "scanGDrive": + scanGDrive() + default: + flag.PrintDefaults() + } + +} + +func scanGDrive() { + loadSettings() + getLocTime() + setupSQL() + getDrive() + +} + +func migrateToV2() { + +} + +func daemonV1() { loadSettings() getLocTime() setupBolt() @@ -31,5 +61,4 @@ func main() { }() wg.Wait() - } diff --git a/migration/20190219200159_initial_tables.go b/migration/20190219200159_initial_tables.go new file mode 100644 index 0000000..22a6bf7 --- /dev/null +++ b/migration/20190219200159_initial_tables.go @@ -0,0 +1,79 @@ +package migration + +import ( + "database/sql" + "github.com/pressly/goose" +) + +func init() { + goose.AddMigration(Up20190219200159, Down20190219200159) +} + +func Up20190219200159(tx *sql.Tx) error { + // This code is executed when the migration is applied. + _, err := tx.Exec(` +CREATE TABLE localFile ( + id INTEGER PRIMARY KEY, + parentID INTEGER, + fileType TEXT NOT NULL, + name TEXT NOT NULL, + md5 TEXT NOT NULL, + gDriveFileID TEXT, + uploadStatus TEXT NOT NULL DEFAULT "unknown", + updatedAt TEXT NOT NULL, + deleted INTEGER NOT NULL DEFAULT 0 +); +`) + if err != nil { + return err + } + _, err = tx.Exec(` +CREATE TABLE localFileMetric ( + localFileID INTEGER NOT NULL, + foundAt TEXT, + queuedAt TEXT, + uploadStartedAt TEXT, + uploadEndedAt TEXT, + uploadRetries INTEGER NOT NULL DEFAULT 0 +); +`) + if err != nil { + return err + } + _, err = tx.Exec(` +CREATE TABLE gDriveFile ( + id TEXT PRIMARY KEY, + parentID TEXT, + fileType TEXT NOT NULL, + name TEXT NOT NULL, + md5 TEXT NOT NULL, + localFileID INTEGER, + updatedAt TEXT NOT NULL, + deleted INTEGER NOT NULL DEFAULT 0 +); +`) + if err != nil { + return err + } + _, err = tx.Exec(` +CREATE TABLE gDriveFileMetric ( + gDriveFileID TEXT, + foundAt TEXT +); +`) + if err != nil { + return err + } + return nil +} + +func Down20190219200159(tx *sql.Tx) error { + // This code is executed when the migration is rolled back. + tx.Exec(` +DROP TABLE localFile; +DROP TABLE localFileMetric; +DROP TABLE gDriveFile; +DROP TABLE gDriveFileMetric; +`) + return nil +} diff --git a/paths.go b/paths.go index a0ddda6..0d3939d 100644 --- a/paths.go +++ b/paths.go @@ -112,16 +112,4 @@ func makeSortKey(path *Path) string { return path.ModTime.Format("20060102") } -var locTime *time.Location -func getLocTime() *time.Location { - if locTime != nil { - return locTime - } - loc, err := time.LoadLocation(settings.LocalTime) - if err != nil { - log.Fatalf("Unable to local time location: %v", err) - } - locTime = loc - return locTime -} diff --git a/schema.go b/schema.go new file mode 100644 index 0000000..84bf37f --- /dev/null +++ b/schema.go @@ -0,0 +1,36 @@ +package main + +import ( + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + _ "github.com/pjankovsky/gdriver-go/migration" + "github.com/pressly/goose" + "log" +) + +func setupSQL() { + db := getDB() + + err :=goose.SetDialect("sqlite3") + if err != nil { + log.Fatalf("Unable to set goose dialect: %v", err) + } + + err = goose.Up(db.DB, "migration") + if err != nil { + log.Fatalf("Unable to run goose migrations: %v", err) + } +} + +var cachedDB *sqlx.DB + +func getDB() *sqlx.DB { + if cachedDB == nil { + db, err := sqlx.Connect("sqlite3", settings.SQLitePath) + if err != nil { + log.Fatalf("Unable to connect to sqlite3 db: %v", err) + } + cachedDB = db + } + return cachedDB +} diff --git a/schema/common.go b/schema/common.go new file mode 100644 index 0000000..95d66c9 --- /dev/null +++ b/schema/common.go @@ -0,0 +1,32 @@ +package schema + +// LocalFileID : DB autoinc ID +type LocalFileID uint64 + +// GDriveFileID : string ID from Google Drive +type GDriveFileID string + +// MD5 : byte slice for MD5 hash +type MD5 [32]byte + +// FileType : designate if the file is a Directory or File +type FileType string + +// enum options for FileType +const ( + FileTypeFile FileType = "F" + FileTypeDir FileType = "D" +) + +// UploadStatus : the status of the upload +type UploadStatus string + +// enum options for UploadStatus +const ( + UploadStatusUnknown UploadStatus = "unknown" + UploadStatusError UploadStatus = "error" + UploadStatusReady UploadStatus = "ready" + UploadStatusPending UploadStatus = "pending" + UploadStatusInProgress UploadStatus = "inprogress" + UploadStatusDone UploadStatus = "done" +) diff --git a/schema/gdrive.go b/schema/gdrive.go new file mode 100644 index 0000000..3b548d7 --- /dev/null +++ b/schema/gdrive.go @@ -0,0 +1,21 @@ +package schema + +import "time" + +// GDriveFileRow is for db table gDriveFile +type GDriveFileRow struct { + id GDriveFileID + parentID GDriveFileID + fileType FileType + name string + md5 MD5 + localFileID LocalFileID + updatedAt time.Time + deleted bool +} + +// GDriveFileMetricRow is for db table gDriveFileMetric +type GDriveFileMetricRow struct { + gDriveFileID GDriveFileID + foundAt time.Time +} diff --git a/schema/local.go b/schema/local.go new file mode 100644 index 0000000..2e4ca49 --- /dev/null +++ b/schema/local.go @@ -0,0 +1,26 @@ +package schema + +import "time" + +// LocalFileRow is for db table localFile +type LocalFileRow struct { + id LocalFileID + parentID LocalFileID + fileType FileType + name string + md5 MD5 + gDriveFileID GDriveFileID + uploadStatus UploadStatus + updatedAt time.Time + deleted bool +} + +// LocalFileMetricRow is for db table localFileMetric +type LocalFileMetricRow struct { + localFileID LocalFileID + foundAt time.Time + queuedAt time.Time + uploadStartedAt time.Time + uploadEndedAt time.Time + uploadRetries int +} diff --git a/settings.go b/settings.go index e95aa53..5624dcb 100644 --- a/settings.go +++ b/settings.go @@ -5,6 +5,7 @@ import ( "log" "os" "path" + "time" ) type Settings struct { @@ -19,6 +20,7 @@ type Settings struct { DbPath string LocalTime string UploadMaxWorkers int + SQLitePath string } var settings Settings @@ -48,6 +50,21 @@ func loadSettings() { log.Printf(" -- LocalRoot: %v", settings.LocalRoot) log.Printf(" -- DriveRoot: %v", settings.DriveRoot) log.Printf(" -- DbPath: %v", settings.DbPath) + log.Printf(" -- SQLitePath: %v", settings.SQLitePath) log.Printf(" -- LocalTime: %v", settings.LocalTime) log.Printf(" -- UploadMaxWorkers: %v", settings.UploadMaxWorkers) } + +var locTime *time.Location + +func getLocTime() *time.Location { + if locTime != nil { + return locTime + } + loc, err := time.LoadLocation(settings.LocalTime) + if err != nil { + log.Fatalf("Unable to local time location: %v", err) + } + locTime = loc + return locTime +}