From 7b7f6a7faed20210461f3a926230d430258b1b67 Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sat, 3 Aug 2024 19:00:09 +0200 Subject: [PATCH 01/15] added all data/ to ignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 24056b0..07bebaa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ tmp* tools dedupe/bin/* -*.exe \ No newline at end of file +*.exe +data/* From c8cd133e0fdc96939b60e71495336fcf7d3b1fe4 Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sat, 3 Aug 2024 21:37:51 +0200 Subject: [PATCH 02/15] Started on making google timeline feature --- google_location_data/lib/lib.go | 41 ++++++++++++++++ google_location_data/lib/lib_test.go | 51 ++++++++++++++++++++ google_location_data/main.go | 29 +++++++++++ photoMetadata/README.md | 4 ++ photoMetadata/lib/lib.go | 50 +++++++++++++++++++ photoMetadata/main.go | 72 ++++++++++++++++++++++++++++ 6 files changed, 247 insertions(+) create mode 100644 google_location_data/lib/lib.go create mode 100644 google_location_data/lib/lib_test.go create mode 100644 google_location_data/main.go create mode 100644 photoMetadata/README.md create mode 100644 photoMetadata/lib/lib.go create mode 100644 photoMetadata/main.go diff --git a/google_location_data/lib/lib.go b/google_location_data/lib/lib.go new file mode 100644 index 0000000..870ee5a --- /dev/null +++ b/google_location_data/lib/lib.go @@ -0,0 +1,41 @@ +package lib + +import ( + "encoding/json" + "io" + "log" + "os" +) + +type SourceLocationData struct { + Locations []SourceLocationRecord `json:"locations"` +} + +type SourceLocationRecord struct { + LatitudeE7 float64 `json:"latitudeE7"` + LongitudeE7 float64 `json:"longitudeE7"` + TimeStamp string `json:"timestamp"` +} + +func ImportSourceLocationData(path string) SourceLocationData { + // Read the data from the file + jsonFile, err := os.Open(path) + if err != nil { + log.Fatalf("Unable to open source data file: %v", err) + } + defer jsonFile.Close() + // Unmarshal the data into a struct + bytes, err := io.ReadAll(jsonFile) + if err != nil { + log.Fatalf("Unable to read source data file: %v", err) + } + + sourceData := SourceLocationData{} + err = json.Unmarshal(bytes, &sourceData) + if err != nil { + log.Fatalf("Unable to unmarshal source data: %v", err) + } + + // Return the list of records + return sourceData2 +} diff --git a/google_location_data/lib/lib_test.go b/google_location_data/lib/lib_test.go new file mode 100644 index 0000000..c14ebc8 --- /dev/null +++ b/google_location_data/lib/lib_test.go @@ -0,0 +1,51 @@ +package lib + +import ( + "os" + "path/filepath" + "testing" +) + +const TEST_DATA_FOLDER = "lib_test_data" + +func TestImportSourceLocationData(t *testing.T) { + // Create a test data file + testDataFolder := filepath.Join(TEST_DATA_FOLDER, "TestImportSourceLocationData") + testSourceData := filepath.Join(testDataFolder, "data.json") + err := os.MkdirAll(testDataFolder, 0o755) + if err != nil { + t.Fatalf("Error creating test data folder: %v", err) + } + defer os.RemoveAll(TEST_DATA_FOLDER) + + data := `{ + "locations": [ + {"latitudeE7": 1234567, "longitudeE7": 2345678, "timestamp": "2021-01-01T12:00:00Z"}, + {"latitudeE7": 2345678, "longitudeE7": 3456789, "timestamp": "2021-01-02T12:00:00Z"}, + {"latitudeE7": 3456789, "longitudeE7": 4567890, "timestamp": "2021-01-03T12:00:00Z"} + ] + }` + + err = os.WriteFile(testSourceData, []byte(data), 0o644) + if err != nil { + t.Fatalf("Error creating test data file: %v", err) + } + + // Test the ImportSourceLocationData function + sourceData := ImportSourceLocationData(testSourceData) + locations := sourceData.Locations + if len(locations) != 3 { + t.Errorf("Expected 3 records, got %d", len(locations)) + } + + // Check the values of the records + if locations[0].LatitudeE7 != 1234567 { + t.Errorf("Expected latitude 1234567, got %f", locations[0].LatitudeE7) + } + if locations[1].LongitudeE7 != 3456789 { + t.Errorf("Expected longitude 3456789, got %f", locations[0].LongitudeE7) + } + if locations[2].TimeStamp != "2021-01-03T12:00:00Z" { + t.Errorf("Expected timestamp 2021-01-03T12:00:00Z, got %s", locations[0].TimeStamp) + } +} diff --git a/google_location_data/main.go b/google_location_data/main.go new file mode 100644 index 0000000..558ceb7 --- /dev/null +++ b/google_location_data/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "image" + "image/png" + "os" + + "github.com/dustin/go-heatmap" + "github.com/dustin/go-heatmap/schemes" + "github.com/sander-skjulsvik/tools/google_location_data/lib" +) + +func main() { + googleLocationPath := "data/takeout-20240803T162513Z-001/Takeout/Location History (Timeline)/Records.json" + locationRecords := lib.ImportSourceLocationData(googleLocationPath) + + points := []heatmap.DataPoint{} + for _, record := range locationRecords.Locations { + points = append(points, + heatmap.P(record.LatitudeE7/1e7, record.LongitudeE7/1e7)) + } + + // scheme, _ := schemes.FromImage("../schemes/fire.png") + scheme := schemes.AlphaFire + + img := heatmap.Heatmap(image.Rect(0, 0, 1024, 1024), + points, 150, 128, scheme) + png.Encode(os.Stdout, img) +} diff --git a/photoMetadata/README.md b/photoMetadata/README.md new file mode 100644 index 0000000..eb0865f --- /dev/null +++ b/photoMetadata/README.md @@ -0,0 +1,4 @@ +## Requirements + +### ExifTool +sudo apt-get install exiftool diff --git a/photoMetadata/lib/lib.go b/photoMetadata/lib/lib.go new file mode 100644 index 0000000..e2ff3bc --- /dev/null +++ b/photoMetadata/lib/lib.go @@ -0,0 +1,50 @@ +package lib + +import ( + "fmt" + + "github.com/barasher/go-exiftool" +) + +func GetAllExifData(filePath string) ([]exiftool.FileMetadata, error) { + et, err := exiftool.NewExiftool() + if err != nil { + return nil, fmt.Errorf("Error when intializing: %v", err) + } + defer et.Close() + return et.ExtractMetadata(filePath), nil +} + +func PrintAllExifData(fileInfos []exiftool.FileMetadata) error { + for _, fileInfo := range fileInfos { + if fileInfo.Err != nil { + fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err) + continue + } + + for k, v := range fileInfo.Fields { + fmt.Printf("[%v] %v\n", k, v) + } + } + return nil +} + +func WriteExifDataToFile(key, value, filePath string) error { + et, err := exiftool.NewExiftool() + if err != nil { + return fmt.Errorf("Error when intializing: %v", err) + } + defer et.Close() + currentData := et.ExtractMetadata(filePath) + + currentData[0].SetString(key, value) + + et.WriteMetadata(currentData) + for _, d := range currentData { + if d.Err != nil { + return fmt.Errorf("Error concerning %v: %v\n", d.File, d.Err) + } + } + + return nil +} diff --git a/photoMetadata/main.go b/photoMetadata/main.go new file mode 100644 index 0000000..5357205 --- /dev/null +++ b/photoMetadata/main.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "os" + + "github.com/sander-skjulsvik/tools/photoMetadata/lib" +) + +func main() { + method := os.Args[1] + photoPath := os.Args[2] + + switch method { + case "list": + List(photoPath) + case "write": + Write(photoPath) + // List(photoPath) + pos := Search(photoPath, "GPSPosition") + fmt.Println(pos) + case "search": + pos := Search(photoPath, "GPSPosition") + fmt.Println(pos) + default: + fmt.Println("Invalid method") + } +} + +func List(path string) { + data, err := lib.GetAllExifData(path) + if err != nil { + fmt.Printf("Error: %v", err) + return + } + err = lib.PrintAllExifData(data) + if err != nil { + fmt.Printf("Error: %v", err) + } +} + +func Write(path string) error { + err := lib.WriteExifDataToFile("GPSPosition", "61 deg 39' 50.71\" N, 9 deg 39' 57.94\" E", path) + if err != nil { + return fmt.Errorf("Error writing to file: %v", err) + } + return nil +} + +func Search(path string, search string) interface{} { + fmt.Println("Searching for data") + + fileInfos, err := lib.GetAllExifData(path) + if err != nil { + fmt.Printf("Error: %v", err) + return nil + } + + for _, fileInfo := range fileInfos { + if fileInfo.Err != nil { + fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err) + continue + } + + for k, v := range fileInfo.Fields { + if k == search { + return v + } + } + } + return nil +} From 9eb32db4fe7bfbbbd97401fabb8e4910663298da Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sat, 10 Aug 2024 23:03:24 +0200 Subject: [PATCH 03/15] continue on the parsing of google data --- go.mod | 2 + go.sum | 8 ++ google_location_data/lib/lib.go | 114 ++++++++++++++++++++++++++- google_location_data/lib/lib_test.go | 86 +++++++++++++++++--- 4 files changed, 198 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 6000997..0d85722 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,9 @@ require ( ) require ( + github.com/barasher/go-exiftool v1.10.0 github.com/deckarep/golang-set/v2 v2.6.0 + github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a golang.org/x/sys v0.19.0 // indirect gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index 39755a2..e6d6185 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,10 @@ +github.com/barasher/go-exiftool v1.10.0 h1:f5JY5jc42M7tzR6tbL9508S2IXdIcG9QyieEXNMpIhs= +github.com/barasher/go-exiftool v1.10.0/go.mod h1:F9s/a3uHSM8YniVfwF+sbQUtP8Gmh9nyzigNF+8vsWo= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a h1:o0oeorr6BKW/lHrT2NWX1CQn6LDhr5lvAPIIjTDC1bg= +github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a/go.mod h1:VBmwC4U3p2SMEKr+/m5j0eby7rmUtSoA5TGLwe6P+3A= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= @@ -10,6 +15,9 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/google_location_data/lib/lib.go b/google_location_data/lib/lib.go index 870ee5a..8444860 100644 --- a/google_location_data/lib/lib.go +++ b/google_location_data/lib/lib.go @@ -5,12 +5,78 @@ import ( "io" "log" "os" + "sort" + "time" ) +var RFC3339_LAYOUT string = "2006-01-02T15:04:05Z07:00" + +type GoogleTimelineLocationStore struct { + SourceLocations SourceLocationData + + // Time difference thresholds + // Low time difference threshold + LowTimeDiffThreshold time.Duration + // Medium time difference threshold + MediumTimeDiffThreshold time.Duration + // High time difference threshold + HighTimeDiffThreshold time.Duration +} + +const ( + LOW_TIME_DIFF_THRESHOLD = 30 * time.Minute + MEDIUM_TIME_DIFF_THRESHOLD = 2 * time.Hour + HIGH_TIME_DIFF_THRESHOLD = 12 * time.Hour +) + +func NewGoogleTimelineLocationStoreFromFile(path string) *GoogleTimelineLocationStore { + return &GoogleTimelineLocationStore{ + SourceLocations: ImportSourceLocationData(path), + LowTimeDiffThreshold: LOW_TIME_DIFF_THRESHOLD, + MediumTimeDiffThreshold: MEDIUM_TIME_DIFF_THRESHOLD, + HighTimeDiffThreshold: HIGH_TIME_DIFF_THRESHOLD, + } +} + type SourceLocationData struct { Locations []SourceLocationRecord `json:"locations"` } +// This function assumes Locations is sorted by time +func (sourceData *SourceLocationData) GetLocation(time time.Time) (before SourceLocationRecord, after SourceLocationRecord, err error) { + locationBeforeInd := sort.Search(len(sourceData.Locations), func(i int) bool { + return sourceData.Locations[i].GetTime().After(time) + }) + + locationAfterInd := locationBeforeInd + 1 + + return sourceData.Locations[locationBeforeInd], sourceData.Locations[locationAfterInd], nil +} + +func (sourceData *SourceLocationData) SortByTime() { + // Sort the data by time + sort.Sort(ByTime(sourceData.Locations)) + // return sourceData.Locations +} + +type ByTime []SourceLocationRecord + +func (bt ByTime) Less(i, j int) bool { + return bt[i].GetTime().Before(bt[j].GetTime()) +} + +func (bt ByTime) Swap(i, j int) { + bt[i], bt[j] = bt[j], bt[i] +} + +func (bt ByTime) Len() int { + return len(bt) +} + +func (bt ByTime) Comp(i, j int) int { + return bt[i].GetTime().Compare(bt[j].GetTime()) +} + type SourceLocationRecord struct { LatitudeE7 float64 `json:"latitudeE7"` LongitudeE7 float64 `json:"longitudeE7"` @@ -35,7 +101,51 @@ func ImportSourceLocationData(path string) SourceLocationData { if err != nil { log.Fatalf("Unable to unmarshal source data: %v", err) } - // Return the list of records - return sourceData2 + return sourceData } + +func (locStore *SourceLocationRecord) GetTime() time.Time { + return ParseTime(locStore.TimeStamp) +} + +func ParseTime(timeStr string) time.Time { + t, err := time.Parse(RFC3339_LAYOUT, timeStr) + if err != nil { + log.Fatalf("Unable to parse timestamp: %v", err) + } + return t +} + +type Location struct { + LatitudeE7 float64 + LongitudeE7 float64 +} + +/* +This function will have to do some assumtions when the time between location stamps is too large. + +It is implemented with 3 types of assumptions: + +- If the time between the photo and the location is low we will assume it is accurate. +- If the time between the photo and the location is medium we will assume we will attempt a linear interpolation between the two locations. +- If the time is large we will return an error, and assume the user will have to provide the data themselves. +*/ +// func (locStore *GoogleTimelineLocationStore) GetLocationByTime(time time.Time) (Location, error) { +// // Find the closest location to the given time + +// // Return the location +// // return Location{ +// // LatitudeE7: closestLocation.LatitudeE7, +// // LongitudeE7: closestLocation.LongitudeE7, +// // } +// } + +// func (locStore *GoogleTimelineLocationStore) GetLocationByTimeBefore(time time.Time) (Location, error) { +// for _, record := range locStore.SourceLocations.Locations { +// time.Parse(RFC3339_LAYOUT, record.TimeStamp) +// } +// } + +// func (locStore *GoogleTimelineLocationStore) GetLocationByTimeAfter(time time.Time) (Location, error) { +// } diff --git a/google_location_data/lib/lib_test.go b/google_location_data/lib/lib_test.go index c14ebc8..ca861fb 100644 --- a/google_location_data/lib/lib_test.go +++ b/google_location_data/lib/lib_test.go @@ -6,7 +6,24 @@ import ( "testing" ) -const TEST_DATA_FOLDER = "lib_test_data" +const ( + TEST_DATA_FOLDER = "lib_test_data" + SIMPLE_TEST_DATA_STRING = `{ + "locations": [ + {"latitudeE7": 1234567, "longitudeE7": 2345678, "timestamp": "2021-01-01T12:00:00Z"}, + {"latitudeE7": 2345678, "longitudeE7": 3456789, "timestamp": "2021-01-02T12:00:00Z"}, + {"latitudeE7": 3456789, "longitudeE7": 4567890, "timestamp": "2021-01-03T12:00:00Z"} + ] + }` +) + +var SIMPLE_TEST_DATA_SOURCE_LOCATION = SourceLocationData{ + Locations: []SourceLocationRecord{ + {LatitudeE7: 1234567, LongitudeE7: 2345678, TimeStamp: "2021-01-01T12:00:00Z"}, + {LatitudeE7: 2345678, LongitudeE7: 3456789, TimeStamp: "2021-01-02T12:00:00Z"}, + {LatitudeE7: 3456789, LongitudeE7: 4567890, TimeStamp: "2021-01-03T12:00:00Z"}, + }, +} func TestImportSourceLocationData(t *testing.T) { // Create a test data file @@ -18,15 +35,7 @@ func TestImportSourceLocationData(t *testing.T) { } defer os.RemoveAll(TEST_DATA_FOLDER) - data := `{ - "locations": [ - {"latitudeE7": 1234567, "longitudeE7": 2345678, "timestamp": "2021-01-01T12:00:00Z"}, - {"latitudeE7": 2345678, "longitudeE7": 3456789, "timestamp": "2021-01-02T12:00:00Z"}, - {"latitudeE7": 3456789, "longitudeE7": 4567890, "timestamp": "2021-01-03T12:00:00Z"} - ] - }` - - err = os.WriteFile(testSourceData, []byte(data), 0o644) + err = os.WriteFile(testSourceData, []byte(SIMPLE_TEST_DATA_STRING), 0o644) if err != nil { t.Fatalf("Error creating test data file: %v", err) } @@ -49,3 +58,60 @@ func TestImportSourceLocationData(t *testing.T) { t.Errorf("Expected timestamp 2021-01-03T12:00:00Z, got %s", locations[0].TimeStamp) } } + +func TestSortByTime(t *testing.T) { + sourceData := SIMPLE_TEST_DATA_SOURCE_LOCATION + // Swap + sourceData.Locations[0], sourceData.Locations[1] = sourceData.Locations[1], sourceData.Locations[0] + sourceData.SortByTime() + locations := sourceData.Locations + + // Check the order of the records + if locations[0].TimeStamp != "2021-01-01T12:00:00Z" { + t.Errorf("Expected first record timestamp 2021-01-01T12:00:00Z, got %s", locations[0].TimeStamp) + } + if locations[1].TimeStamp != "2021-01-02T12:00:00Z" { + t.Errorf("Expected second record timestamp 2021-01-02T12:00:00Z, got %s", locations[1].TimeStamp) + } + if locations[2].TimeStamp != "2021-01-03T12:00:00Z" { + t.Errorf("Expected third record timestamp 2021-01-03T12:00:00Z, got %s", locations[2].TimeStamp) + } +} + +func TestByTimeComp(t *testing.T) { + var byTime ByTime + byTime = SIMPLE_TEST_DATA_SOURCE_LOCATION.Locations + + if byTime.Comp(0, 1) != -1 { + t.Errorf("Expected -1, got %d", byTime.Comp(0, 1)) + } + if byTime.Comp(1, 0) != 1 { + t.Errorf("Expected 1, got %d", byTime.Comp(1, 0)) + } + if byTime.Comp(0, 0) != 0 { + t.Errorf("Expected 0, got %d", byTime.Comp(0, 0)) + } +} + +func TestGetLocation(t *testing.T) { + // Setup + sourceData := SIMPLE_TEST_DATA_SOURCE_LOCATION + sourceData.Locations = append(sourceData.Locations, SourceLocationRecord{ + LatitudeE7: 4567890, LongitudeE7: 5678901, TimeStamp: "2021-01-04T12:00:00Z", + }) + sourceData.SortByTime() + + // Test the GetLocation function + locationBefore, locationAfter, err := sourceData.GetLocation(ParseTime("2021-01-02T12:00:00Z")) + if err != nil { + t.Errorf("Error getting location: %v", err) + } + + // Check the values of the records + if locationBefore.TimeStamp != "2021-01-01T12:00:00Z" { + t.Errorf("Expected location before timestamp 2021-01-01T12:00:00Z, got %s", locationBefore.TimeStamp) + } + if locationAfter.TimeStamp != "2021-01-03T12:00:00Z" { + t.Errorf("Expected location after timestamp 2021-01-03T12:00:00Z, got %s", locationAfter.TimeStamp) + } +} From 35d54acab662201dfccea213d76eb4cc7a5371dc Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sun, 11 Aug 2024 11:48:48 +0200 Subject: [PATCH 04/15] FindClosestLocation now tested --- google_location_data/lib/lib.go | 27 ++++++++++++-- google_location_data/lib/lib_test.go | 56 +++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/google_location_data/lib/lib.go b/google_location_data/lib/lib.go index 8444860..19f3b2b 100644 --- a/google_location_data/lib/lib.go +++ b/google_location_data/lib/lib.go @@ -43,14 +43,33 @@ type SourceLocationData struct { } // This function assumes Locations is sorted by time -func (sourceData *SourceLocationData) GetLocation(time time.Time) (before SourceLocationRecord, after SourceLocationRecord, err error) { - locationBeforeInd := sort.Search(len(sourceData.Locations), func(i int) bool { +func (sourceData *SourceLocationData) FindClosestLocation(time time.Time) (ind int, err error) { + var ( + locationBeforeInd int + locationAfterInd int + ) + + locationAfterInd = sort.Search(len(sourceData.Locations), func(i int) bool { return sourceData.Locations[i].GetTime().After(time) }) + log.Default().Printf("locationAfterInd: %d", locationAfterInd) + // Handling edge cases, max and min + if locationAfterInd == len(sourceData.Locations) { + return locationAfterInd - 1, nil + } + if locationAfterInd == 0 { + return locationAfterInd, nil + } - locationAfterInd := locationBeforeInd + 1 + locationBeforeInd = locationAfterInd - 1 + afterTime := sourceData.Locations[locationAfterInd].GetTime() + beforeTime := sourceData.Locations[locationBeforeInd].GetTime() - return sourceData.Locations[locationBeforeInd], sourceData.Locations[locationAfterInd], nil + if time.Sub(afterTime).Abs() <= time.Sub(beforeTime).Abs() { + return locationAfterInd, nil + } else { + return locationBeforeInd, nil + } } func (sourceData *SourceLocationData) SortByTime() { diff --git a/google_location_data/lib/lib_test.go b/google_location_data/lib/lib_test.go index ca861fb..8432c59 100644 --- a/google_location_data/lib/lib_test.go +++ b/google_location_data/lib/lib_test.go @@ -101,17 +101,53 @@ func TestGetLocation(t *testing.T) { }) sourceData.SortByTime() - // Test the GetLocation function - locationBefore, locationAfter, err := sourceData.GetLocation(ParseTime("2021-01-02T12:00:00Z")) - if err != nil { - t.Errorf("Error getting location: %v", err) - } + // Test the GetLocation function with exact match + { + timeStamp := "2021-01-02T12:00:00Z" + locationInd, err := sourceData.FindClosestLocation(ParseTime(timeStamp)) + if err != nil { + t.Errorf("Error getting location: %v", err) + } + location := sourceData.Locations[locationInd] - // Check the values of the records - if locationBefore.TimeStamp != "2021-01-01T12:00:00Z" { - t.Errorf("Expected location before timestamp 2021-01-01T12:00:00Z, got %s", locationBefore.TimeStamp) + // Check the values of the records + if location.TimeStamp != timeStamp { + t.Errorf("Expected location before timestamp %s, got %s", timeStamp, location.TimeStamp) + } } - if locationAfter.TimeStamp != "2021-01-03T12:00:00Z" { - t.Errorf("Expected location after timestamp 2021-01-03T12:00:00Z, got %s", locationAfter.TimeStamp) + // Test the GetLocation function with in-between time + { + timeStamp := "2021-01-02T18:00:00Z" + locationInd, err := sourceData.FindClosestLocation(ParseTime(timeStamp)) + if err != nil { + t.Errorf("Error getting location: %v", err) + } + locationBefore := sourceData.Locations[locationInd] + if locationBefore.TimeStamp != "2021-01-02T12:00:00Z" { + t.Errorf("Expected location before timestamp 2021-01-02T12:00:00Z, got %s", locationBefore.TimeStamp) + } + } + + // Testing limits + { + timestampFarAfter := "2022-01-01T12:00:00Z" + beforeInd, err := sourceData.FindClosestLocation(ParseTime(timestampFarAfter)) + before := sourceData.Locations[beforeInd] + if err != nil { + t.Errorf("Error getting location: %v", err) + } + if before.TimeStamp != "2021-01-04T12:00:00Z" { + t.Errorf("Expected location before timestamp 2021-01-04T12:00:00Z, got %s", before.TimeStamp) + } + + timestampFarBefore := "2020-01-01T12:00:00Z" + afterInd, err := sourceData.FindClosestLocation(ParseTime(timestampFarBefore)) + after := sourceData.Locations[afterInd] + if err != nil { + t.Errorf("Error getting location: %v", err) + } + if after.TimeStamp != sourceData.Locations[0].TimeStamp { + t.Errorf("Expected location after timestamp %s, got %s", sourceData.Locations[0].TimeStamp, after.TimeStamp) + } } } From 83196828d6581537df800d5c1bcf91db8160b59c Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Fri, 16 Aug 2024 13:50:35 +0200 Subject: [PATCH 05/15] Moved google import to its own file, made the coordinates type, cleaned some --- go.mod | 6 + go.sum | 12 ++ google_location_data/lib/lib.go | 189 ++++++++++------- google_location_data/lib/lib_test.go | 198 +++++++++++------- google_location_data/lib/sourceGoogle.go | 66 ++++++ google_location_data/lib/sourceGoogle_test.go | 32 +++ 6 files changed, 348 insertions(+), 155 deletions(-) create mode 100644 google_location_data/lib/sourceGoogle.go create mode 100644 google_location_data/lib/sourceGoogle_test.go diff --git a/go.mod b/go.mod index 0d85722..5e1f03d 100644 --- a/go.mod +++ b/go.mod @@ -5,16 +5,22 @@ go 1.22 require github.com/gosuri/uiprogress v0.0.1 require ( + github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/gosuri/uilive v0.0.4 // indirect + github.com/kylelemons/go-gypsy v1.0.0 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/ziutek/mymysql v1.5.4 // indirect ) require ( github.com/barasher/go-exiftool v1.10.0 github.com/deckarep/golang-set/v2 v2.6.0 github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a + github.com/jftuga/geodist v1.0.0 + github.com/kellydunn/golang-geo v0.7.0 golang.org/x/sys v0.19.0 // indirect gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index e6d6185..27b7702 100644 --- a/go.sum +++ b/go.sum @@ -5,12 +5,22 @@ github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80N github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a h1:o0oeorr6BKW/lHrT2NWX1CQn6LDhr5lvAPIIjTDC1bg= github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a/go.mod h1:VBmwC4U3p2SMEKr+/m5j0eby7rmUtSoA5TGLwe6P+3A= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/jftuga/geodist v1.0.0 h1:PFPQlZtj10u8ETAYTyxE0DWMl1bwA+Xzrqb4+oLkkC0= +github.com/jftuga/geodist v1.0.0/go.mod h1:BohEDxpZ8S5ADAxW/9EKPSKWOVl0+3wHENIT40m4UO4= +github.com/kellydunn/golang-geo v0.7.0 h1:A5j0/BvNgGwY6Yb6inXQxzYwlPHc6WVZR+MrarZYNNg= +github.com/kellydunn/golang-geo v0.7.0/go.mod h1:YYlQPJ+DPEzrHx8kT3oPHC/NjyvCCXE+IuKGKdrjrcU= +github.com/kylelemons/go-gypsy v1.0.0 h1:7/wQ7A3UL1bnqRMnZ6T8cwCOArfZCxFmb1iTxaOOo1s= +github.com/kylelemons/go-gypsy v1.0.0/go.mod h1:chkXM0zjdpXOiqkCW1XcCHDfjfk14PH2KKkQWxfJUcU= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -18,6 +28,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/google_location_data/lib/lib.go b/google_location_data/lib/lib.go index 19f3b2b..7d2edba 100644 --- a/google_location_data/lib/lib.go +++ b/google_location_data/lib/lib.go @@ -1,18 +1,18 @@ package lib import ( - "encoding/json" - "io" + "fmt" "log" - "os" "sort" "time" + + geo "github.com/kellydunn/golang-geo" ) var RFC3339_LAYOUT string = "2006-01-02T15:04:05Z07:00" -type GoogleTimelineLocationStore struct { - SourceLocations SourceLocationData +type LocationStore struct { + SourceLocations SourceLocations // Time difference thresholds // Low time difference threshold @@ -29,28 +29,19 @@ const ( HIGH_TIME_DIFF_THRESHOLD = 12 * time.Hour ) -func NewGoogleTimelineLocationStoreFromFile(path string) *GoogleTimelineLocationStore { - return &GoogleTimelineLocationStore{ - SourceLocations: ImportSourceLocationData(path), - LowTimeDiffThreshold: LOW_TIME_DIFF_THRESHOLD, - MediumTimeDiffThreshold: MEDIUM_TIME_DIFF_THRESHOLD, - HighTimeDiffThreshold: HIGH_TIME_DIFF_THRESHOLD, - } -} - -type SourceLocationData struct { - Locations []SourceLocationRecord `json:"locations"` +type SourceLocations struct { + Locations []LocationRecord `json:"locations"` } // This function assumes Locations is sorted by time -func (sourceData *SourceLocationData) FindClosestLocation(time time.Time) (ind int, err error) { +func (sourceData *SourceLocations) FindClosestLocation(time time.Time) (ind int, err error) { var ( locationBeforeInd int locationAfterInd int ) locationAfterInd = sort.Search(len(sourceData.Locations), func(i int) bool { - return sourceData.Locations[i].GetTime().After(time) + return sourceData.Locations[i].Time.After(time) }) log.Default().Printf("locationAfterInd: %d", locationAfterInd) // Handling edge cases, max and min @@ -62,8 +53,8 @@ func (sourceData *SourceLocationData) FindClosestLocation(time time.Time) (ind i } locationBeforeInd = locationAfterInd - 1 - afterTime := sourceData.Locations[locationAfterInd].GetTime() - beforeTime := sourceData.Locations[locationBeforeInd].GetTime() + afterTime := sourceData.Locations[locationAfterInd].Time + beforeTime := sourceData.Locations[locationBeforeInd].Time if time.Sub(afterTime).Abs() <= time.Sub(beforeTime).Abs() { return locationAfterInd, nil @@ -72,16 +63,16 @@ func (sourceData *SourceLocationData) FindClosestLocation(time time.Time) (ind i } } -func (sourceData *SourceLocationData) SortByTime() { +func (sourceData *SourceLocations) SortByTime() { // Sort the data by time sort.Sort(ByTime(sourceData.Locations)) // return sourceData.Locations } -type ByTime []SourceLocationRecord +type ByTime []LocationRecord func (bt ByTime) Less(i, j int) bool { - return bt[i].GetTime().Before(bt[j].GetTime()) + return bt[i].Time.Before(bt[j].Time) } func (bt ByTime) Swap(i, j int) { @@ -92,53 +83,57 @@ func (bt ByTime) Len() int { return len(bt) } -func (bt ByTime) Comp(i, j int) int { - return bt[i].GetTime().Compare(bt[j].GetTime()) -} - -type SourceLocationRecord struct { - LatitudeE7 float64 `json:"latitudeE7"` - LongitudeE7 float64 `json:"longitudeE7"` - TimeStamp string `json:"timestamp"` +type LocationRecord struct { + Corrdinates Corrdinates `json:"coordinates"` + Time time.Time `json:"timestamp"` } -func ImportSourceLocationData(path string) SourceLocationData { - // Read the data from the file - jsonFile, err := os.Open(path) - if err != nil { - log.Fatalf("Unable to open source data file: %v", err) - } - defer jsonFile.Close() - // Unmarshal the data into a struct - bytes, err := io.ReadAll(jsonFile) +func ParseTime(timeStr string) time.Time { + t, err := time.Parse(RFC3339_LAYOUT, timeStr) if err != nil { - log.Fatalf("Unable to read source data file: %v", err) + log.Fatalf("Unable to parse timestamp: %v", err) } + return t +} - sourceData := SourceLocationData{} - err = json.Unmarshal(bytes, &sourceData) - if err != nil { - log.Fatalf("Unable to unmarshal source data: %v", err) - } - // Return the list of records - return sourceData +type Corrdinates struct { + LatitudeE7 int + LongitudeE7 int } -func (locStore *SourceLocationRecord) GetTime() time.Time { - return ParseTime(locStore.TimeStamp) +var ( + ErrLatitudeOutOfRange = fmt.Errorf("Latitude out of range") + ErrLongitudeOutOfRange = fmt.Errorf("Longitude out of range") +) + +func NewCorrdinatesE7(lat, long int) (Corrdinates, error) { + if lat < -90e7 || lat > 90e7 { + return Corrdinates{}, fmt.Errorf("%w: %d", ErrLatitudeOutOfRange, lat) + } + if long < -180e7 || long > 180e7 { + return Corrdinates{}, fmt.Errorf("%w: %d", ErrLongitudeOutOfRange, long) + } + return Corrdinates{ + LatitudeE7: lat, + LongitudeE7: long, + }, nil } -func ParseTime(timeStr string) time.Time { - t, err := time.Parse(RFC3339_LAYOUT, timeStr) - if err != nil { - log.Fatalf("Unable to parse timestamp: %v", err) +func NewCorrdinatesE2(lat, long float64) (Corrdinates, error) { + if lat < -90 || lat > 90 { + return Corrdinates{}, fmt.Errorf("%w: %f", ErrLatitudeOutOfRange, lat) } - return t + if long < -180 || long > 180 { + return Corrdinates{}, fmt.Errorf("%w: %f", ErrLongitudeOutOfRange, long) + } + return Corrdinates{ + LatitudeE7: int(lat * 1e7), + LongitudeE7: int(long * 1e7), + }, nil } -type Location struct { - LatitudeE7 float64 - LongitudeE7 float64 +func (coordinate *Corrdinates) GetE2Coord() (lat float64, long float64) { + return float64(coordinate.LatitudeE7) / 1e7, float64(coordinate.LongitudeE7) / 1e7 } /* @@ -150,21 +145,63 @@ It is implemented with 3 types of assumptions: - If the time between the photo and the location is medium we will assume we will attempt a linear interpolation between the two locations. - If the time is large we will return an error, and assume the user will have to provide the data themselves. */ -// func (locStore *GoogleTimelineLocationStore) GetLocationByTime(time time.Time) (Location, error) { -// // Find the closest location to the given time - -// // Return the location -// // return Location{ -// // LatitudeE7: closestLocation.LatitudeE7, -// // LongitudeE7: closestLocation.LongitudeE7, -// // } -// } - -// func (locStore *GoogleTimelineLocationStore) GetLocationByTimeBefore(time time.Time) (Location, error) { -// for _, record := range locStore.SourceLocations.Locations { -// time.Parse(RFC3339_LAYOUT, record.TimeStamp) -// } -// } - -// func (locStore *GoogleTimelineLocationStore) GetLocationByTimeAfter(time time.Time) (Location, error) { -// } + +var ( + ErrTimeDiffTooHigh = fmt.Errorf("Time difference too high") + ErrTimeDiffMedium = fmt.Errorf("Time difference medium") + ErrNoLocation = fmt.Errorf("No location found") +) + +func (locStore *LocationStore) GetLocationByTime(time time.Time) (Corrdinates, error) { + // Find the closest location to the given time + closestLocationInd, err := locStore.SourceLocations.FindClosestLocation(time) + if err != nil { + return Corrdinates{}, err + } + closestLocation := locStore.SourceLocations.Locations[closestLocationInd] + + // Check the time difference + timeDiff := time.Sub(closestLocation.Time) + switch { + case timeDiff <= locStore.LowTimeDiffThreshold: + // If the time difference is low, return the location + return closestLocation.Corrdinates, nil + case timeDiff <= locStore.MediumTimeDiffThreshold: + // If the time difference is medium, attempt linear interpolation + // Find the previous location + return closestLocation.Corrdinates, ErrTimeDiffMedium + case timeDiff <= locStore.HighTimeDiffThreshold: + // If the time difference is high, return an error + return closestLocation.Corrdinates, fmt.Errorf("Time difference too high: %v", timeDiff) + } + + // Return the location + return Corrdinates{}, ErrNoLocation +} + +func interpolation(loc1, loc2 LocationRecord, time time.Time) Corrdinates { + // Calculate the ratio of the time difference + timeRatio := timeRatio(loc1.Time, loc2.Time, time) + // Normalized ratio + loc1LatitudeE2, loc1LongitudeE2 := loc1.Corrdinates.GetE2Coord() + loc2LatitudeE2, loc2LongitudeE2 := loc2.Corrdinates.GetE2Coord() + + p1 := geo.NewPoint(loc1LatitudeE2, loc1LongitudeE2) + p2 := geo.NewPoint(loc2LatitudeE2, loc2LongitudeE2) + + bearing := p1.BearingTo(p2) + distance := p1.GreatCircleDistance(p2) + + p3 := p1.PointAtDistanceAndBearing(distance*timeRatio, bearing) + + c, err := NewCorrdinatesE2(p3.Lat(), p3.Lng()) + if err != nil { + log.Fatalf("Unable to create coordinates: %v", err) + } + + return c +} + +func timeRatio(time1, time2, time time.Time) float64 { + return time.Sub(time1).Seconds() / time2.Sub(time1).Seconds() +} diff --git a/google_location_data/lib/lib_test.go b/google_location_data/lib/lib_test.go index 8432c59..c0c2ca0 100644 --- a/google_location_data/lib/lib_test.go +++ b/google_location_data/lib/lib_test.go @@ -1,8 +1,7 @@ package lib import ( - "os" - "path/filepath" + "math" "testing" ) @@ -17,47 +16,12 @@ const ( }` ) -var SIMPLE_TEST_DATA_SOURCE_LOCATION = SourceLocationData{ - Locations: []SourceLocationRecord{ - {LatitudeE7: 1234567, LongitudeE7: 2345678, TimeStamp: "2021-01-01T12:00:00Z"}, - {LatitudeE7: 2345678, LongitudeE7: 3456789, TimeStamp: "2021-01-02T12:00:00Z"}, - {LatitudeE7: 3456789, LongitudeE7: 4567890, TimeStamp: "2021-01-03T12:00:00Z"}, - }, -} - -func TestImportSourceLocationData(t *testing.T) { - // Create a test data file - testDataFolder := filepath.Join(TEST_DATA_FOLDER, "TestImportSourceLocationData") - testSourceData := filepath.Join(testDataFolder, "data.json") - err := os.MkdirAll(testDataFolder, 0o755) - if err != nil { - t.Fatalf("Error creating test data folder: %v", err) - } - defer os.RemoveAll(TEST_DATA_FOLDER) - - err = os.WriteFile(testSourceData, []byte(SIMPLE_TEST_DATA_STRING), 0o644) - if err != nil { - t.Fatalf("Error creating test data file: %v", err) - } - - // Test the ImportSourceLocationData function - sourceData := ImportSourceLocationData(testSourceData) - locations := sourceData.Locations - if len(locations) != 3 { - t.Errorf("Expected 3 records, got %d", len(locations)) - } - - // Check the values of the records - if locations[0].LatitudeE7 != 1234567 { - t.Errorf("Expected latitude 1234567, got %f", locations[0].LatitudeE7) - } - if locations[1].LongitudeE7 != 3456789 { - t.Errorf("Expected longitude 3456789, got %f", locations[0].LongitudeE7) - } - if locations[2].TimeStamp != "2021-01-03T12:00:00Z" { - t.Errorf("Expected timestamp 2021-01-03T12:00:00Z, got %s", locations[0].TimeStamp) - } +var LOCATIONS = []LocationRecord{ + {Corrdinates{LatitudeE7: 1234567, LongitudeE7: 2345678}, ParseTime("2021-01-01T12:00:00Z")}, + {Corrdinates{LatitudeE7: 2345678, LongitudeE7: 3456789}, ParseTime("2021-01-02T12:00:00Z")}, + {Corrdinates{LatitudeE7: 3456789, LongitudeE7: 4567890}, ParseTime("2021-01-03T12:00:00Z")}, } +var SIMPLE_TEST_DATA_SOURCE_LOCATION = SourceLocations{Locations: LOCATIONS} func TestSortByTime(t *testing.T) { sourceData := SIMPLE_TEST_DATA_SOURCE_LOCATION @@ -67,52 +31,37 @@ func TestSortByTime(t *testing.T) { locations := sourceData.Locations // Check the order of the records - if locations[0].TimeStamp != "2021-01-01T12:00:00Z" { - t.Errorf("Expected first record timestamp 2021-01-01T12:00:00Z, got %s", locations[0].TimeStamp) + if locations[0].Time != ParseTime("2021-01-01T12:00:00Z") { + t.Errorf("Expected first record timestamp 2021-01-01T12:00:00Z, got %s", locations[0].Time) } - if locations[1].TimeStamp != "2021-01-02T12:00:00Z" { - t.Errorf("Expected second record timestamp 2021-01-02T12:00:00Z, got %s", locations[1].TimeStamp) + if locations[1].Time != ParseTime("2021-01-02T12:00:00Z") { + t.Errorf("Expected second record timestamp 2021-01-02T12:00:00Z, got %s", locations[1].Time) } - if locations[2].TimeStamp != "2021-01-03T12:00:00Z" { - t.Errorf("Expected third record timestamp 2021-01-03T12:00:00Z, got %s", locations[2].TimeStamp) - } -} - -func TestByTimeComp(t *testing.T) { - var byTime ByTime - byTime = SIMPLE_TEST_DATA_SOURCE_LOCATION.Locations - - if byTime.Comp(0, 1) != -1 { - t.Errorf("Expected -1, got %d", byTime.Comp(0, 1)) - } - if byTime.Comp(1, 0) != 1 { - t.Errorf("Expected 1, got %d", byTime.Comp(1, 0)) - } - if byTime.Comp(0, 0) != 0 { - t.Errorf("Expected 0, got %d", byTime.Comp(0, 0)) + if locations[2].Time != ParseTime("2021-01-03T12:00:00Z") { + t.Errorf("Expected third record timestamp 2021-01-03T12:00:00Z, got %s", locations[2].Time) } } func TestGetLocation(t *testing.T) { // Setup sourceData := SIMPLE_TEST_DATA_SOURCE_LOCATION - sourceData.Locations = append(sourceData.Locations, SourceLocationRecord{ - LatitudeE7: 4567890, LongitudeE7: 5678901, TimeStamp: "2021-01-04T12:00:00Z", + sourceData.Locations = append(sourceData.Locations, LocationRecord{ + Corrdinates{LatitudeE7: 4567890, LongitudeE7: 5678901}, ParseTime("2021-01-04T12:00:00Z"), }) sourceData.SortByTime() // Test the GetLocation function with exact match { - timeStamp := "2021-01-02T12:00:00Z" - locationInd, err := sourceData.FindClosestLocation(ParseTime(timeStamp)) + timeStamp := ParseTime("2021-01-02T12:00:00Z") + locationInd, err := sourceData.FindClosestLocation(timeStamp) if err != nil { t.Errorf("Error getting location: %v", err) } location := sourceData.Locations[locationInd] // Check the values of the records - if location.TimeStamp != timeStamp { - t.Errorf("Expected location before timestamp %s, got %s", timeStamp, location.TimeStamp) + if location.Time != timeStamp { + t.Errorf("Expected location before timestamp %s, got %s", timeStamp, location.Time) } } // Test the GetLocation function with in-between time @@ -123,31 +72,122 @@ func TestGetLocation(t *testing.T) { t.Errorf("Error getting location: %v", err) } locationBefore := sourceData.Locations[locationInd] - if locationBefore.TimeStamp != "2021-01-02T12:00:00Z" { - t.Errorf("Expected location before timestamp 2021-01-02T12:00:00Z, got %s", locationBefore.TimeStamp) + if locationBefore.Time != ParseTime("2021-01-02T12:00:00Z") { + t.Errorf("Expected location before timestamp 2021-01-02T12:00:00Z, got %s", locationBefore.Time) } } // Testing limits { - timestampFarAfter := "2022-01-01T12:00:00Z" - beforeInd, err := sourceData.FindClosestLocation(ParseTime(timestampFarAfter)) + timestampFarAfter := ParseTime("2022-01-01T12:00:00Z") + beforeInd, err := sourceData.FindClosestLocation(timestampFarAfter) before := sourceData.Locations[beforeInd] if err != nil { t.Errorf("Error getting location: %v", err) } - if before.TimeStamp != "2021-01-04T12:00:00Z" { - t.Errorf("Expected location before timestamp 2021-01-04T12:00:00Z, got %s", before.TimeStamp) + if before.Time != ParseTime("2021-01-04T12:00:00Z") { + t.Errorf("Expected location before timestamp 2021-01-04T12:00:00Z, got %s", before.Time) } - timestampFarBefore := "2020-01-01T12:00:00Z" - afterInd, err := sourceData.FindClosestLocation(ParseTime(timestampFarBefore)) + timestampFarBefore := ParseTime("2020-01-01T12:00:00Z") + afterInd, err := sourceData.FindClosestLocation(timestampFarBefore) after := sourceData.Locations[afterInd] if err != nil { t.Errorf("Error getting location: %v", err) } - if after.TimeStamp != sourceData.Locations[0].TimeStamp { - t.Errorf("Expected location after timestamp %s, got %s", sourceData.Locations[0].TimeStamp, after.TimeStamp) + if after.Time != sourceData.Locations[0].Time { + t.Errorf("Expected location after timestamp %s, got %s", sourceData.Locations[0].Time, after.Time) + } + } +} + +func TestInterpolation(t *testing.T) { + // Setup + var ( + diff = 1 + locRecord1 = LocationRecord{ + Corrdinates{ + LatitudeE7: 1234567, + LongitudeE7: 2345678, + }, + ParseTime("2021-01-01T12:00:00Z"), + } + locRecord2 = LocationRecord{ + Corrdinates{ + LatitudeE7: locRecord1.Corrdinates.LatitudeE7 + diff, + LongitudeE7: locRecord1.Corrdinates.LongitudeE7 + diff, + }, + ParseTime("2021-01-02T12:00:00Z"), } + ) + + // In the middle + calc_middle_1_2 := interpolation( + locRecord1, locRecord2, locRecord1.Time.Add(locRecord2.Time.Sub(locRecord1.Time)/2)) + + // Check the values of the records + expectedLatitude := locRecord1.Corrdinates.LatitudeE7 + diff/2 + if calc_middle_1_2.LatitudeE7 != expectedLatitude { + t.Errorf("Expected latitude %d, got %d", expectedLatitude, calc_middle_1_2.LatitudeE7) + } + expectedLongitude := locRecord1.Corrdinates.LongitudeE7 + diff/2 + if calc_middle_1_2.LongitudeE7 != expectedLongitude { + t.Errorf("Expected longitude %d, got %d", expectedLongitude, calc_middle_1_2.LongitudeE7) + } +} + +func TestTimeRatio(t *testing.T) { + // Setup + var ( + time1 = ParseTime("2021-01-01T12:00:00Z") + time2 = ParseTime("2021-01-02T12:00:00Z") + time3 = ParseTime("2021-01-03T12:00:00Z") + ) + + // In the middle + ratio_middle_1_2 := timeRatio(time1, time2, time1.Add(time2.Sub(time1)/2)) + ratio_middle_1_3 := timeRatio(time1, time3, time1.Add(time3.Sub(time1)/2)) + + // Check the values of the records + if ratio_middle_1_2 != 0.5 { + t.Errorf("Expected ratio 0.5, got %f", ratio_middle_1_2) + } + if ratio_middle_1_3 != 0.5 { + t.Errorf("Expected ratio 0.5, got %f", ratio_middle_1_3) + } + // 3rd + ratio_3rd_1_2 := timeRatio(time1, time2, time1.Add(time2.Sub(time1)/3)) + ratio_3rd_1_3 := timeRatio(time1, time3, time1.Add(time3.Sub(time1)/3)) + + // Check the values of the records + if math.Abs(ratio_3rd_1_2-0.3) < 1e-14 { + t.Errorf("Expected ratio 0.3, got %f", ratio_3rd_1_2) + } + if math.Abs(ratio_3rd_1_3-0.3) < 1e-14 { + t.Errorf("Expected ratio 0.3, got %f", ratio_3rd_1_3) + } + + // At the start + ratio_start_1_2 := timeRatio(time1, time2, time1) + ratio_start_1_3 := timeRatio(time1, time3, time1) + + // Check the values of the records + if ratio_start_1_2 != 0 { + t.Errorf("Expected ratio 0, got %f", ratio_start_1_2) + } + if ratio_start_1_3 != 0 { + t.Errorf("Expected ratio 0, got %f", ratio_start_1_3) + } + + // At the end + ratio_end_1_2 := timeRatio(time1, time2, time2) + ratio_end_1_3 := timeRatio(time1, time3, time3) + + // Check the values of the records + if ratio_end_1_2 != 1 { + t.Errorf("Expected ratio 1, got %f", ratio_end_1_2) + } + if ratio_end_1_3 != 1 { + t.Errorf("Expected ratio 1, got %f", ratio_end_1_3) } } diff --git a/google_location_data/lib/sourceGoogle.go b/google_location_data/lib/sourceGoogle.go new file mode 100644 index 0000000..11ede2c --- /dev/null +++ b/google_location_data/lib/sourceGoogle.go @@ -0,0 +1,66 @@ +package lib + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" +) + +type GoogleTimelineTakeout struct { + Locations []GoogleTimelineLocations `json:"locations"` +} + +type GoogleTimelineLocations struct { + LatitudeE7 int `json:"latitudeE7"` + LongitudeE7 int `json:"longitudeE7"` + Timestamp string `json:"timestampMs"` +} + +var ( + ErrUnableToOpenSourceDataFile = errors.New("Unable to open source data file") + ErrUnableToReadSourceDataFile = errors.New("Unable to read source data file") + ErrUnableToUnmarshalSourceDataFile = errors.New("Unable to unmarshal source data file") +) + +func NewGoogleTimelineLocationsFromFile(path string) (*GoogleTimelineLocations, error) { + // Read the file + jsonFile, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUnableToOpenSourceDataFile, err) + } + defer jsonFile.Close() + // Unmarshal the data into a struct + bytes, err := io.ReadAll(jsonFile) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUnableToReadSourceDataFile, err) + } + gogleTimeLineLocations := &GoogleTimelineLocations{} + if err := json.Unmarshal(bytes, &GoogleTimelineLocations{}); err != nil { + return nil, fmt.Errorf("%w: %v", ErrUnableToUnmarshalSourceDataFile, err) + } + // Convert the data into a SourceLocations struct + return gogleTimeLineLocations, nil +} + +func (g *GoogleTimelineTakeout) ToLocationRecords() (*SourceLocations, error) { + SourceLocations := SourceLocations{} + + locations := make([]LocationRecord, len(g.Locations)) + + for i, loc := range g.Locations { + c, err := NewCorrdinatesE7(loc.LatitudeE7, loc.LongitudeE7) + if err != nil { + return nil, fmt.Errorf("Unable to create coordinates: %v", err) + } + + locations[i] = LocationRecord{ + Corrdinates: c, + Time: ParseTime(loc.Timestamp), + } + } + + SourceLocations.Locations = locations + return &SourceLocations, nil +} diff --git a/google_location_data/lib/sourceGoogle_test.go b/google_location_data/lib/sourceGoogle_test.go new file mode 100644 index 0000000..f727bc4 --- /dev/null +++ b/google_location_data/lib/sourceGoogle_test.go @@ -0,0 +1,32 @@ +package lib + +import "testing" + +func TestToLocationRecords(t *testing.T) { + // Create a GoogleTimelineTakeout struct with some test data + googleTimelineTakeout := GoogleTimelineTakeout{ + Locations: []GoogleTimelineLocations{ + { + LatitudeE7: 1234567, + LongitudeE7: 1234567, + Timestamp: "123456789", + }, + { + LatitudeE7: 1234567, + LongitudeE7: 1234567, + Timestamp: "123456789", + }, + }, + } + + // Call the ToLocationRecords method + sourceLocations, err := googleTimelineTakeout.ToLocationRecords() + if err != nil { + t.Errorf("Error calling ToLocationRecords: %v", err) + } + + // Check that the sourceLocations struct has the correct number of locations + if len(sourceLocations.Locations) != 2 { + t.Errorf("Expected 2 locations, got %d", len(sourceLocations.Locations)) + } +} From 0a66183c1e2f5d1fad227edb2a949520959488b6 Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sat, 17 Aug 2024 16:47:38 +0200 Subject: [PATCH 06/15] fixed tests for google ToLocationRecords --- google_location_data/lib/sourceGoogle_test.go | 81 +++++++++++++++++-- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/google_location_data/lib/sourceGoogle_test.go b/google_location_data/lib/sourceGoogle_test.go index f727bc4..36699a2 100644 --- a/google_location_data/lib/sourceGoogle_test.go +++ b/google_location_data/lib/sourceGoogle_test.go @@ -1,20 +1,24 @@ package lib -import "testing" +import ( + "errors" + "testing" + "time" +) func TestToLocationRecords(t *testing.T) { // Create a GoogleTimelineTakeout struct with some test data googleTimelineTakeout := GoogleTimelineTakeout{ Locations: []GoogleTimelineLocations{ { - LatitudeE7: 1234567, - LongitudeE7: 1234567, - Timestamp: "123456789", + LatitudeE7: 633954185, + LongitudeE7: 103719669, + Timestamp: "2014-04-22T12:15:05.138Z", }, { - LatitudeE7: 1234567, - LongitudeE7: 1234567, - Timestamp: "123456789", + LatitudeE7: 633954162, + LongitudeE7: 103720388, + Timestamp: "2014-04-22T12:16:05.138Z", }, }, } @@ -29,4 +33,67 @@ func TestToLocationRecords(t *testing.T) { if len(sourceLocations.Locations) != 2 { t.Errorf("Expected 2 locations, got %d", len(sourceLocations.Locations)) } + + // Check that the sourceLocations struct has the correct locations + if sourceLocations.Locations[0].Corrdinates.LatitudeE7 != 633954185 { + t.Errorf("Expected first location latitude 633954185, got %d", sourceLocations.Locations[0].Corrdinates.LatitudeE7) + } + if sourceLocations.Locations[0].Corrdinates.LongitudeE7 != 103719669 { + t.Errorf("Expected first location longitude 103719669, got %d", sourceLocations.Locations[0].Corrdinates.LongitudeE7) + } + if sourceLocations.Locations[1].Corrdinates.LatitudeE7 != 633954162 { + t.Errorf("Expected second location latitude 633954162, got %d", sourceLocations.Locations[1].Corrdinates.LatitudeE7) + } + if sourceLocations.Locations[1].Corrdinates.LongitudeE7 != 103720388 { + t.Errorf("Expected second location longitude 103720388, got %d", sourceLocations.Locations[1].Corrdinates.LongitudeE7) + } + + // Check that the sourceLocations struct has the correct timestamps + expectedTime1 := "2014-04-22 12:15:05.138 +0000 UTC" + expectedTime2 := "2014-04-22 12:16:05.138 +0000 UTC" + + if sourceLocations.Locations[0].Time.String() != expectedTime1 { + t.Errorf("Expected first location timestamp %s, got %s", expectedTime1, sourceLocations.Locations[0].Time) + } + if sourceLocations.Locations[1].Time.String() != expectedTime2 { + t.Errorf("Expected second location timestamp %s, got %s", expectedTime2, sourceLocations.Locations[1].Time) + } + sourceLocations.SortByTime() + if sourceLocations.Locations[1].Time.Before(sourceLocations.Locations[0].Time) { + t.Errorf("Expected first location timestamp to be before second location timestamp") + } + if sourceLocations.Locations[1].Time.Sub(sourceLocations.Locations[0].Time) == time.Hour*1 { + t.Errorf("Expected first location timestamp to be before second location timestamp") + } + + // Tesing with strange coordinates + latitudeOutOfRange := GoogleTimelineTakeout{ + Locations: []GoogleTimelineLocations{ + // Latitude out of range + { + LatitudeE7: 1000000000, + LongitudeE7: 103719669, + Timestamp: "2014-04-22T12:15:05.138Z", + }, + }, + } + longitudeOutOfRange := GoogleTimelineTakeout{ + Locations: []GoogleTimelineLocations{ + // Longitude out of range + { + LatitudeE7: 633954185, + LongitudeE7: -100000000, + Timestamp: "2014-04-22T12:15:05.138Z", + }, + }, + } + + _, err = latitudeOutOfRange.ToLocationRecords() + if errors.Is(err, ErrLatitudeOutOfRange) { + t.Errorf("Expected ErrLatitudeOutOfRange error, got: %v", err) + } + _, err = longitudeOutOfRange.ToLocationRecords() + if errors.Is(err, ErrLongitudeOutOfRange) { + t.Errorf("Expected ErrLongitudeOutOfRange error, got: %v", err) + } } From 601e43e7bfaf0ef9beb5f7e391edada973974dd2 Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sun, 25 Aug 2024 15:10:06 +0200 Subject: [PATCH 07/15] refactored coordinates and location --- google_location_data/lib/coordinates.go | 43 +++++ google_location_data/lib/lib.go | 177 ++---------------- google_location_data/lib/lib_test.go | 71 +++---- google_location_data/lib/location.go | 110 +++++++++++ google_location_data/lib/sourceGoogle.go | 27 ++- google_location_data/lib/sourceGoogle_test.go | 53 +----- libs/time/lib.go | 18 ++ 7 files changed, 249 insertions(+), 250 deletions(-) create mode 100644 google_location_data/lib/coordinates.go create mode 100644 google_location_data/lib/location.go create mode 100644 libs/time/lib.go diff --git a/google_location_data/lib/coordinates.go b/google_location_data/lib/coordinates.go new file mode 100644 index 0000000..50f4d42 --- /dev/null +++ b/google_location_data/lib/coordinates.go @@ -0,0 +1,43 @@ +package lib + +import ( + geo "github.com/kellydunn/golang-geo" +) + +type Corrdinates struct { + geo.Point +} + +func NewCorrdinatesE2(lat, long float64) Corrdinates { + return Corrdinates{ + Point: *geo.NewPoint(lat, long), + } +} + +func NewCorrdinatesE7(lat, long int) Corrdinates { + return Corrdinates{ + Point: *geo.NewPoint(float64(lat)/1e7, float64(long)/1e7), + } +} + +func NewCoordinatesFromGeopoint(point geo.Point) Corrdinates { + return Corrdinates{ + Point: point, + } +} + +func (coordinate *Corrdinates) GetE2Coord() (lat float64, long float64) { + return coordinate.Lat(), coordinate.Lng() +} + +func (coordinate *Corrdinates) GetE7Coord() (lat int, long int) { + return int(coordinate.Lat() * 1e7), int(coordinate.Lng() * 1e7) +} + +func (coordinate *Corrdinates) LatE7() int { + return int(coordinate.Point.Lat() * 1e7) +} + +func (coordinate *Corrdinates) LngE7() int { + return int(coordinate.Point.Lng() * 1e7) +} diff --git a/google_location_data/lib/lib.go b/google_location_data/lib/lib.go index 7d2edba..fcba1ac 100644 --- a/google_location_data/lib/lib.go +++ b/google_location_data/lib/lib.go @@ -1,141 +1,12 @@ package lib import ( - "fmt" - "log" - "sort" + "errors" "time" geo "github.com/kellydunn/golang-geo" ) -var RFC3339_LAYOUT string = "2006-01-02T15:04:05Z07:00" - -type LocationStore struct { - SourceLocations SourceLocations - - // Time difference thresholds - // Low time difference threshold - LowTimeDiffThreshold time.Duration - // Medium time difference threshold - MediumTimeDiffThreshold time.Duration - // High time difference threshold - HighTimeDiffThreshold time.Duration -} - -const ( - LOW_TIME_DIFF_THRESHOLD = 30 * time.Minute - MEDIUM_TIME_DIFF_THRESHOLD = 2 * time.Hour - HIGH_TIME_DIFF_THRESHOLD = 12 * time.Hour -) - -type SourceLocations struct { - Locations []LocationRecord `json:"locations"` -} - -// This function assumes Locations is sorted by time -func (sourceData *SourceLocations) FindClosestLocation(time time.Time) (ind int, err error) { - var ( - locationBeforeInd int - locationAfterInd int - ) - - locationAfterInd = sort.Search(len(sourceData.Locations), func(i int) bool { - return sourceData.Locations[i].Time.After(time) - }) - log.Default().Printf("locationAfterInd: %d", locationAfterInd) - // Handling edge cases, max and min - if locationAfterInd == len(sourceData.Locations) { - return locationAfterInd - 1, nil - } - if locationAfterInd == 0 { - return locationAfterInd, nil - } - - locationBeforeInd = locationAfterInd - 1 - afterTime := sourceData.Locations[locationAfterInd].Time - beforeTime := sourceData.Locations[locationBeforeInd].Time - - if time.Sub(afterTime).Abs() <= time.Sub(beforeTime).Abs() { - return locationAfterInd, nil - } else { - return locationBeforeInd, nil - } -} - -func (sourceData *SourceLocations) SortByTime() { - // Sort the data by time - sort.Sort(ByTime(sourceData.Locations)) - // return sourceData.Locations -} - -type ByTime []LocationRecord - -func (bt ByTime) Less(i, j int) bool { - return bt[i].Time.Before(bt[j].Time) -} - -func (bt ByTime) Swap(i, j int) { - bt[i], bt[j] = bt[j], bt[i] -} - -func (bt ByTime) Len() int { - return len(bt) -} - -type LocationRecord struct { - Corrdinates Corrdinates `json:"coordinates"` - Time time.Time `json:"timestamp"` -} - -func ParseTime(timeStr string) time.Time { - t, err := time.Parse(RFC3339_LAYOUT, timeStr) - if err != nil { - log.Fatalf("Unable to parse timestamp: %v", err) - } - return t -} - -type Corrdinates struct { - LatitudeE7 int - LongitudeE7 int -} - -var ( - ErrLatitudeOutOfRange = fmt.Errorf("Latitude out of range") - ErrLongitudeOutOfRange = fmt.Errorf("Longitude out of range") -) - -func NewCorrdinatesE7(lat, long int) (Corrdinates, error) { - if lat < -90e7 || lat > 90e7 { - return Corrdinates{}, fmt.Errorf("%w: %d", ErrLatitudeOutOfRange, lat) - } - if long < -180e7 || long > 180e7 { - return Corrdinates{}, fmt.Errorf("%w: %d", ErrLongitudeOutOfRange, long) - } - return Corrdinates{ - LatitudeE7: lat, - LongitudeE7: long, - }, nil -} - -func NewCorrdinatesE2(lat, long float64) (Corrdinates, error) { - if lat < -90 || lat > 90 { - return Corrdinates{}, fmt.Errorf("%w: %f", ErrLatitudeOutOfRange, lat) - } - if long < -180 || long > 180 { - return Corrdinates{}, fmt.Errorf("%w: %f", ErrLongitudeOutOfRange, long) - } - return Corrdinates{ - LatitudeE7: int(lat * 1e7), - LongitudeE7: int(long * 1e7), - }, nil -} - -func (coordinate *Corrdinates) GetE2Coord() (lat float64, long float64) { - return float64(coordinate.LatitudeE7) / 1e7, float64(coordinate.LongitudeE7) / 1e7 -} - /* This function will have to do some assumtions when the time between location stamps is too large. @@ -146,38 +17,17 @@ It is implemented with 3 types of assumptions: - If the time is large we will return an error, and assume the user will have to provide the data themselves. */ -var ( - ErrTimeDiffTooHigh = fmt.Errorf("Time difference too high") - ErrTimeDiffMedium = fmt.Errorf("Time difference medium") - ErrNoLocation = fmt.Errorf("No location found") +const ( + LOW_TIME_DIFF_THRESHOLD = 30 * time.Minute + MEDIUM_TIME_DIFF_THRESHOLD = 2 * time.Hour + HIGH_TIME_DIFF_THRESHOLD = 12 * time.Hour ) -func (locStore *LocationStore) GetLocationByTime(time time.Time) (Corrdinates, error) { - // Find the closest location to the given time - closestLocationInd, err := locStore.SourceLocations.FindClosestLocation(time) - if err != nil { - return Corrdinates{}, err - } - closestLocation := locStore.SourceLocations.Locations[closestLocationInd] - - // Check the time difference - timeDiff := time.Sub(closestLocation.Time) - switch { - case timeDiff <= locStore.LowTimeDiffThreshold: - // If the time difference is low, return the location - return closestLocation.Corrdinates, nil - case timeDiff <= locStore.MediumTimeDiffThreshold: - // If the time difference is medium, attempt linear interpolation - // Find the previous location - return closestLocation.Corrdinates, ErrTimeDiffMedium - case timeDiff <= locStore.HighTimeDiffThreshold: - // If the time difference is high, return an error - return closestLocation.Corrdinates, fmt.Errorf("Time difference too high: %v", timeDiff) - } - - // Return the location - return Corrdinates{}, ErrNoLocation -} +var ( + ErrTimeDiffTooHigh = errors.New("Time difference too high") + ErrTimeDiffMedium = errors.New("Time difference medium") + ErrNoLocation = errors.New("No location found") +) func interpolation(loc1, loc2 LocationRecord, time time.Time) Corrdinates { // Calculate the ratio of the time difference @@ -194,12 +44,7 @@ func interpolation(loc1, loc2 LocationRecord, time time.Time) Corrdinates { p3 := p1.PointAtDistanceAndBearing(distance*timeRatio, bearing) - c, err := NewCorrdinatesE2(p3.Lat(), p3.Lng()) - if err != nil { - log.Fatalf("Unable to create coordinates: %v", err) - } - - return c + return NewCoordinatesFromGeopoint(*p3) } func timeRatio(time1, time2, time time.Time) float64 { diff --git a/google_location_data/lib/lib_test.go b/google_location_data/lib/lib_test.go index c0c2ca0..39a4b36 100644 --- a/google_location_data/lib/lib_test.go +++ b/google_location_data/lib/lib_test.go @@ -3,6 +3,8 @@ package lib import ( "math" "testing" + + toolsTime "github.com/sander-skjulsvik/tools/libs/time" ) const ( @@ -17,11 +19,15 @@ const ( ) var LOCATIONS = []LocationRecord{ - {Corrdinates{LatitudeE7: 1234567, LongitudeE7: 2345678}, ParseTime("2021-01-01T12:00:00Z")}, - {Corrdinates{LatitudeE7: 2345678, LongitudeE7: 3456789}, ParseTime("2021-01-02T12:00:00Z")}, - {Corrdinates{LatitudeE7: 3456789, LongitudeE7: 4567890}, ParseTime("2021-01-03T12:00:00Z")}, + {NewCorrdinatesE7(1234567, 2345678), *toolsTime.ParseTimeNoErrorRFC3339("2021-01-01T12:00:00Z")}, + {NewCorrdinatesE7(2345678, 3456789), *toolsTime.ParseTimeNoErrorRFC3339("2021-01-02T12:00:00Z")}, + {NewCorrdinatesE7(3456789, 4567890), *toolsTime.ParseTimeNoErrorRFC3339("2021-01-03T12:00:00Z")}, } -var SIMPLE_TEST_DATA_SOURCE_LOCATION = SourceLocations{Locations: LOCATIONS} + +var ( + LOCATIONS2 = []LocationRecord{} + SIMPLE_TEST_DATA_SOURCE_LOCATION = SourceLocations{Locations: LOCATIONS} +) func TestSortByTime(t *testing.T) { sourceData := SIMPLE_TEST_DATA_SOURCE_LOCATION @@ -31,13 +37,13 @@ func TestSortByTime(t *testing.T) { locations := sourceData.Locations // Check the order of the records - if locations[0].Time != ParseTime("2021-01-01T12:00:00Z") { + if locations[0].Time != *toolsTime.ParseTimeNoErrorRFC3339("2021-01-01T12:00:00Z") { t.Errorf("Expected first record timestamp 2021-01-01T12:00:00Z, got %s", locations[0].Time) } - if locations[1].Time != ParseTime("2021-01-02T12:00:00Z") { + if locations[1].Time != *toolsTime.ParseTimeNoErrorRFC3339("2021-01-02T12:00:00Z") { t.Errorf("Expected second record timestamp 2021-01-02T12:00:00Z, got %s", locations[1].Time) } - if locations[2].Time != ParseTime("2021-01-03T12:00:00Z") { + if locations[2].Time != *toolsTime.ParseTimeNoErrorRFC3339("2021-01-03T12:00:00Z") { t.Errorf("Expected third record timestamp 2021-01-03T12:00:00Z, got %s", locations[2].Time) } } @@ -46,13 +52,13 @@ func TestGetLocation(t *testing.T) { // Setup sourceData := SIMPLE_TEST_DATA_SOURCE_LOCATION sourceData.Locations = append(sourceData.Locations, LocationRecord{ - Corrdinates{LatitudeE7: 4567890, LongitudeE7: 5678901}, ParseTime("2021-01-04T12:00:00Z"), + NewCorrdinatesE7(4567890, 5678901), *toolsTime.ParseTimeNoErrorRFC3339("2021-01-04T12:00:00Z"), }) sourceData.SortByTime() // Test the GetLocation function with exact match { - timeStamp := ParseTime("2021-01-02T12:00:00Z") + timeStamp := *toolsTime.ParseTimeNoErrorRFC3339("2021-01-02T12:00:00Z") locationInd, err := sourceData.FindClosestLocation(timeStamp) if err != nil { t.Errorf("Error getting location: %v", err) @@ -67,29 +73,29 @@ func TestGetLocation(t *testing.T) { // Test the GetLocation function with in-between time { timeStamp := "2021-01-02T18:00:00Z" - locationInd, err := sourceData.FindClosestLocation(ParseTime(timeStamp)) + locationInd, err := sourceData.FindClosestLocation(*toolsTime.ParseTimeNoErrorRFC3339(timeStamp)) if err != nil { t.Errorf("Error getting location: %v", err) } locationBefore := sourceData.Locations[locationInd] - if locationBefore.Time != ParseTime("2021-01-02T12:00:00Z") { + if locationBefore.Time != *toolsTime.ParseTimeNoErrorRFC3339("2021-01-02T12:00:00Z") { t.Errorf("Expected location before timestamp 2021-01-02T12:00:00Z, got %s", locationBefore.Time) } } // Testing limits { - timestampFarAfter := ParseTime("2022-01-01T12:00:00Z") - beforeInd, err := sourceData.FindClosestLocation(timestampFarAfter) + timestampFarAfter := toolsTime.ParseTimeNoErrorRFC3339("2022-01-01T12:00:00Z") + beforeInd, err := sourceData.FindClosestLocation(*timestampFarAfter) before := sourceData.Locations[beforeInd] if err != nil { t.Errorf("Error getting location: %v", err) } - if before.Time != ParseTime("2021-01-04T12:00:00Z") { + if before.Time != *toolsTime.ParseTimeNoErrorRFC3339("2021-01-04T12:00:00Z") { t.Errorf("Expected location before timestamp 2021-01-04T12:00:00Z, got %s", before.Time) } - timestampFarBefore := ParseTime("2020-01-01T12:00:00Z") + timestampFarBefore := *toolsTime.ParseTimeNoErrorRFC3339("2020-01-01T12:00:00Z") afterInd, err := sourceData.FindClosestLocation(timestampFarBefore) after := sourceData.Locations[afterInd] if err != nil { @@ -106,18 +112,15 @@ func TestInterpolation(t *testing.T) { var ( diff = 1 locRecord1 = LocationRecord{ - Corrdinates{ - LatitudeE7: 1234567, - LongitudeE7: 2345678, - }, - ParseTime("2021-01-01T12:00:00Z"), + NewCorrdinatesE7(1234567, 2345678), + *toolsTime.ParseTimeNoErrorRFC3339("2021-01-01T12:00:00Z"), } locRecord2 = LocationRecord{ - Corrdinates{ - LatitudeE7: locRecord1.Corrdinates.LatitudeE7 + diff, - LongitudeE7: locRecord1.Corrdinates.LongitudeE7 + diff, - }, - ParseTime("2021-01-02T12:00:00Z"), + NewCorrdinatesE7( + locRecord1.Corrdinates.LatE7()+diff, + locRecord1.Corrdinates.LngE7()+diff, + ), + *toolsTime.ParseTimeNoErrorRFC3339("2021-01-02T12:00:00Z"), } ) @@ -126,22 +129,22 @@ func TestInterpolation(t *testing.T) { locRecord1, locRecord2, locRecord1.Time.Add(locRecord2.Time.Sub(locRecord1.Time)/2)) // Check the values of the records - expectedLatitude := locRecord1.Corrdinates.LatitudeE7 + diff/2 - if calc_middle_1_2.LatitudeE7 != expectedLatitude { - t.Errorf("Expected latitude %d, got %d", expectedLatitude, calc_middle_1_2.LatitudeE7) + expectedLatitude := locRecord1.Corrdinates.LatE7() + diff/2 + if calc_middle_1_2.LatE7() != expectedLatitude { + t.Errorf("Expected latitude %d, got %d", expectedLatitude, calc_middle_1_2.LatE7()) } - expectedLongitude := locRecord1.Corrdinates.LongitudeE7 + diff/2 - if calc_middle_1_2.LongitudeE7 != expectedLongitude { - t.Errorf("Expected longitude %d, got %d", expectedLongitude, calc_middle_1_2.LongitudeE7) + expectedLongitude := locRecord1.Corrdinates.LngE7() + diff/2 + if calc_middle_1_2.LngE7() != expectedLongitude { + t.Errorf("Expected longitude %d, got %d", expectedLongitude, calc_middle_1_2.LngE7()) } } func TestTimeRatio(t *testing.T) { // Setup var ( - time1 = ParseTime("2021-01-01T12:00:00Z") - time2 = ParseTime("2021-01-02T12:00:00Z") - time3 = ParseTime("2021-01-03T12:00:00Z") + time1 = *toolsTime.ParseTimeNoErrorRFC3339("2021-01-01T12:00:00Z") + time2 = *toolsTime.ParseTimeNoErrorRFC3339("2021-01-02T12:00:00Z") + time3 = *toolsTime.ParseTimeNoErrorRFC3339("2021-01-03T12:00:00Z") ) // In the middle diff --git a/google_location_data/lib/location.go b/google_location_data/lib/location.go new file mode 100644 index 0000000..09f0bdf --- /dev/null +++ b/google_location_data/lib/location.go @@ -0,0 +1,110 @@ +package lib + +import ( + "errors" + "fmt" + "log" + "sort" + "time" +) + +type LocationStore struct { + SourceLocations SourceLocations + + // Time difference thresholds + // Low time difference threshold + LowTimeDiffThreshold time.Duration + // Medium time difference threshold + MediumTimeDiffThreshold time.Duration + // High time difference threshold + HighTimeDiffThreshold time.Duration +} + +func (locStore *LocationStore) GetLocationByTime(time time.Time) (Corrdinates, error) { + // Find the closest location to the given time + closestLocationInd, err := locStore.SourceLocations.FindClosestLocation(time) + if err != nil { + return Corrdinates{}, err + } + closestLocation := locStore.SourceLocations.Locations[closestLocationInd] + + // Check the time difference + timeDiff := time.Sub(closestLocation.Time) + switch { + case timeDiff <= locStore.LowTimeDiffThreshold: + // If the time difference is low, return the location + return closestLocation.Corrdinates, nil + case timeDiff <= locStore.MediumTimeDiffThreshold: + // If the time difference is medium, attempt linear interpolation + // Find the previous location + return closestLocation.Corrdinates, ErrTimeDiffMedium + case timeDiff <= locStore.HighTimeDiffThreshold: + // If the time difference is high, return an error + return closestLocation.Corrdinates, errors.Join( + ErrTimeDiffTooHigh, + fmt.Errorf("Diff: %s", timeDiff), + ) + } + + // Return the location + return Corrdinates{}, ErrNoLocation +} + +type SourceLocations struct { + Locations []LocationRecord `json:"locations"` +} + +// This function assumes Locations is sorted by time +func (sourceData *SourceLocations) FindClosestLocation(time time.Time) (ind int, err error) { + var ( + locationBeforeInd int + locationAfterInd int + ) + + locationAfterInd = sort.Search(len(sourceData.Locations), func(i int) bool { + return sourceData.Locations[i].Time.After(time) + }) + log.Default().Printf("locationAfterInd: %d", locationAfterInd) + // Handling edge cases, max and min + if locationAfterInd == len(sourceData.Locations) { + return locationAfterInd - 1, nil + } + if locationAfterInd == 0 { + return locationAfterInd, nil + } + + locationBeforeInd = locationAfterInd - 1 + afterTime := sourceData.Locations[locationAfterInd].Time + beforeTime := sourceData.Locations[locationBeforeInd].Time + + if time.Sub(afterTime).Abs() <= time.Sub(beforeTime).Abs() { + return locationAfterInd, nil + } else { + return locationBeforeInd, nil + } +} + +func (sourceData *SourceLocations) SortByTime() { + // Sort the data by time + sort.Sort(ByTime(sourceData.Locations)) + // return sourceData.Locations +} + +type ByTime []LocationRecord + +func (bt ByTime) Less(i, j int) bool { + return bt[i].Time.Before(bt[j].Time) +} + +func (bt ByTime) Swap(i, j int) { + bt[i], bt[j] = bt[j], bt[i] +} + +func (bt ByTime) Len() int { + return len(bt) +} + +type LocationRecord struct { + Corrdinates Corrdinates `json:"coordinates"` + Time time.Time `json:"timestamp"` +} diff --git a/google_location_data/lib/sourceGoogle.go b/google_location_data/lib/sourceGoogle.go index 11ede2c..42fa908 100644 --- a/google_location_data/lib/sourceGoogle.go +++ b/google_location_data/lib/sourceGoogle.go @@ -6,6 +6,9 @@ import ( "fmt" "io" "os" + "time" + + toolsTime "github.com/sander-skjulsvik/tools/libs/time" ) type GoogleTimelineTakeout struct { @@ -44,23 +47,35 @@ func NewGoogleTimelineLocationsFromFile(path string) (*GoogleTimelineLocations, return gogleTimeLineLocations, nil } -func (g *GoogleTimelineTakeout) ToLocationRecords() (*SourceLocations, error) { +var ErrUnableToCreateCoordinates = fmt.Errorf("Unable to create coordinates") + +func (g *GoogleTimelineTakeout) ToLocationRecords() *SourceLocations { SourceLocations := SourceLocations{} locations := make([]LocationRecord, len(g.Locations)) for i, loc := range g.Locations { - c, err := NewCorrdinatesE7(loc.LatitudeE7, loc.LongitudeE7) + c := NewCorrdinatesE7(loc.LatitudeE7, loc.LongitudeE7) + parsedTime, err := g.ParseTime(loc.Timestamp) if err != nil { - return nil, fmt.Errorf("Unable to create coordinates: %v", err) + fmt.Printf("Error parsing time for record: %v, err: %v\n", loc, err) + continue } - locations[i] = LocationRecord{ Corrdinates: c, - Time: ParseTime(loc.Timestamp), + Time: *parsedTime, } } SourceLocations.Locations = locations - return &SourceLocations, nil + return &SourceLocations +} + +func (g GoogleTimelineTakeout) ParseTime(timeStr string) (*time.Time, error) { + googleTimelineTimeLayout := toolsTime.RFC3339 + t, err := time.Parse(googleTimelineTimeLayout, timeStr) + if err != nil { + return nil, err + } + return &t, nil } diff --git a/google_location_data/lib/sourceGoogle_test.go b/google_location_data/lib/sourceGoogle_test.go index 36699a2..773f9d5 100644 --- a/google_location_data/lib/sourceGoogle_test.go +++ b/google_location_data/lib/sourceGoogle_test.go @@ -1,7 +1,6 @@ package lib import ( - "errors" "testing" "time" ) @@ -24,10 +23,7 @@ func TestToLocationRecords(t *testing.T) { } // Call the ToLocationRecords method - sourceLocations, err := googleTimelineTakeout.ToLocationRecords() - if err != nil { - t.Errorf("Error calling ToLocationRecords: %v", err) - } + sourceLocations := googleTimelineTakeout.ToLocationRecords() // Check that the sourceLocations struct has the correct number of locations if len(sourceLocations.Locations) != 2 { @@ -35,17 +31,17 @@ func TestToLocationRecords(t *testing.T) { } // Check that the sourceLocations struct has the correct locations - if sourceLocations.Locations[0].Corrdinates.LatitudeE7 != 633954185 { - t.Errorf("Expected first location latitude 633954185, got %d", sourceLocations.Locations[0].Corrdinates.LatitudeE7) + if sourceLocations.Locations[0].Corrdinates.LatE7() != 633954185 { + t.Errorf("Expected first location latitude 633954185, got %d", sourceLocations.Locations[0].Corrdinates.LatE7()) } - if sourceLocations.Locations[0].Corrdinates.LongitudeE7 != 103719669 { - t.Errorf("Expected first location longitude 103719669, got %d", sourceLocations.Locations[0].Corrdinates.LongitudeE7) + if sourceLocations.Locations[0].Corrdinates.LngE7() != 103719669 { + t.Errorf("Expected first location longitude 103719669, got %d", sourceLocations.Locations[0].Corrdinates.LngE7()) } - if sourceLocations.Locations[1].Corrdinates.LatitudeE7 != 633954162 { - t.Errorf("Expected second location latitude 633954162, got %d", sourceLocations.Locations[1].Corrdinates.LatitudeE7) + if sourceLocations.Locations[1].Corrdinates.LatE7() != 633954162 { + t.Errorf("Expected second location latitude 633954162, got %d", sourceLocations.Locations[1].Corrdinates.LatE7()) } - if sourceLocations.Locations[1].Corrdinates.LongitudeE7 != 103720388 { - t.Errorf("Expected second location longitude 103720388, got %d", sourceLocations.Locations[1].Corrdinates.LongitudeE7) + if sourceLocations.Locations[1].Corrdinates.LngE7() != 103720388 { + t.Errorf("Expected second location longitude 103720388, got %d", sourceLocations.Locations[1].Corrdinates.LngE7()) } // Check that the sourceLocations struct has the correct timestamps @@ -65,35 +61,4 @@ func TestToLocationRecords(t *testing.T) { if sourceLocations.Locations[1].Time.Sub(sourceLocations.Locations[0].Time) == time.Hour*1 { t.Errorf("Expected first location timestamp to be before second location timestamp") } - - // Tesing with strange coordinates - latitudeOutOfRange := GoogleTimelineTakeout{ - Locations: []GoogleTimelineLocations{ - // Latitude out of range - { - LatitudeE7: 1000000000, - LongitudeE7: 103719669, - Timestamp: "2014-04-22T12:15:05.138Z", - }, - }, - } - longitudeOutOfRange := GoogleTimelineTakeout{ - Locations: []GoogleTimelineLocations{ - // Longitude out of range - { - LatitudeE7: 633954185, - LongitudeE7: -100000000, - Timestamp: "2014-04-22T12:15:05.138Z", - }, - }, - } - - _, err = latitudeOutOfRange.ToLocationRecords() - if errors.Is(err, ErrLatitudeOutOfRange) { - t.Errorf("Expected ErrLatitudeOutOfRange error, got: %v", err) - } - _, err = longitudeOutOfRange.ToLocationRecords() - if errors.Is(err, ErrLongitudeOutOfRange) { - t.Errorf("Expected ErrLongitudeOutOfRange error, got: %v", err) - } } diff --git a/libs/time/lib.go b/libs/time/lib.go new file mode 100644 index 0000000..4cdcc0c --- /dev/null +++ b/libs/time/lib.go @@ -0,0 +1,18 @@ +package time + +import ( + "fmt" + "time" +) + +var RFC3339 string = "2006-01-02T15:04:05Z07:00" + +func ParseTimeNoErrorRFC3339(timeString string) *time.Time { + t, err := time.Parse(RFC3339, timeString) + if err != nil { + panic( + fmt.Sprintf("Error parsing time: %s, err: %v", timeString, err), + ) + } + return &t +} From 92d82d4f2cfa04e24e23381b5af06e7143deff19 Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sun, 25 Aug 2024 15:30:35 +0200 Subject: [PATCH 08/15] using errors.join --- google_location_data/lib/sourceGoogle.go | 41 +++++++++++++++--------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/google_location_data/lib/sourceGoogle.go b/google_location_data/lib/sourceGoogle.go index 42fa908..e698d47 100644 --- a/google_location_data/lib/sourceGoogle.go +++ b/google_location_data/lib/sourceGoogle.go @@ -11,44 +11,55 @@ import ( toolsTime "github.com/sander-skjulsvik/tools/libs/time" ) -type GoogleTimelineTakeout struct { - Locations []GoogleTimelineLocations `json:"locations"` -} - -type GoogleTimelineLocations struct { +type GoogleTimelineLocation struct { LatitudeE7 int `json:"latitudeE7"` LongitudeE7 int `json:"longitudeE7"` - Timestamp string `json:"timestampMs"` + Timestamp string `json:"timestamp"` +} + +type GoogleTimelineTakeout struct { + Locations []GoogleTimelineLocation `json:"locations"` } var ( ErrUnableToOpenSourceDataFile = errors.New("Unable to open source data file") ErrUnableToReadSourceDataFile = errors.New("Unable to read source data file") ErrUnableToUnmarshalSourceDataFile = errors.New("Unable to unmarshal source data file") + ErrUnableToCreateCoordinates = errors.New("Unable to create coordinates") ) -func NewGoogleTimelineLocationsFromFile(path string) (*GoogleTimelineLocations, error) { +func NewGoogleTimelineLocationsFromFile(path string) (*GoogleTimelineTakeout, error) { // Read the file jsonFile, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrUnableToOpenSourceDataFile, err) + return nil, errors.Join( + ErrUnableToOpenSourceDataFile, + fmt.Errorf("file: %s,", path), + err, + ) } defer jsonFile.Close() // Unmarshal the data into a struct bytes, err := io.ReadAll(jsonFile) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrUnableToReadSourceDataFile, err) + return nil, errors.Join( + ErrUnableToReadSourceDataFile, + fmt.Errorf("file: %s", path), + err, + ) } - gogleTimeLineLocations := &GoogleTimelineLocations{} - if err := json.Unmarshal(bytes, &GoogleTimelineLocations{}); err != nil { - return nil, fmt.Errorf("%w: %v", ErrUnableToUnmarshalSourceDataFile, err) + takeout := &GoogleTimelineTakeout{} + if err := json.Unmarshal(bytes, &GoogleTimelineTakeout{}); err != nil { + return nil, errors.Join( + ErrUnableToUnmarshalSourceDataFile, + fmt.Errorf("file: %s", path), + err, + ) } // Convert the data into a SourceLocations struct - return gogleTimeLineLocations, nil + return takeout, nil } -var ErrUnableToCreateCoordinates = fmt.Errorf("Unable to create coordinates") - func (g *GoogleTimelineTakeout) ToLocationRecords() *SourceLocations { SourceLocations := SourceLocations{} From 46e95dc207a8e561d9deea497da20805dee84a80 Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sun, 25 Aug 2024 15:38:08 +0200 Subject: [PATCH 09/15] tested TestNewGoogleTimelineLocationsFromFile, and fixed unmarshal issue --- google_location_data/lib/sourceGoogle.go | 2 +- google_location_data/lib/sourceGoogle_test.go | 36 ++++++++- .../test_data/test_google_location_data.json | 78 +++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 google_location_data/lib/test_data/test_google_location_data.json diff --git a/google_location_data/lib/sourceGoogle.go b/google_location_data/lib/sourceGoogle.go index e698d47..4bfb371 100644 --- a/google_location_data/lib/sourceGoogle.go +++ b/google_location_data/lib/sourceGoogle.go @@ -49,7 +49,7 @@ func NewGoogleTimelineLocationsFromFile(path string) (*GoogleTimelineTakeout, er ) } takeout := &GoogleTimelineTakeout{} - if err := json.Unmarshal(bytes, &GoogleTimelineTakeout{}); err != nil { + if err := json.Unmarshal(bytes, &takeout); err != nil { return nil, errors.Join( ErrUnableToUnmarshalSourceDataFile, fmt.Errorf("file: %s", path), diff --git a/google_location_data/lib/sourceGoogle_test.go b/google_location_data/lib/sourceGoogle_test.go index 773f9d5..f51416c 100644 --- a/google_location_data/lib/sourceGoogle_test.go +++ b/google_location_data/lib/sourceGoogle_test.go @@ -5,10 +5,44 @@ import ( "time" ) +func TestNewGoogleTimelineLocationsFromFile(t *testing.T) { + // Open test file + path := "test_data/test_google_location_data.json" + takeout, err := NewGoogleTimelineLocationsFromFile(path) + if err != nil { + t.Errorf("Expected no error opening data, got %v", err) + } + + // Check that the googleTimelineLocations struct has the correct number of locations + if len(takeout.Locations) != 3 { + t.Errorf("Expected 3 locations, got %d, takeout: %v", len(takeout.Locations), takeout) + } + + // Check that the googleTimelineLocations struct has the correct locations + if takeout.Locations[0].LatitudeE7 != 1 { + t.Errorf("Expected Lat 1, got %d from location 0", takeout.Locations[0].LatitudeE7) + } + if takeout.Locations[0].LongitudeE7 != 2 { + t.Errorf("Expected Long 2, got %d from location 0", takeout.Locations[0].LongitudeE7) + } + if takeout.Locations[1].LatitudeE7 != 3 { + t.Errorf("Expected Lat 3, got %d from location 1", takeout.Locations[1].LongitudeE7) + } + if takeout.Locations[1].LongitudeE7 != 4 { + t.Errorf("Expected Long 4, got %d from location 1", takeout.Locations[1].LongitudeE7) + } + if takeout.Locations[2].LatitudeE7 != 5 { + t.Errorf("Expected Lat 5, got %d from location 2", takeout.Locations[2].LatitudeE7) + } + if takeout.Locations[2].LongitudeE7 != 6 { + t.Errorf("Expected Long 6, got %d from location 2", takeout.Locations[2].LongitudeE7) + } +} + func TestToLocationRecords(t *testing.T) { // Create a GoogleTimelineTakeout struct with some test data googleTimelineTakeout := GoogleTimelineTakeout{ - Locations: []GoogleTimelineLocations{ + Locations: []GoogleTimelineLocation{ { LatitudeE7: 633954185, LongitudeE7: 103719669, diff --git a/google_location_data/lib/test_data/test_google_location_data.json b/google_location_data/lib/test_data/test_google_location_data.json new file mode 100644 index 0000000..593d532 --- /dev/null +++ b/google_location_data/lib/test_data/test_google_location_data.json @@ -0,0 +1,78 @@ +{ + "locations": [ + { + "latitudeE7": 1, + "longitudeE7": 2, + "accuracy": 20, + "activity": [ + { + "activity": [ + { + "type": "UNKNOWN", + "confidence": 60 + }, + { + "type": "STILL", + "confidence": 19 + }, + { + "type": "IN_VEHICLE", + "confidence": 17 + }, + { + "type": "ON_BICYCLE", + "confidence": 1 + }, + { + "type": "ON_FOOT", + "confidence": 1 + } + ], + "timestamp": "2014-04-22T12:14:29.557Z" + } + ], + "source": "WIFI", + "deviceTag": 1000000000, + "timestamp": "2014-04-22T12:15:05.148Z" + }, + { + "latitudeE7": 3, + "longitudeE7": 4, + "accuracy": 20, + "source": "WIFI", + "deviceTag": 2116609889, + "timestamp": "2014-04-22T12:16:05.138Z" + }, + { + "latitudeE7": 5, + "longitudeE7": 6, + "accuracy": 20, + "activity": [ + { + "activity": [ + { + "type": "UNKNOWN", + "confidence": 67 + }, + { + "type": "STILL", + "confidence": 20 + }, + { + "type": "IN_VEHICLE", + "confidence": 5 + }, + { + "type": "ON_FOOT", + "confidence": 5 + } + ], + "timestamp": "2014-04-22T12:17:19.641Z" + } + ], + "source": "WIFI", + "deviceTag": 1000000000, + "timestamp": "2014-04-22T12:17:05.248Z" + } + ] +} From 65861c9b305c20f8cdc10435efbf4e5c76a9a96f Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Sun, 25 Aug 2024 21:13:05 +0200 Subject: [PATCH 10/15] started using google time line data in photos --- google_location_data/lib/coordinates.go | 26 +++++++ google_location_data/main.go | 29 -------- photoMetadata/lib/photo.go | 96 +++++++++++++++++++++++++ photoMetadata/main.go | 32 ++------- 4 files changed, 128 insertions(+), 55 deletions(-) delete mode 100644 google_location_data/main.go create mode 100644 photoMetadata/lib/photo.go diff --git a/google_location_data/lib/coordinates.go b/google_location_data/lib/coordinates.go index 50f4d42..a146574 100644 --- a/google_location_data/lib/coordinates.go +++ b/google_location_data/lib/coordinates.go @@ -1,6 +1,10 @@ package lib import ( + "errors" + "fmt" + + nmea "github.com/adrianmo/go-nmea" geo "github.com/kellydunn/golang-geo" ) @@ -26,6 +30,28 @@ func NewCoordinatesFromGeopoint(point geo.Point) Corrdinates { } } +var ErrInvalidDMS = errors.New("Invalid DMS") + +func NewCoordinatesFromDMS(latitude, longitude string) (Corrdinates, error) { + lat, err := nmea.ParseDMS(latitude) + if err != nil { + return Corrdinates{}, errors.Join( + ErrInvalidDMS, + fmt.Errorf("latitude: %s", latitude), + err, + ) + } + lng, err := nmea.ParseDMS(longitude) + if err != nil { + return Corrdinates{}, errors.Join( + ErrInvalidDMS, + fmt.Errorf("longitude: %s", longitude), + err, + ) + } + return NewCorrdinatesE2(lat, lng), nil +} + func (coordinate *Corrdinates) GetE2Coord() (lat float64, long float64) { return coordinate.Lat(), coordinate.Lng() } diff --git a/google_location_data/main.go b/google_location_data/main.go deleted file mode 100644 index 558ceb7..0000000 --- a/google_location_data/main.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "image" - "image/png" - "os" - - "github.com/dustin/go-heatmap" - "github.com/dustin/go-heatmap/schemes" - "github.com/sander-skjulsvik/tools/google_location_data/lib" -) - -func main() { - googleLocationPath := "data/takeout-20240803T162513Z-001/Takeout/Location History (Timeline)/Records.json" - locationRecords := lib.ImportSourceLocationData(googleLocationPath) - - points := []heatmap.DataPoint{} - for _, record := range locationRecords.Locations { - points = append(points, - heatmap.P(record.LatitudeE7/1e7, record.LongitudeE7/1e7)) - } - - // scheme, _ := schemes.FromImage("../schemes/fire.png") - scheme := schemes.AlphaFire - - img := heatmap.Heatmap(image.Rect(0, 0, 1024, 1024), - points, 150, 128, scheme) - png.Encode(os.Stdout, img) -} diff --git a/photoMetadata/lib/photo.go b/photoMetadata/lib/photo.go new file mode 100644 index 0000000..dbaa5b5 --- /dev/null +++ b/photoMetadata/lib/photo.go @@ -0,0 +1,96 @@ +package lib + +import ( + "errors" + "fmt" + "strings" + "time" + + locationData "github.com/sander-skjulsvik/tools/google_location_data/lib" +) + +type Photo struct { + Path string +} + +const ( + DateTimeOriginal = "DateTimeOriginal" + GPSPosition = "GPSPosition" + GPSDateTime = "GPSDateTime" + ExifDateTimeLatout = "2006:01:02 15:04:05-07:00" +) + +// New photo funcs + +// Photo methods + +func (photo *Photo) SearchExifData(search string) interface{} { + fmt.Println("Searching for data") + + fileInfos, err := GetAllExifData(photo.Path) + if err != nil { + fmt.Printf("Error: %v", err) + return nil + } + + for _, fileInfo := range fileInfos { + if fileInfo.Err != nil { + fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err) + continue + } + + for k, v := range fileInfo.Fields { + if k == search { + return v + } + } + } + return nil +} + +/* +fuji date time format: 2023:11:07 11:46:28+01:00 +go layout: 2006:01:02 15:04:05-07:00 +*/ +func (photo *Photo) GetDateTimeOriginal() (time.Time, error) { + dateTimeOriginal, ok := photo.SearchExifData(DateTimeOriginal).(string) + if !ok { + return time.Time{}, errors.New("dateTimeOriginal not found") + } + if dateTimeOriginal == "" { + return time.Time{}, errors.New("dateTimeOriginal empty") + } + parsedTime, err := time.Parse(ExifDateTimeLatout, dateTimeOriginal) + if err != nil { + return time.Time{}, fmt.Errorf("Error parsing dateTimeOriginal: %v", err) + } + + return parsedTime, nil +} + +func (photo *Photo) GetLocationRecord() (*locationData.LocationRecord, error) { + // Location + gpsPosition, ok := photo.SearchExifData("GPSPosition").(string) + if !ok { + return nil, errors.New("GPSPosition unable to string assert") + } + if gpsPosition == "" { + return nil, errors.New("GPSPosition empty") + } + latLong := strings.Split(gpsPosition, ",") + coords, err := locationData.NewCoordinatesFromDMS(latLong[0], latLong[1]) + if err != nil { + return nil, fmt.Errorf("Error parsing GPSPosition: %v", err) + } + + // Time + dateTimeOriginal, err := photo.GetDateTimeOriginal() + if err != nil { + return nil, fmt.Errorf("Error getting dateTimeOriginal: %v", err) + } + + return &locationData.LocationRecord{ + Corrdinates: coords, + Time: dateTimeOriginal, + }, nil +} diff --git a/photoMetadata/main.go b/photoMetadata/main.go index 5357205..7b364f6 100644 --- a/photoMetadata/main.go +++ b/photoMetadata/main.go @@ -11,22 +11,26 @@ func main() { method := os.Args[1] photoPath := os.Args[2] + p := lib.Photo{Path: photoPath} + switch method { case "list": List(photoPath) case "write": Write(photoPath) // List(photoPath) - pos := Search(photoPath, "GPSPosition") + pos := p.SearchExifData("GPSPosition") fmt.Println(pos) case "search": - pos := Search(photoPath, "GPSPosition") + pos := p.SearchExifData("GPSPosition") fmt.Println(pos) default: fmt.Println("Invalid method") } } +// func ApplyLocationData(locationRecord locationData.LocationRecord, photo ) + func List(path string) { data, err := lib.GetAllExifData(path) if err != nil { @@ -46,27 +50,3 @@ func Write(path string) error { } return nil } - -func Search(path string, search string) interface{} { - fmt.Println("Searching for data") - - fileInfos, err := lib.GetAllExifData(path) - if err != nil { - fmt.Printf("Error: %v", err) - return nil - } - - for _, fileInfo := range fileInfos { - if fileInfo.Err != nil { - fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err) - continue - } - - for k, v := range fileInfo.Fields { - if k == search { - return v - } - } - } - return nil -} From 5d8b92add44efc71a285dd8e402151d895246058 Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Wed, 28 Aug 2024 12:40:56 +0200 Subject: [PATCH 11/15] moved generate files to main libs and addded GetAllFilesOfType. Tested --- dupes/lib/producerConsumer/main_test.go | 31 +++++++-------- dupes/lib/test/files.go | 4 +- dupes/lib/test/lib.go | 3 +- dupes/lib/test/run.go | 9 ----- libs/files/lib.go | 43 +++++++++++++++++++++ libs/files/lib_test.go | 51 +++++++++++++++++++++++++ 6 files changed, 115 insertions(+), 26 deletions(-) create mode 100644 libs/files/lib_test.go diff --git a/dupes/lib/producerConsumer/main_test.go b/dupes/lib/producerConsumer/main_test.go index 3e21528..8006dd5 100644 --- a/dupes/lib/producerConsumer/main_test.go +++ b/dupes/lib/producerConsumer/main_test.go @@ -11,6 +11,7 @@ import ( "github.com/sander-skjulsvik/tools/dupes/lib/common" "github.com/sander-skjulsvik/tools/dupes/lib/test" + "github.com/sander-skjulsvik/tools/libs/files" "github.com/sander-skjulsvik/tools/libs/progressbar" "gotest.tools/assert" ) @@ -36,7 +37,7 @@ func TestGetFiles(t *testing.T) { workDir + "folder/" + "folder/" + "nesting_file_name", } for _, file := range expectedFilePaths { - test.CreateFile(file, "nesting_file_content") + files.CreateFile(file, "nesting_file_content") } calculatedFilePaths := make(chan string) go getFiles(workDir, calculatedFilePaths) @@ -59,8 +60,8 @@ func TestGetFiles(t *testing.T) { { workDir := baseDir + "test_emtpy_file/" os.MkdirAll(filepath.Clean(workDir), 0o755) - test.CreateEmptyFile(workDir + "empty_file") - test.CreateFile(workDir+"not_empty_file", "not_empty_file") + files.CreateEmptyFile(workDir + "empty_file") + files.CreateFile(workDir+"not_empty_file", "not_empty_file") calculatedFilePaths := make(chan string) go getFiles(workDir, calculatedFilePaths) @@ -77,7 +78,7 @@ func TestGetFiles(t *testing.T) { { workDir := baseDir + "test_symlink/" os.MkdirAll(filepath.Clean(workDir), 0o755) - test.CreateEmptyFile(workDir + "source_file") + files.CreateEmptyFile(workDir + "source_file") os.Symlink(workDir+"source_file", workDir+"destination_file") calculatedFilePaths := make(chan string) @@ -107,9 +108,9 @@ func TestGetFiles(t *testing.T) { { workDir := baseDir + "sleeping_before_consuming/" os.MkdirAll(filepath.Clean(workDir), 0o755) - test.CreateFile(filepath.Join(workDir, "1"), "1") - test.CreateFile(filepath.Join(workDir, "2"), "2") - test.CreateFile(filepath.Join(workDir, "3"), "3") + files.CreateFile(filepath.Join(workDir, "1"), "1") + files.CreateFile(filepath.Join(workDir, "2"), "2") + files.CreateFile(filepath.Join(workDir, "3"), "3") calculatedFilePathsChan := make(chan string) calculatedFilePathsSlice := []string{} @@ -135,7 +136,7 @@ func TestAppendFileTreadSafe(t *testing.T) { path := workDir + "single_file" lock := sync.Mutex{} - test.CreateFile(path, "I am a single file") + files.CreateFile(path, "I am a single file") expectedHash := "1be3d7cfb6df7ff4ed6235a70603dc3ee8fa636a5e44a5c2ea8ffbcd38b41bd0" appendFileTreadSafe(&d, filepath.Clean(path), &lock) @@ -165,7 +166,7 @@ func TestAppendFileTreadSafe(t *testing.T) { d := common.NewDupes() n := 1000 for i := 0; i < n; i++ { - test.CreateFile(workDir+strconv.Itoa(i), "I am one of many files") + files.CreateFile(workDir+strconv.Itoa(i), "I am one of many files") } lock := sync.Mutex{} @@ -203,7 +204,7 @@ func TestAppendFileTreadSafe(t *testing.T) { d := common.NewDupes() n := 1000 for i := 0; i < n; i++ { - test.CreateFile(workDir+strconv.Itoa(i), "I am one of many files: "+strconv.Itoa(i)) + files.CreateFile(workDir+strconv.Itoa(i), "I am one of many files: "+strconv.Itoa(i)) } lock := sync.Mutex{} @@ -235,7 +236,7 @@ func TestAppendFileTreadSafe(t *testing.T) { lock := sync.Mutex{} n := 10 for i := 0; i < n; i++ { - test.CreateFile(workDir+strconv.Itoa(i), "") + files.CreateFile(workDir+strconv.Itoa(i), "") } expectedHash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" @@ -268,7 +269,7 @@ func TestProcessFiles(t *testing.T) { os.MkdirAll(filepath.Clean(workDir), 0o755) path := workDir + "single_file" - test.CreateFile(path, "I am a single file") + files.CreateFile(path, "I am a single file") expectedHash := "1be3d7cfb6df7ff4ed6235a70603dc3ee8fa636a5e44a5c2ea8ffbcd38b41bd0" filePaths := make(chan string) @@ -324,7 +325,7 @@ func TestProcessFiles(t *testing.T) { Content: "I am not unique", }} for _, file := range expectedFilePaths { - test.CreateFile(file.Path, file.Content) + files.CreateFile(file.Path, file.Content) } filePaths := make(chan string) @@ -383,7 +384,7 @@ func TestProcessFilesNConsumers(t *testing.T) { os.MkdirAll(filepath.Clean(workDir), 0o755) path := workDir + "single_file" - test.CreateFile(path, "I am a single file") + files.CreateFile(path, "I am a single file") expectedHash := "1be3d7cfb6df7ff4ed6235a70603dc3ee8fa636a5e44a5c2ea8ffbcd38b41bd0" filePaths := make(chan string) @@ -441,7 +442,7 @@ func TestProcessFilesNConsumers(t *testing.T) { Content: "I am not unique", }} for _, file := range expectedFilePaths { - test.CreateFile(file.Path, file.Content) + files.CreateFile(file.Path, file.Content) } filePaths := make(chan string) diff --git a/dupes/lib/test/files.go b/dupes/lib/test/files.go index 116ddf3..8170993 100644 --- a/dupes/lib/test/files.go +++ b/dupes/lib/test/files.go @@ -4,6 +4,8 @@ import ( "os" "path/filepath" "slices" + + files "github.com/sander-skjulsvik/tools/libs/files" ) type Folder struct { @@ -28,7 +30,7 @@ func (folder *Folder) Generate(parents string) { // Create files in the folder for _, file := range folder.Files { filePath := filepath.Join(parents, folder.Name, file.Name) - CreateFile(filePath, file.Content) + files.CreateFile(filePath, file.Content) } // Create child folders diff --git a/dupes/lib/test/lib.go b/dupes/lib/test/lib.go index 84dab36..35d1389 100644 --- a/dupes/lib/test/lib.go +++ b/dupes/lib/test/lib.go @@ -6,6 +6,7 @@ import ( "path/filepath" "github.com/sander-skjulsvik/tools/dupes/lib/common" + files "github.com/sander-skjulsvik/tools/libs/files" ) func SetupExpectedDupes(path string) { @@ -17,7 +18,7 @@ func SetupExpectedDupes(path string) { // Create the parent folders if they don't exist CrateParentFolders(fullPath) // Create the file with content from ExpectedDupesHashMap - CreateFile(fullPath, ExpectedDupesHashMap[hash]) + files.CreateFile(fullPath, ExpectedDupesHashMap[hash]) } } } diff --git a/dupes/lib/test/run.go b/dupes/lib/test/run.go index 0102b62..b8ff72c 100644 --- a/dupes/lib/test/run.go +++ b/dupes/lib/test/run.go @@ -118,12 +118,3 @@ func check(e error) { panic(e) } } - -func CreateEmptyFile(path string) { - d := []byte("") - check(os.WriteFile(filepath.Clean(path), d, 0o644)) -} - -func CreateFile(path, content string) { - check(os.WriteFile(filepath.Clean(path), []byte(content), 0o644)) -} diff --git a/libs/files/lib.go b/libs/files/lib.go index e9a8aa1..a14a08e 100644 --- a/libs/files/lib.go +++ b/libs/files/lib.go @@ -8,6 +8,49 @@ import ( "path/filepath" ) +func CreateEmptyFileWithFolders(path string) error { + err := os.MkdirAll(filepath.Dir(path), os.ModePerm) + if err != nil { + return fmt.Errorf("unable to create folders: %w", err) + } + return CreateEmptyFile(path) +} + +func CreateEmptyFile(path string) error { + d := []byte("") + return os.WriteFile(filepath.Clean(path), d, 0o644) +} + +func CreateFile(path, content string) error { + return os.WriteFile(filepath.Clean(path), []byte(content), 0o644) +} + +func GetAllFilesOfType(path string, fileType string) ([]string, error) { + var files []string + err := filepath.Walk( + path, + func(path string, info fs.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("unable to walk path: %w", err) + } + if info == nil { + return fmt.Errorf("file info is nil") + } + if info.IsDir() { + return nil + } + if filepath.Ext(path) == fileType { + files = append(files, path) + } + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("unable to get all files of type: %w", err) + } + return files, nil +} + func GetNumberOfFiles(path string) (int, error) { n := 0 err := filepath.Walk( diff --git a/libs/files/lib_test.go b/libs/files/lib_test.go new file mode 100644 index 0000000..b6eff4c --- /dev/null +++ b/libs/files/lib_test.go @@ -0,0 +1,51 @@ +package files + +import ( + "os" + "path/filepath" + "slices" + "testing" +) + +func TestGetAllFilesOfType(t *testing.T) { + // Setup + var ( + basePath = "../test_files" + fileType = ".txt" + paths = []string{ + "/file1.txt", + "/file2.txt/abc.raf", + "/file3/abc.txt", + "/file4.txt/abc.txt", + "/file5.txt", + "/file6.tx", + "/file7t.xt", + } // Create files + expectedFilePaths = []string{ + "file1.txt", + "/file3/abc.txt", + "/file4.txt/abc.txt", + "/file5.txt", + } + ) + defer os.RemoveAll(basePath) + for _, path := range paths { + err := CreateEmptyFileWithFolders( + filepath.Join(basePath, path), + ) + if err != nil { + t.Errorf("Error creating test files: %v", err) + } + } + // Test + files, err := GetAllFilesOfType(basePath, fileType) + if err != nil { + t.Errorf("Error getting files of type: %v", err) + } + if len(files) != len(expectedFilePaths) { + t.Errorf("Expected %d files, got %d", len(expectedFilePaths), len(files)) + } + if slices.Equal(expectedFilePaths, files) { + t.Errorf("Expected %v, got %v", expectedFilePaths, files) + } +} From 2ad69a4d27de4276bc407f57c433d15296d927fb Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Wed, 28 Aug 2024 13:15:50 +0200 Subject: [PATCH 12/15] Added filtering for mulitple file types --- libs/files/lib.go | 12 +++++++++-- libs/files/lib_test.go | 13 +++++++----- photoMetadata/lib/photo.go | 30 +++++++++++++++++++++++++- photoMetadata/lib/photo_test.go | 37 +++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 photoMetadata/lib/photo_test.go diff --git a/libs/files/lib.go b/libs/files/lib.go index a14a08e..b2541c6 100644 --- a/libs/files/lib.go +++ b/libs/files/lib.go @@ -6,6 +6,8 @@ import ( "log" "os" "path/filepath" + "slices" + "strings" ) func CreateEmptyFileWithFolders(path string) error { @@ -25,7 +27,11 @@ func CreateFile(path, content string) error { return os.WriteFile(filepath.Clean(path), []byte(content), 0o644) } -func GetAllFilesOfType(path string, fileType string) ([]string, error) { +/* +GetAllFilesOfTypes returns all files of the specified types in the specified directory. +filetypes needs to be prefixed with a dot. E.g. ".txt", and lowercase. +*/ +func GetAllFilesOfTypes(path string, fileTypes []string) ([]string, error) { var files []string err := filepath.Walk( path, @@ -39,7 +45,9 @@ func GetAllFilesOfType(path string, fileType string) ([]string, error) { if info.IsDir() { return nil } - if filepath.Ext(path) == fileType { + p := strings.ToLower(filepath.Ext(path)) + b := slices.Contains(fileTypes, p) + if b { files = append(files, path) } return nil diff --git a/libs/files/lib_test.go b/libs/files/lib_test.go index b6eff4c..c612a19 100644 --- a/libs/files/lib_test.go +++ b/libs/files/lib_test.go @@ -10,22 +10,25 @@ import ( func TestGetAllFilesOfType(t *testing.T) { // Setup var ( - basePath = "../test_files" - fileType = ".txt" - paths = []string{ + basePath = "../test_files" + fileTypes = []string{".txt", ".raf"} + + paths = []string{ "/file1.txt", "/file2.txt/abc.raf", + "/file2.txt/abc.raf1", "/file3/abc.txt", "/file4.txt/abc.txt", "/file5.txt", "/file6.tx", "/file7t.xt", - } // Create files + } expectedFilePaths = []string{ "file1.txt", "/file3/abc.txt", "/file4.txt/abc.txt", "/file5.txt", + "/file2.txt/abc.raf", } ) defer os.RemoveAll(basePath) @@ -38,7 +41,7 @@ func TestGetAllFilesOfType(t *testing.T) { } } // Test - files, err := GetAllFilesOfType(basePath, fileType) + files, err := GetAllFilesOfTypes(basePath, fileTypes) if err != nil { t.Errorf("Error getting files of type: %v", err) } diff --git a/photoMetadata/lib/photo.go b/photoMetadata/lib/photo.go index dbaa5b5..0949295 100644 --- a/photoMetadata/lib/photo.go +++ b/photoMetadata/lib/photo.go @@ -7,17 +7,45 @@ import ( "time" locationData "github.com/sander-skjulsvik/tools/google_location_data/lib" + "github.com/sander-skjulsvik/tools/libs/files" ) +// should all be lowercase +var SUPPORTED_FILE_TYPES = []string{ + ".raf", +} + +type PhotoCollection struct { + Photos []Photo +} + +func NewPhotoCollectionFromPath(path string) (*PhotoCollection, error) { + collection := PhotoCollection{} + paths, err := files.GetAllFilesOfTypes(path, SUPPORTED_FILE_TYPES) + numberOfFiles, _ := files.GetNumberOfFiles(path) + fmt.Printf("Number of files: %v\n", numberOfFiles) + if err != nil { + return nil, fmt.Errorf("Error getting files %v", err) + } + for _, path := range paths { + collection.Photos = append(collection.Photos, *NewPhotoFromPath(path)) + } + return &collection, nil +} + type Photo struct { Path string } +func NewPhotoFromPath(path string) *Photo { + return &Photo{Path: path} +} + const ( DateTimeOriginal = "DateTimeOriginal" GPSPosition = "GPSPosition" GPSDateTime = "GPSDateTime" - ExifDateTimeLatout = "2006:01:02 15:04:05-07:00" + ExifDateTimeLatout = "2006:01:02 15:04:05-07:00" // Atleast for fuji ) // New photo funcs diff --git a/photoMetadata/lib/photo_test.go b/photoMetadata/lib/photo_test.go new file mode 100644 index 0000000..febbeb2 --- /dev/null +++ b/photoMetadata/lib/photo_test.go @@ -0,0 +1,37 @@ +package lib + +import ( + "fmt" + "os" + "path" + "path/filepath" + "testing" + + "github.com/sander-skjulsvik/tools/libs/files" +) + +func TestNewPhotoCollectionFromPath(t *testing.T) { + // Setup + basePath := "TestNewPhotoCollectionFromPath" + defer os.RemoveAll(basePath) + + // Test with single file + path := path.Join( + basePath, + "singleFile", + fmt.Sprintf("testfile%s", SUPPORTED_FILE_TYPES[0]), + ) + files.CreateEmptyFileWithFolders(path) + + photoCollection, err := NewPhotoCollectionFromPath(path) + if err != nil { + t.Errorf("Failed to create photo collection from path: %v", err) + } + if len(photoCollection.Photos) != 1 { + t.Errorf("Expected photo collection to have 1 photo, got %v", len(photoCollection.Photos)) + } + calcPath := filepath.Clean(path) + if photoCollection.Photos[0].Path != calcPath { + t.Errorf("Expected photo path to be %v, got %v", calcPath, photoCollection.Photos[0].Path) + } +} From 14d524e6e34a5072d592d07b18cdb72c5f23d940 Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Wed, 28 Aug 2024 15:06:03 +0200 Subject: [PATCH 13/15] NewPhotoCollectionFromPath created and tested --- photoMetadata/lib/photo.go | 5 +---- photoMetadata/lib/photo_test.go | 39 +++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/photoMetadata/lib/photo.go b/photoMetadata/lib/photo.go index 0949295..0c93800 100644 --- a/photoMetadata/lib/photo.go +++ b/photoMetadata/lib/photo.go @@ -22,8 +22,6 @@ type PhotoCollection struct { func NewPhotoCollectionFromPath(path string) (*PhotoCollection, error) { collection := PhotoCollection{} paths, err := files.GetAllFilesOfTypes(path, SUPPORTED_FILE_TYPES) - numberOfFiles, _ := files.GetNumberOfFiles(path) - fmt.Printf("Number of files: %v\n", numberOfFiles) if err != nil { return nil, fmt.Errorf("Error getting files %v", err) } @@ -37,6 +35,7 @@ type Photo struct { Path string } +// New photo funcs func NewPhotoFromPath(path string) *Photo { return &Photo{Path: path} } @@ -48,8 +47,6 @@ const ( ExifDateTimeLatout = "2006:01:02 15:04:05-07:00" // Atleast for fuji ) -// New photo funcs - // Photo methods func (photo *Photo) SearchExifData(search string) interface{} { diff --git a/photoMetadata/lib/photo_test.go b/photoMetadata/lib/photo_test.go index febbeb2..481eca7 100644 --- a/photoMetadata/lib/photo_test.go +++ b/photoMetadata/lib/photo_test.go @@ -3,8 +3,8 @@ package lib import ( "fmt" "os" - "path" "path/filepath" + "slices" "testing" "github.com/sander-skjulsvik/tools/libs/files" @@ -16,7 +16,7 @@ func TestNewPhotoCollectionFromPath(t *testing.T) { defer os.RemoveAll(basePath) // Test with single file - path := path.Join( + path := filepath.Join( basePath, "singleFile", fmt.Sprintf("testfile%s", SUPPORTED_FILE_TYPES[0]), @@ -34,4 +34,39 @@ func TestNewPhotoCollectionFromPath(t *testing.T) { if photoCollection.Photos[0].Path != calcPath { t.Errorf("Expected photo path to be %v, got %v", calcPath, photoCollection.Photos[0].Path) } + + // Test with directory + dirPath := filepath.Join( + basePath, + "directory", + ) + defer os.RemoveAll(dirPath) + filePaths := []string{ + "testfile1", + "testfile1.raf", + "d/d/d/d/d/testfile2.raf", + "d/d/d/d/d/testfile2.noVaid", + } + expectedFilePaths := []string{ + filepath.Join(dirPath, filePaths[1]), + filepath.Join(dirPath, filePaths[2]), + } + for _, file := range filePaths { + files.CreateEmptyFileWithFolders(filepath.Join(dirPath, file)) + } + calc, err := NewPhotoCollectionFromPath(dirPath) + if err != nil { + t.Errorf("Failed to create photo collection from path: %v", err) + } + calcFilePaths := []string{} + for _, photo := range calc.Photos { + calcFilePaths = append(calcFilePaths, photo.Path) + } + if len(calc.Photos) != 2 { + t.Errorf("Expected photo collection to have 2 photos, got %v", len(calc.Photos)) + } + + if slices.Equal(calcFilePaths, expectedFilePaths) { + t.Errorf("Expected photo collection to have paths %v, got %v", expectedFilePaths, calcFilePaths) + } } From dfa3734caf3d8808124389d12a0011e8d75c420e Mon Sep 17 00:00:00 2001 From: sander skjulsvik Date: Wed, 28 Aug 2024 15:06:49 +0200 Subject: [PATCH 14/15] mod tidy --- go.mod | 7 ++++--- go.sum | 14 ++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 5e1f03d..00b4794 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module github.com/sander-skjulsvik/tools go 1.22 -require github.com/gosuri/uiprogress v0.0.1 +require ( + github.com/adrianmo/go-nmea v1.10.0 + github.com/gosuri/uiprogress v0.0.1 +) require ( github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect @@ -18,8 +21,6 @@ require ( require ( github.com/barasher/go-exiftool v1.10.0 github.com/deckarep/golang-set/v2 v2.6.0 - github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a - github.com/jftuga/geodist v1.0.0 github.com/kellydunn/golang-geo v0.7.0 golang.org/x/sys v0.19.0 // indirect gotest.tools v2.2.0+incompatible diff --git a/go.sum b/go.sum index 27b7702..cd524b2 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,12 @@ +github.com/adrianmo/go-nmea v1.10.0 h1:L1aYaebZ4cXFCoXNSeDeQa0tApvSKvIbqMsK+iaRiCo= +github.com/adrianmo/go-nmea v1.10.0/go.mod h1:u8bPnpKt/D/5rll/5l9f6iDfeq5WZW0+/SXdkwix6Tg= github.com/barasher/go-exiftool v1.10.0 h1:f5JY5jc42M7tzR6tbL9508S2IXdIcG9QyieEXNMpIhs= github.com/barasher/go-exiftool v1.10.0/go.mod h1:F9s/a3uHSM8YniVfwF+sbQUtP8Gmh9nyzigNF+8vsWo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a h1:o0oeorr6BKW/lHrT2NWX1CQn6LDhr5lvAPIIjTDC1bg= -github.com/dustin/go-heatmap v0.0.0-20180603032536-b89dbd73785a/go.mod h1:VBmwC4U3p2SMEKr+/m5j0eby7rmUtSoA5TGLwe6P+3A= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -13,8 +15,6 @@ github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= -github.com/jftuga/geodist v1.0.0 h1:PFPQlZtj10u8ETAYTyxE0DWMl1bwA+Xzrqb4+oLkkC0= -github.com/jftuga/geodist v1.0.0/go.mod h1:BohEDxpZ8S5ADAxW/9EKPSKWOVl0+3wHENIT40m4UO4= github.com/kellydunn/golang-geo v0.7.0 h1:A5j0/BvNgGwY6Yb6inXQxzYwlPHc6WVZR+MrarZYNNg= github.com/kellydunn/golang-geo v0.7.0/go.mod h1:YYlQPJ+DPEzrHx8kT3oPHC/NjyvCCXE+IuKGKdrjrcU= github.com/kylelemons/go-gypsy v1.0.0 h1:7/wQ7A3UL1bnqRMnZ6T8cwCOArfZCxFmb1iTxaOOo1s= @@ -25,13 +25,19 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= From 25addf538a989117cc61f41a5cd39ce4769cf23a Mon Sep 17 00:00:00 2001 From: sander-skjulsvik Date: Sat, 30 Nov 2024 13:33:17 +0100 Subject: [PATCH 15/15] added windows exiftool install --- photoMetadata/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/photoMetadata/README.md b/photoMetadata/README.md index eb0865f..f1146f3 100644 --- a/photoMetadata/README.md +++ b/photoMetadata/README.md @@ -2,3 +2,5 @@ ### ExifTool sudo apt-get install exiftool + +winget install --id=OliverBetz.ExifTool -e