Merge branch 'develop'

README.md

@@ -60,7 +60,7 @@ The corrected version of the trace is then exported in the GPX format.
 
 In order to run the application, please follow the subsequent steps: 
 
-1) Install the package by executing the following command: `pip install -i https://test.pypi.org/simple/ gpsclean`
+1) Install the package by executing the following command: `pip install -i https://test.pypi.org/simple/ --extra-index-url https://google-coral.github.io/py-repo/ gpsclean`
 2) Now you are ready to clean your first GPS trace by executing the following command: `gpsclean path/to/your_trace.gpx`
 3) In the same folder where the original trace resides, you will find a cleaned version of the trace with the suffix "_cleaned"
 

setup.py

@@ -1,11 +1,13 @@
 import setuptools
+import sys
+import platform
 
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 setuptools.setup(
     name="gpsclean",
-    version="0.2.0",
+    version="0.3.0",
     author="Davide Sbetti",
     author_email="[email protected]",
     description="An application to correct a GPS trace using machine learning techniques",
@@ -22,21 +24,20 @@
     ],
     package_dir={"": "src"},
     packages=setuptools.find_packages(where="src"),
-    python_requires=">=3.6",
+    python_requires=">=3.7, <3.10",
     install_requires=[
         "art==5.3",
         "filterpy==1.4.5",
         "geojson==2.5.0",
         "gpxpy==1.4.2",
-        "tensorflow==2.7.0",
-        "pandas==0.25.3",
-        "scipy==1.6.1",
-        "pyproj==3.1.0",
-        "numpy==1.21.4",
-        "pyinstaller==4.5.1",
+        "tflite-runtime>=2.5.0.post1",
+        "pandas>=0.25.3",
+        "scipy>=1.6.1",
+        "pyproj>=3.0.0",
+        "numpy>=1.20.0",
     ],
     package_data={
-         "gpsclean": ["data/*.h5"],
+         "gpsclean": ["data/*.tflite"],
     },
     entry_points={
     'console_scripts': [

src/gpsclean.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gpsclean
-Version: 0.2.0
+Version: 0.3.0
 Summary: An application to correct a GPS trace using machine learning techniques
 Home-page: https://github.com/sbettid/GPSClean
 Author: Davide Sbetti
@@ -11,7 +11,7 @@ Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
+Requires-Python: >=3.7, <3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 
@@ -77,7 +77,7 @@ The corrected version of the trace is then exported in the GPX format.
 
 In order to run the application, please follow the subsequent steps: 
 
-1) Install the package by executing the following command: `pip install -i https://test.pypi.org/simple/ gpsclean`
+1) Install the package by executing the following command: `pip install -i https://test.pypi.org/simple/ --extra-index-url https://google-coral.github.io/py-repo/ gpsclean`
 2) Now you are ready to clean your first GPS trace by executing the following command: `gpsclean path/to/your_trace.gpx`
 3) In the same folder where the original trace resides, you will find a cleaned version of the trace with the suffix "_cleaned"
 
@@ -89,6 +89,6 @@ On the other hand, if you prefer to download a all-in-one executable which does
 ## How to manually create the all-in-one executable
 
 1) Clone the [repository](https://github.com/sbettid/GPSClean) and open a shell in the `src/gpsclean` folder
-2) Execute the following command: `pyinstaller -F --add-data "data/model_42t_traces.h5;." gpsclean.py`
+2) Execute the following command: `pyinstaller -F --add-data "data/model_42t_traces.h5;data" gpsclean.py`
 3) At the end of the process, a dist folder can be found inside the `src/gpsclean` folder, containing the packaged executable. 
 

src/gpsclean.egg-info/SOURCES.txt

@@ -13,4 +13,4 @@ src/gpsclean.egg-info/dependency_links.txt
 src/gpsclean.egg-info/entry_points.txt
 src/gpsclean.egg-info/requires.txt
 src/gpsclean.egg-info/top_level.txt
-src/gpsclean/data/model_42t_traces.h5
+src/gpsclean/data/model_42t_traces.tflite

src/gpsclean.egg-info/requires.txt

@@ -2,9 +2,8 @@ art==5.3
 filterpy==1.4.5
 geojson==2.5.0
 gpxpy==1.4.2
-tensorflow==2.7.0
-pandas==0.25.3
-scipy==1.6.1
-pyproj==3.1.0
-numpy==1.21.4
-pyinstaller==4.5.1
+tflite-runtime>=2.5.0.post1
+pandas>=0.25.3
+scipy>=1.6.1
+pyproj>=3.0.0
+numpy>=1.20.0

src/gpsclean/Correction.py

@@ -6,9 +6,6 @@
 from scipy.linalg import block_diag
 import pandas as pd
 from datetime import datetime
-from tensorflow import clip_by_value, GradientTape, convert_to_tensor, zeros_like
-from tensorflow.keras.optimizers import Adam
-from tensorflow.keras.losses import SparseCategoricalCrossentropy
 
 
 #defining conversions between the two used coordinates systems
@@ -81,212 +78,7 @@ def remove_pauses(all_coords, all_coordtimes, predictions, deltas):
                     cur_point += 1
 
     #return the results as numpy arrays
-    return np.array(points), np.array(coordTimes), np.array(reduced_deltas), np.array(reduced_predictions), np.array(original_points), np.array(original_times)
-
-
-#function used to clip a delta tensor using the specified EPS value
-def clip_eps(tensor, eps):
-    # clip the values of the tensor to a given range and return it
-    return clip_by_value(tensor, clip_value_min=-eps,clip_value_max=eps)
-
-
-#function used to generate a delta correction for a segment provided as input, using the specified model for
-#the learning process
-def generate_adversaries(model, example, delta, prediction, classIdx, steps=30):
-    # iterate over the number of steps
-    optimizer = Adam()
-    sccLoss = SparseCategoricalCrossentropy()
-    prediction = prediction.squeeze()
-    for step in range(0, steps):
-        # record our gradients
-        with GradientTape() as tape:
-
-            # explicitly indicate that our perturbation vector should
-            # be tracked for gradient updates
-            tape.watch(delta)
-            #print(tf.trainable_variables())
-            adversary =  example + delta
-
-            predictions = model(adversary, training=False)
-            #print(predictions)
-
-            loss = sccLoss(convert_to_tensor([classIdx]),predictions)
-
-            #if step % 5 == 0:
-            #    print("step: {}, loss: {}...".format(step,loss.numpy()))
-
-            gradients = tape.gradient(loss, delta)
-
-            # update the weights, clip the perturbation vector, and
-            # update its values....
-            optimizer.apply_gradients([(gradients, delta)])
-            delta.assign_add(clip_eps(delta, EPS))
-
-            #but keep the time column to 0..
-            delta[:,:,3].assign(zeros_like(delta[:,:,3]))
-
-            #as well as the rows corresponding to already correct points
-            for i in range(len(prediction)):
-                if prediction[i]:
-                    delta[:,i, :].assign(zeros_like(delta[:,i, :]))
-
-            #delta.assign_add(delta)
-
-    # return the perturbation vector
-    adversary =  example + delta
-
-    predictions = model.predict_classes(adversary)
-    prediction_or = model.predict_classes(np.array([example]))
-    print("New pred: ", predictions, " was ", prediction_or)
-    return delta
-
-#method used to correct outliers using customly learned deltas, applied after we have removed the pauses
-#if a point was predicted as incorrect but not a pause,
-#convert to ECEF coordinates and applicate delta
-def apply_deltas(points, times, deltas, predictions):
-
-    #array that will host the corrected points
-    corrected_points = []        
-    #go over each point
-    for i in range(points.shape[0]):
-
-        point = points[i]
-        # if point is incorrect
-        if predictions[i] > 1:
-            #convert to ecef and apply delta
-            cur_lon, cur_lat, cur_alt = point[0], point[1], point[2]
-            x, y, z = pyproj.transform(lla, ecef, cur_lon, cur_lat, cur_alt, radians=False)
-            x += deltas[i][0]
-            y += deltas[i][1]
-            z += deltas[i][2]
-            #transform back to lat lon and add to list
-            lon, lat, alt = pyproj.transform(ecef, lla, x, y, z, radians=False)
-
-            corrected_points.append(np.array([lon,lat,alt]))
-        else: #keep the original point if it is correct
-             corrected_points.append(point)
-
-    #convert to a numpy array and return        
-    corrected_points = np.array(corrected_points)
-
-    return corrected_points, times
-
-
-
-#method used to correct outliers using linear interpolation (based on time)
-#here we need all points in the ECEF system to apply interpolation
-#using the whole history of the trace
-def correct_using_interpolation(points, times, predictions):
-
-    #list that will host converted data
-    data = []
-
-    #convert all points to ECEF
-    for i in range(points.shape[0]):
-        #get current points
-        point = points[i]
-        #if it is an outlier add it to the list with NaN as value for the properties
-        if predictions[i] > 1:
-
-            data.append(np.array([times[i], np.nan, np.nan, np.nan]))
-
-        else: #otherwise convert to ECEF and add
-            lon, lat, alt = point[0], point[1], point[2]
-            x, y, z = pyproj.transform(lla, ecef, lon, lat, alt, radians=False)
-
-            data.append(np.array([times[i], x, y, z]))
-
-    #convert to numpy array and create a Pandas dataframe    
-    data = np.array(data)
-    points_df = pd.DataFrame(data = data, columns=['t', 'x', 'y', 'z'])
-
-    #convert timestamps column to date time format
-    points_df['t'] = pd.to_datetime(points_df['t'])
-
-    #removing timezone infos
-    points_df['t'] = points_df['t'].dt.tz_localize(None)
-
-    #convert to numeric
-    points_df['x'] = pd.to_numeric(points_df['x'], errors='coerce')
-    points_df['y'] = pd.to_numeric(points_df['y'], errors='coerce')
-    points_df['z'] = pd.to_numeric(points_df['z'], errors='coerce')
-
-    #set index
-    points_df = points_df.set_index('t')
-
-    #interpolate using time (linear interpolation based on time)
-    points_df = points_df.interpolate(method='time')
-
-    #array that will host the corrected points converted back to lat long
-    points = []
-
-
-    #create final array by converting back each point to lat/long
-    for index, row in points_df.iterrows():
-        lon, lat, alt = pyproj.transform(ecef, lla, row['x'], row['y'], row['z'], radians=False)
-        points.append(np.array([lon, lat, alt]))
-
-    #return results and 
-    return np.array(points), np.array(times)
-
-#correct the trace using spline interpolation
-def correct_using_spline(points, times, predictions, order):
-    data = []
-
-    #convert all points to ECEF
-    for i in range(points.shape[0]):
-
-        point = points[i]
-
-        #but if the point is an outlier use NaN instead of the values
-        if predictions[i] > 1:
-
-            data.append(np.array([times[i], np.nan, np.nan, np.nan]))
-
-        else: #otherwise the real values
-            lon, lat, alt = point[0], point[1], point[2]
-            x, y, z = pyproj.transform(lla, ecef, lon, lat, alt, radians=False)
-
-            data.append(np.array([times[i], x, y, z]))
-
-    #add data to dataframe
-    data = np.array(data)
-    points_df = pd.DataFrame(data = data, columns=['t', 'x', 'y', 'z'])
-
-    #convert timestamps column to date time format
-    points_df['t'] = pd.to_datetime(points_df['t'])
-
-    #removing timezone infos
-    points_df['t'] = points_df['t'].dt.tz_localize(None)
-
-    #convert time to timedelta, to keep correct distance between points during interpolation
-    points_df['t_i'] = pd.to_timedelta(points_df.t).dt.total_seconds().astype(int)
-
-    #convert to numeric
-    points_df['x'] = pd.to_numeric(points_df['x'], errors='coerce')
-    points_df['y'] = pd.to_numeric(points_df['y'], errors='coerce')
-    points_df['z'] = pd.to_numeric(points_df['z'], errors='coerce')
-
-    #set index to our time delta column
-    points_df = points_df.set_index('t_i')
-
-    #interpolate using spline and the desired order
-    points_df = points_df.interpolate(method='spline', order=order, limit_direction='both')
-
-    #convert back to lat long
-    final_points = []
-
-
-    #create final array converting back to lat/long
-    for index, row in points_df.iterrows():
-        if not row.isna().any():
-            lon, lat, alt = pyproj.transform(ecef, lla, row['x'], row['y'], row['z'], radians=False)
-            final_points.append(np.array([lon, lat, alt]))
-
-    final_points = np.array(final_points)
-
-    return final_points, times
-
+    return np.array(points), np.array(coordTimes), np.array(reduced_deltas), np.array(reduced_predictions), np.array(original_points), np.array(original_times)  
 
 #correct the trace by applying a Kalman Filter (https://en.wikipedia.org/wiki/Kalman_filter) over the entire trace
 #Kalman filters are created and manages using the Filterpy library (https://filterpy.readthedocs.io/en/latest/)