Merge branch 'v0.2.0'

README.md

@@ -460,6 +460,19 @@ In case of any issues with the installation and usage of `mapillary_tools`, chec
 #### Execution
  - Windows users sometimes have trouble with the bare execution of `mapillary_tools`, since it is not inserted in the PATH automatically.
  If you are trying to execute `mapillary_tools` on Windows and dont have its path inserted in the PATH, make sure you execute the installed executable under Pythons scripts, for example `C:\python27\Scripts`. Due to the Python package naming convention, the package and the directory with the modules are also called `mapillary_tools`, so users often mistakenly try to run those instead of the executable called `mapillary_tools`, located in `mapillary_tools/mapillary_tools/bin`.
+ - Execution issues can occur in case the executable is saved with additional characters, like for example `\r`, which results in error message `env: python\r: No such file or directory`. In that case the executable needs to be edited to remove additional characters as suggested [here](https://stackoverflow.com/questions/19425857/env-python-r-no-such-file-or-directory), one example:
+
+ in terminal:
+
+ ```
+ sudo vim /usr/local/bin/mapillary_tools
+ ```
+then in vim editor:
+
+```
+ :set ff=unix
+ :wq
+ ```
 
 #### Run time issues
  - HTTP Errors can occur due to poor network connection or high load on the import pipeline. In most cases the images eventually get uploaded regardless. But in some cases HTTP Errors can occur due to authentication issues, which can be resolved by either removing the config file with the users credentials, located in `~/.config/mapillary/config` or running the `authenticate` command available under advanced usage of `mapillary_tools`.

mapillary_tools/__init__.py

@@ -21,5 +21,6 @@
 import commands
 import process_csv
 import interpolation
+import post_process
 
-VERSION = "0.1.4"
+VERSION = "0.1.6"

mapillary_tools/commands/__init__.py

@@ -13,6 +13,7 @@
 from . import process_csv
 from . import authenticate
 from . import interpolate
+from . import post_process
 
 mapillary_tools_advanced_commands = [
     sample_video,
@@ -26,8 +27,10 @@
     exif_insert,
     process_csv,
     authenticate,
-    interpolate
+    interpolate,
+    post_process
 ]
+
 mapillary_tools_commands = [
     process,
     upload,

mapillary_tools/commands/extract_import_meta_data.py

@@ -27,6 +27,10 @@ def add_basic_arguments(self, parser):
                             choices=[0, 90, 180, 270], type=int, default=None, required=False)
         parser.add_argument(
             "--GPS_accuracy", help="GPS accuracy in meters. Note this input has precedence over the input read from the import source file.", default=None, required=False)
+        parser.add_argument(
+            "--camera_uuid", help="Custom string used to differentiate different captures taken with the same camera make and model.", default=None, required=False)
+        parser.add_argument('--custom_meta_data', help='Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234"',
+                            default=None, required=False)
 
     def add_advanced_arguments(self, parser):
         pass

mapillary_tools/commands/post_process.py

@@ -0,0 +1,42 @@
+import inspect
+from mapillary_tools.post_process import post_process
+
+
+class Command:
+    name = 'post_process'
+    help = 'Post process tool : Post process for a given import path, including import summary and grouping/moving based on import status.'
+
+    def add_basic_arguments(self, parser):
+
+        parser.add_argument(
+            '--skip_subfolders', help='Skip all subfolders and import only the images in the given directory path.', action='store_true', default=False, required=False)
+
+    def add_advanced_arguments(self, parser):
+        # video file
+        parser.add_argument('--video_file', help='Provide the path to a video file or a directory containing a set of Blackvue video files.',
+                            action='store', required=False, default=None)
+
+        # post process
+        parser.add_argument('--summarize', help='Summarize import for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--move_images', help='Move images corresponding to import status.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--move_duplicates', help='Move images in case they were flagged as duplicates.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--move_uploaded', help='Move images in case they were uploaded successfully.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--save_as_json', help='Save summary or file status list in a json.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--list_file_status', help='List file status for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--push_images', help='Push images uploaded in given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--split_import_path', help='Provide the path where the images should be moved to based on the import status.',
+                            action='store', required=False, default=None)
+
+    def run(self, args):
+
+        vars_args = vars(args)
+
+        post_process(**({k: v for k, v in vars_args.iteritems()
+                         if k in inspect.getargspec(post_process).args}))

mapillary_tools/commands/process.py

@@ -5,6 +5,7 @@
 from mapillary_tools.process_sequence_properties import process_sequence_properties
 from mapillary_tools.process_upload_params import process_upload_params
 from mapillary_tools.insert_MAPJson import insert_MAPJson
+from mapillary_tools.post_process import post_process
 
 
 class Command:
@@ -44,10 +45,12 @@ def add_advanced_arguments(self, parser):
                             choices=[0, 90, 180, 270], type=int, default=None, required=False)
         parser.add_argument(
             "--GPS_accuracy", help="GPS accuracy in meters. Note this input has precedence over the input read from the import source file.", default=None, required=False)
+        parser.add_argument(
+            "--camera_uuid", help="Custom string used to differentiate different captures taken with the same camera make and model.", default=None, required=False)
 
         # geotagging
         parser.add_argument('--geotag_source', help='Provide the source of date/time and gps information needed for geotagging.', action='store',
-                            choices=['exif', 'gpx', 'gopro_video', 'nmea'], default="exif", required=False)
+                            choices=['exif', 'gpx', 'gopro_video', 'nmea', 'blackvue_videos'], default="exif", required=False)
         parser.add_argument(
             '--geotag_source_path', help='Provide the path to the file source of date/time and gps information needed for geotagging.', action='store',
             default=None, required=False)
@@ -83,6 +86,25 @@ def add_advanced_arguments(self, parser):
         parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory by adding suffix "_processed" to the import_path.',
                             action='store_true', default=False, required=False)
 
+        # post process
+        parser.add_argument('--summarize', help='Summarize import for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--move_images', help='Move images corresponding to sequence uuid, duplicate flag and upload status.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--save_as_json', help='Save summary or file status list in a json.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--list_file_status', help='List file status for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--push_images', help='Push images uploaded in given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument(
+            '--split_import_path', help='If splitting the import path into duplicates, sequences, success and failed uploads, provide a path for the splits.', default=None, required=False)
+
+        # add custom meta data in a form of a string consisting of a triplet
+        # "name,type,value"
+        parser.add_argument('--custom_meta_data', help='Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234"',
+                            default=None, required=False)
+
     def run(self, args):
 
         vars_args = vars(args)
@@ -104,4 +126,7 @@ def run(self, args):
         insert_MAPJson(**({k: v for k, v in vars_args.iteritems()
                            if k in inspect.getargspec(insert_MAPJson).args}))
 
-        print("Process done.")
+        print("Process done.")
+
+        post_process(**({k: v for k, v in vars_args.iteritems()
+                         if k in inspect.getargspec(post_process).args}))

mapillary_tools/commands/process_and_upload.py

@@ -47,6 +47,8 @@ def add_advanced_arguments(self, parser):
                             choices=[0, 90, 180, 270], type=int, default=None, required=False)
         parser.add_argument(
             "--GPS_accuracy", help="GPS accuracy in meters. Note this input has precedence over the input read from the import source file.", default=None, required=False)
+        parser.add_argument(
+            "--camera_uuid", help="Custom string used to differentiate different captures taken with the same camera make and model.", default=None, required=False)
 
         # geotagging
         parser.add_argument('--geotag_source', help='Provide the source of date/time and gps information needed for geotagging.', action='store',
@@ -84,6 +86,29 @@ def add_advanced_arguments(self, parser):
                             action='store_true', default=False, required=False)
         parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory by adding suffix "_processed" to the import_path.',
                             action='store_true', default=False, required=False)
+        parser.add_argument(
+            '--number_threads', help='Specify the number of upload threads.', type=int, default=None, required=False)
+        parser.add_argument(
+            '--max_attempts', help='Specify the maximum number of attempts to upload.', type=int, default=None, required=False)
+
+        # post process
+        parser.add_argument('--summarize', help='Summarize import for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--move_images', help='Move images corresponding to sequence uuid, duplicate flag and upload status.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--save_as_json', help='Save summary or file status list in a json.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--list_file_status', help='List file status for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--push_images', help='Push images uploaded in given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument(
+            '--split_import_path', help='If splitting the import path into duplicates, sequences, success and failed uploads, provide a path for the splits.', default=None, required=False)
+
+        # add custom meta data in a form of a string consisting of a triplet
+        # "name,type,value"
+        parser.add_argument('--custom_meta_data', help='Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234"',
+                            default=None, required=False)
 
     def run(self, args):
 
@@ -105,6 +130,10 @@ def run(self, args):
 
         insert_MAPJson(**({k: v for k, v in vars_args.iteritems()
                            if k in inspect.getargspec(insert_MAPJson).args}))
+        print("Process done.")
 
         upload(**({k: v for k, v in vars_args.iteritems()
                    if k in inspect.getargspec(upload).args}))
+
+        post_process(**({k: v for k, v in vars_args.iteritems()
+                         if k in inspect.getargspec(post_process).args}))

mapillary_tools/commands/upload.py

@@ -1,5 +1,6 @@
-
+import inspect
 from mapillary_tools.upload import upload
+from mapillary_tools.post_process import post_process
 
 
 class Command:
@@ -15,8 +16,29 @@ def add_basic_arguments(self, parser):
             '--skip_subfolders', help='Skip all subfolders and import only the images in the given directory path.', action='store_true', default=False, required=False)
 
     def add_advanced_arguments(self, parser):
-        pass
-
+        parser.add_argument(
+            '--number_threads', help='Specify the number of upload threads.', type=int, default=None, required=False)
+        parser.add_argument(
+            '--max_attempts', help='Specify the maximum number of attempts to upload.', type=int, default=None, required=False)
+
+        # post process
+        parser.add_argument('--summarize', help='Summarize import for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--move_images', help='Move images corresponding to sequence uuid, duplicate flag and upload status.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--save_as_json', help='Save summary or file status list in a json.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--list_file_status', help='List file status for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--push_images', help='Push images uploaded in given import path.',
+                            action='store_true', default=False, required=False)
+
     def run(self, args):
 
-        upload(**vars(args))
+        vars_args = vars(args)
+
+        upload(**({k: v for k, v in vars_args.iteritems()
+                   if k in inspect.getargspec(upload).args}))
+
+        post_process(**({k: v for k, v in vars_args.iteritems()
+                         if k in inspect.getargspec(post_process).args}))

mapillary_tools/commands/video_process.py

@@ -6,6 +6,7 @@
 from mapillary_tools.process_upload_params import process_upload_params
 from mapillary_tools.insert_MAPJson import insert_MAPJson
 from mapillary_tools.process_video import sample_video
+from mapillary_tools.post_process import post_process
 
 
 class Command:
@@ -55,10 +56,12 @@ def add_advanced_arguments(self, parser):
                             choices=[0, 90, 180, 270], type=int, default=None, required=False)
         parser.add_argument(
             "--GPS_accuracy", help="GPS accuracy in meters. Note this input has precedence over the input read from the import source file.", default=None, required=False)
+        parser.add_argument(
+            "--camera_uuid", help="Custom string used to differentiate different captures taken with the same camera make and model.", default=None, required=False)
 
         # geotagging
         parser.add_argument('--geotag_source', help='Provide the source of date/time and gps information needed for geotagging.', action='store',
-                            choices=['exif', 'gpx', 'gopro_video', 'nmea', 'blackvue'], default="exif", required=False)
+                            choices=['exif', 'gpx', 'gopro_video', 'nmea', 'blackvue_videos'], default="exif", required=False)
         parser.add_argument(
             '--geotag_source_path', help='Provide the path to the file source of date/time and gps information needed for geotagging.', action='store',
             default=None, required=False)
@@ -92,6 +95,24 @@ def add_advanced_arguments(self, parser):
                             action='store_true', default=False, required=False)
         parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory by adding suffix "_processed" to the import_path.',
                             action='store_true', default=False, required=False)
+        # post process
+        parser.add_argument('--summarize', help='Summarize import for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--move_images', help='Move images corresponding to sequence uuid, duplicate flag and upload status.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--save_as_json', help='Save summary or file status list in a json.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--list_file_status', help='List file status for given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--push_images', help='Push images uploaded in given import path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument(
+            '--split_import_path', help='If splitting the import path into duplicates, sequences, success and failed uploads, provide a path for the splits.', default=None, required=False)
+
+        # add custom meta data in a form of a string consisting of a triplet
+        # "name,type,value"
+        parser.add_argument('--custom_meta_data', help='Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234"',
+                            default=None, required=False)
 
     def run(self, args):
 
@@ -117,3 +138,8 @@ def run(self, args):
 
         insert_MAPJson(**({k: v for k, v in vars_args.iteritems()
                            if k in inspect.getargspec(insert_MAPJson).args}))
+
+        print("Process done.")
+
+        post_process(**({k: v for k, v in vars_args.iteritems()
+                         if k in inspect.getargspec(post_process).args}))