support for new augmenter model

Examples/changeSessionMetadata.py

@@ -23,10 +23,12 @@
 developer use.
 
 The available options for metadata are:
-    - openSimModel: LaiUhlrich2022
-                    LaiUhlrich2022_shoulder
-    - posemodel:    openpose
-                    hrnet 
+    - openSimModel:     LaiUhlrich2022
+                        LaiUhlrich2022_shoulder
+    - posemodel:        openpose
+                        hrnet
+    - augmentermodel:   v0.2
+                        v0.3
 """
 import os
 import sys
@@ -38,5 +40,6 @@
 
 # Dictionary of metadata fields to change (see sessionMetadata.yaml).
 newMetadata = {'openSimModel':'LaiUhlrich2022_shoulder',
-               'posemodel':'hrnet'}
+               'posemodel':'hrnet',
+               'augmentermodel':'v0.3'}
 changeSessionMetadata(session_ids,newMetadata)

MarkerAugmenter/LSTM/v0.0/metadata.json

@@ -0,0 +1 @@
+{"reference_marker": "midHip"}

MarkerAugmenter/LSTM/v0.1_lower/metadata.json

@@ -0,0 +1 @@
+{"reference_marker": "midHip"}

MarkerAugmenter/LSTM/v0.1_upper/metadata.json

@@ -0,0 +1 @@
+{"reference_marker": "midHip"}

MarkerAugmenter/LSTM/v0.2_lower/metadata.json

@@ -0,0 +1 @@
+{"reference_marker": "midHip"}

MarkerAugmenter/LSTM/v0.2_upper/metadata.json

@@ -0,0 +1 @@
+{"reference_marker": "midHip"}

MarkerAugmenter/LSTM/v0.3_lower/metadata.json

@@ -0,0 +1 @@
+{"reference_marker": "midHip"}

MarkerAugmenter/LSTM/v0.3_lower/model.json

@@ -0,0 +1 @@
+{"class_name": "Sequential", "config": {"name": "sequential_2", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, null, 47], "dtype": "float32", "sparse": false, "ragged": false, "name": "lstm_8_input"}}, {"class_name": "LSTM", "config": {"name": "lstm_8", "trainable": true, "dtype": "float32", "batch_input_shape": [null, null, 47], "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "LSTM", "config": {"name": "lstm_9", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "LSTM", "config": {"name": "lstm_10", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "LSTM", "config": {"name": "lstm_11", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "TimeDistributed", "config": {"name": "time_distributed_2", "trainable": true, "dtype": "float32", "layer": {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 105, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}}}]}, "keras_version": "2.12.0", "backend": "tensorflow"}

MarkerAugmenter/LSTM/v0.3_upper/metadata.json

@@ -0,0 +1 @@
+{"reference_marker": "midHip"}

MarkerAugmenter/LSTM/v0.3_upper/model.json

@@ -0,0 +1 @@
+{"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, null, 23], "dtype": "float32", "sparse": false, "ragged": false, "name": "lstm_input"}}, {"class_name": "LSTM", "config": {"name": "lstm", "trainable": true, "dtype": "float32", "batch_input_shape": [null, null, 23], "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "LSTM", "config": {"name": "lstm_1", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "LSTM", "config": {"name": "lstm_2", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "LSTM", "config": {"name": "lstm_3", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "LSTM", "config": {"name": "lstm_4", "trainable": true, "dtype": "float32", "return_sequences": true, "return_state": false, "go_backwards": false, "stateful": false, "unroll": false, "time_major": false, "units": 128, "activation": "tanh", "recurrent_activation": "sigmoid", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "recurrent_initializer": {"class_name": "Orthogonal", "config": {"gain": 1.0, "seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "unit_forget_bias": true, "kernel_regularizer": null, "recurrent_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "recurrent_constraint": null, "bias_constraint": null, "dropout": 0.0, "recurrent_dropout": 0.0, "implementation": 2}}, {"class_name": "TimeDistributed", "config": {"name": "time_distributed", "trainable": true, "dtype": "float32", "layer": {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "units": 24, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}}}]}, "keras_version": "2.12.0", "backend": "tensorflow"}

MarkerAugmenter/LSTM/versions.txt

@@ -1,5 +1,4 @@
-Changelog:
-
-v0.2: DEFAULT - models trained with augmented dataset (data rotated about vertical axis) -  models 20 and 21 from openpose-augmenter repository.
-v0.1: separate models for lower and upper extremities  = models used for paper - models 13 and 16 from openpose-augmenter repository.
-v0.0: single model for both lower and upper extremities - model 14 from openpose-augmenter repository.
+v0.3: latest - models 77 and 93 from marker-augmenter repository.
+v0.2: DEFAULT - models 20 and 21 from openpose-augmenter repository.
+v0.1: OpenCap paper - models 13 and 16 from openpose-augmenter repository.
+v0.0: deprecated - model 14 from openpose-augmenter repository.

main.py

@@ -36,7 +36,7 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'],
          poseDetector='OpenPose', resolutionPoseDetection='default', 
          scaleModel=False, bbox_thr=0.8, augmenter_model='v0.2',
          genericFolderNames=False, offset=True, benchmark=False,
-         dataDir=None):
+         dataDir=None, overwriteAugmenterModel=False):
 
     # %% High-level settings.
     # Camera calibration.
@@ -86,6 +86,14 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'],
         else:
             poseDetector = 'OpenPose'
 
+    # If augmenter model defined through web app.
+    # If overwriteAugmenterModel is True, the augmenter model is the one
+    # passed as an argument to main(). This is useful for local testing.
+    if 'augmentermodel' in sessionMetadata and not overwriteAugmenterModel:
+        augmenterModel = sessionMetadata['augmentermodel']
+    else:
+        augmenterModel = augmenter_model
+
     # %% Paths to pose detector folder for local testing.
     if poseDetector == 'OpenPose':
         poseDetectorDirectory = getOpenPoseDirectory(isDocker)
@@ -330,10 +338,10 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'],
     else:
         postAugmentationDir = os.path.join(
             sessionDir, markerDataFolderName, 
-            'PostAugmentation_{}'.format(augmenter_model))
+            'PostAugmentation_{}'.format(augmenterModel))
 
     # Get augmenter model.
-    augmenterModel = (
+    augmenterModelName = (
         sessionMetadata['markerAugmentationSettings']['markerAugmenterModel'])
 
     # Set output file name.
@@ -344,10 +352,10 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'],
     else:
         if benchmark:
             pathAugmentedOutputFiles[trialName] = os.path.join(
-                    postAugmentationDir, trialName + "_" + augmenterModel +".trc")
+                    postAugmentationDir, trialName + "_" + augmenterModelName +".trc")
         else:
             pathAugmentedOutputFiles[trialName] = os.path.join(
-                    postAugmentationDir, trial_id + "_" + augmenterModel +".trc")
+                    postAugmentationDir, trial_id + "_" + augmenterModelName +".trc")
 
     if runMarkerAugmentation:
         os.makedirs(postAugmentationDir, exist_ok=True)    
@@ -357,8 +365,8 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'],
             vertical_offset = augmentTRC(
                 pathOutputFiles[trialName],sessionMetadata['mass_kg'], 
                 sessionMetadata['height_m'], pathAugmentedOutputFiles[trialName],
-                augmenterDir, augmenterModelName=augmenterModel,
-                augmenter_model=augmenter_model, offset=offset)
+                augmenterDir, augmenterModelName=augmenterModelName,
+                augmenter_model=augmenterModel, offset=offset)
         except Exception as e:
             if len(e.args) == 2: # specific exception
                 raise Exception(e.args[0], e.args[1])
@@ -479,11 +487,14 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'],
     if not extrinsicsTrial:
         pathSettings = os.path.join(postAugmentationDir, 
                                     'Settings_' + trial_id + '.yaml')
-        settings = {'poseDetector': poseDetector, 'resolutionPoseDetection':
-                    resolutionPoseDetection, 'augmenter_model': 
-                    augmenter_model, 'offset': offset, 'imageUpsampleFactor': 
-                    imageUpsampleFactor}
+        settings = {
+            'poseDetector': poseDetector, 
+            'resolutionPoseDetection': resolutionPoseDetection,
+            'augmenter_model': augmenterModel, 
+            'offset': offset, 
+            'imageUpsampleFactor': imageUpsampleFactor,
+            'openSimModel': sessionMetadata['openSimModel']}
         if poseDetector == 'mmpose':
             settings['bbox_thr']: str(bbox_thr)
         with open(pathSettings, 'w') as file:
-                yaml.dump(settings, file)
+            yaml.dump(settings, file)

utils.py

@@ -377,12 +377,16 @@ def getMetadataFromServer(session_id,justCheckerParams=False):
                     session_desc["posemodel"] = session['meta']['subject']['posemodel']
                 except:
                     session_desc["posemodel"] = 'openpose'
-                # This might happen if openSimModel was changed post data collection.
+                # This might happen if openSimModel/augmentermodel was changed post data collection.
                 if 'settings' in session['meta']:
                     try:
                         session_desc["openSimModel"] = session['meta']['settings']['openSimModel']
                     except:
                         session_desc["openSimModel"] = 'LaiUhlrich2022'
+                    try:
+                        session_desc["augmentermodel"] = session['meta']['settings']['augmentermodel']
+                    except:
+                        session_desc["augmentermodel"] = 'v0.2'
             else:                
                 subject_info = getSubjectJson(session['subject'])                
                 session_desc["subjectID"] = subject_info['name']
@@ -396,6 +400,10 @@ def getMetadataFromServer(session_id,justCheckerParams=False):
                     session_desc["openSimModel"] = session['meta']['settings']['openSimModel']
                 except:
                     session_desc["openSimModel"] = 'LaiUhlrich2022'
+                try:
+                    session_desc["augmentermodel"] = session['meta']['settings']['augmentermodel']
+                except:
+                    session_desc["augmentermodel"] = 'v0.2'
 
         if 'sessionWithCalibration' in session['meta'] and 'checkerboard' not in session['meta']:
             newSessionId = session['meta']['sessionWithCalibration']['id']
@@ -618,7 +626,7 @@ def changeSessionMetadata(session_ids,newMetaDict):
         for newMeta in newMetaDict:
             if not newMeta in addedKey:
                 print("Could not find {} in existing metadata, trying to add it.".format(newMeta))
-                settings_fields = ['framerate', 'posemodel', 'openSimModel']
+                settings_fields = ['framerate', 'posemodel', 'openSimModel', 'augmentermodel']
                 if newMeta in settings_fields:
                     existingMeta['settings'][newMeta] = newMetaDict[newMeta]
                     addedKey[newMeta] = newMetaDict[newMeta]
@@ -1219,6 +1227,29 @@ def getOpenPoseMarkers_lowerExtremity():
 
     return feature_markers, response_markers
 
+# Different order of markers compared to getOpenPoseMarkers_lowerExtremity 
+def getOpenPoseMarkers_lowerExtremity2():
+
+    feature_markers = [
+        "Neck", "RShoulder", "LShoulder", "RHip", "LHip", "RKnee", "LKnee",
+        "RAnkle", "LAnkle", "RHeel", "LHeel", "RSmallToe", "LSmallToe",
+        "RBigToe", "LBigToe"]
+
+    response_markers = [
+        'r.ASIS_study', 'L.ASIS_study', 'r.PSIS_study',
+        'L.PSIS_study', 'r_knee_study', 'r_mknee_study', 
+        'r_ankle_study', 'r_mankle_study', 'r_toe_study', 
+        'r_5meta_study', 'r_calc_study', 'L_knee_study', 
+        'L_mknee_study', 'L_ankle_study', 'L_mankle_study',
+        'L_toe_study', 'L_calc_study', 'L_5meta_study', 
+        'r_shoulder_study', 'L_shoulder_study', 'C7_study', 
+        'r_thigh1_study', 'r_thigh2_study', 'r_thigh3_study',
+        'L_thigh1_study', 'L_thigh2_study', 'L_thigh3_study',
+        'r_sh1_study', 'r_sh2_study', 'r_sh3_study', 'L_sh1_study',
+        'L_sh2_study', 'L_sh3_study', 'RHJC_study', 'LHJC_study']
+
+    return feature_markers, response_markers
+
 def getMMposeMarkers_lowerExtremity():
 
     # Here we replace RSmallToe_mmpose and LSmallToe_mmpose by RSmallToe and
@@ -1266,6 +1297,19 @@ def getMarkers_upperExtremity_noPelvis():
 
     return feature_markers, response_markers
 
+# Different order of markers compared to getMarkers_upperExtremity_noPelvis.
+def getMarkers_upperExtremity_noPelvis2():
+
+    feature_markers = [
+        "Neck", "RShoulder", "LShoulder", "RElbow", "LElbow", "RWrist",
+        "LWrist"]
+
+    response_markers = ["r_lelbow_study", "r_melbow_study", "r_lwrist_study",
+                        "r_mwrist_study", "L_lelbow_study", "L_melbow_study",
+                        "L_lwrist_study", "L_mwrist_study"]
+
+    return feature_markers, response_markers
+
 def delete_multiple_element(list_object, indices):
     indices = sorted(indices, reverse=True)
     for idx in indices: