diff --git a/README.md b/README.md
index a6decaa..e83fe4a 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
## 3d-pose-baseline
+[](https://sourcerer.io/fame/ArashHosseini/ArashHosseini/3d-pose-baseline/links/0)[](https://sourcerer.io/fame/ArashHosseini/ArashHosseini/3d-pose-baseline/links/1)[](https://sourcerer.io/fame/ArashHosseini/ArashHosseini/3d-pose-baseline/links/2)[](https://sourcerer.io/fame/ArashHosseini/ArashHosseini/3d-pose-baseline/links/3)[](https://sourcerer.io/fame/ArashHosseini/ArashHosseini/3d-pose-baseline/links/4)[](https://sourcerer.io/fame/ArashHosseini/ArashHosseini/3d-pose-baseline/links/5)[](https://sourcerer.io/fame/ArashHosseini/ArashHosseini/3d-pose-baseline/links/6)[](https://sourcerer.io/fame/ArashHosseini/ArashHosseini/3d-pose-baseline/links/7)
+
This is the code for the paper
Julieta Martinez, Rayat Hossain, Javier Romero, James J. Little.
@@ -51,6 +53,77 @@ This will produce a visualization similar to this:
+
+### [openpose](https://github.com/CMU-Perceptual-Computing-Lab/openpose.git)/[tf-pose-estimation](https://github.com/ArashHosseini/tf-pose-estimation)/[keras_Realtime_Multi-Person_Pose_Estimation](https://github.com/ArashHosseini/keras_Realtime_Multi-Person_Pose_Estimation) to 3d-Pose-Baseline
+
+
+### Caffe
+
+1. setup [openpose](https://github.com/CMU-Perceptual-Computing-Lab/openpose.git) and use `--write_json` flag to export Pose Keypoints.
+
+or
+
+### Tensorflow
+
+2. fork [tf-pose-estimation](https://github.com/ArashHosseini/tf-pose-estimation) and add `--output_json` flag to export Pose Keypoints like `python run_webcam.py --model=mobilenet_thin --resize=432x368 --camera=0 --output_json /path/to/directory`, check [diff](https://github.com/ArashHosseini/tf-pose-estimation/commit/eb25b197b3c0ed2d424513dbbe2565e910a736d1)
+
+or
+
+### Keras
+
+3. fork [keras_Realtime_Multi-Person_Pose_Estimation](https://github.com/ArashHosseini/keras_Realtime_Multi-Person_Pose_Estimation) and use `python demo_image.py --image sample_images/p1.jpg` for single image or `python demo_camera.py` for webcam feed. check [keypoints diff](https://github.com/ArashHosseini/keras_Realtime_Multi-Person_Pose_Estimation/commit/b5c76a35239aa7496010ff7f5e0b5fc0a9cf59a0) and [webcam diff](https://github.com/ArashHosseini/keras_Realtime_Multi-Person_Pose_Estimation/commit/3e414e68047fd7575bd8832ba776b0b5a93f2eea) for more info.
+
+4. Download Pre-trained model below
+
+5. simply run
+
+`python src/openpose_3dpose_sandbox.py --camera_frame --residual --batch_norm --dropout 0.5 --max_norm --evaluateActionWise --use_sh --epochs 200 --load 4874200 --pose_estimation_json /path/to/json_directory --write_gif --gif_fps 24 `, optional `--verbose 3` for debug and for interpolation add `--interpolation` and use `--multiplier`.
+
+6. or for 'Real Time'
+
+`python3.5 src/openpose_3dpose_sandbox_realtime.py --camera_frame --residual --batch_norm --dropout 0.5 --max_norm --evaluateActionWise --use_sh --epochs 200 --load 4874200 --pose_estimation_json /path/to/json_directory `
+
+
+### Export to DCC application and build skeleton
+
+
+
+1. use `--write_json` and `--write_images` flag to export keypoints and frame image from openpose, image will be used as imageplane inside maya.
+2. run `python src/openpose_3dpose_sandbox.py --camera_frame --residual --batch_norm --dropout 0.5 --max_norm --evaluateActionWise --use_sh --epochs 200 --load 4874200 --pose_estimation_json /path/to/json_directory --write_gif --gif_fps 24 `.
+3. for interpolation add `--interpolation` and use `--multiplier 0.5`.
+
+3d pose baseline now creates a json file `3d_data.json` with `x, y, z` coordinates inside maya folder
+
+4. change variables in `maya/maya_skeleton.py`. set `threed_pose_baseline` to main 3d-pose-baseline and `openpose_images` to same path as `--write_images` (step 1)
+5. open maya and import `maya/maya_skeleton.py`.
+
+`maya_skeleton.py` will load the data(`3d_data.json` and `2d_data.json`) to build a skeleton, parenting joints and setting the predicted animation provided by 3d-pose-baseline.
+
+6. create a imageplane and use created images inside `maya/image_plane/` as sequence.
+
+
+ 
+
+
+7. "real-time" stream, openpose > 3d-pose-baseline > maya (soon)
+
+8. implemented unity stream, check work of Zhenyu Chen [openpose_3d-pose-baseline_unity3d](https://github.com/zhenyuczy/openpose_3d-pose-baseline_unity3d)
+
+### Mapping
+
+
+ 
+
+
+### Result
+
+
+ 
+
+
+
+ 
+
### Training
To train a model with clean 2d detections, run:
diff --git a/imgs/dirty_plot.png b/imgs/dirty_plot.png
new file mode 100644
index 0000000..056e844
Binary files /dev/null and b/imgs/dirty_plot.png differ
diff --git a/imgs/interpolate_plot.png b/imgs/interpolate_plot.png
new file mode 100644
index 0000000..f173d86
Binary files /dev/null and b/imgs/interpolate_plot.png differ
diff --git a/imgs/interpolation.gif b/imgs/interpolation.gif
new file mode 100644
index 0000000..75ed9e6
Binary files /dev/null and b/imgs/interpolation.gif differ
diff --git a/imgs/maya.png b/imgs/maya.png
new file mode 100644
index 0000000..7b075e1
Binary files /dev/null and b/imgs/maya.png differ
diff --git a/imgs/maya_skl.gif b/imgs/maya_skl.gif
new file mode 100644
index 0000000..3dbdf50
Binary files /dev/null and b/imgs/maya_skl.gif differ
diff --git a/imgs/open_pose_input.gif b/imgs/open_pose_input.gif
new file mode 100644
index 0000000..ad834bd
Binary files /dev/null and b/imgs/open_pose_input.gif differ
diff --git a/imgs/output.gif b/imgs/output.gif
new file mode 100644
index 0000000..9d00b1b
Binary files /dev/null and b/imgs/output.gif differ
diff --git a/imgs/smooth_plot.png b/imgs/smooth_plot.png
new file mode 100644
index 0000000..2d47d3d
Binary files /dev/null and b/imgs/smooth_plot.png differ
diff --git a/maya/maya_skeleton.py b/maya/maya_skeleton.py
new file mode 100644
index 0000000..eb2c7cc
--- /dev/null
+++ b/maya/maya_skeleton.py
@@ -0,0 +1,170 @@
+import json
+import maya.cmds as cmds
+import pymel.core as pm
+import maya.OpenMaya as om
+import math
+import re
+import os
+
+
+#path to imageplane content
+openpose_images = "/home/flyn/git/3d-pose-baseline/test_images/" # replace it with abs path like "/path/to/bg_images"
+#path to 3d-pose-baseline
+threed_pose_baseline = "/home/flyn/git/3d-pose-baseline/"
+#for 3d use 3d_data.json and set three_dim to True
+input_json_path = [os.path.join(threed_pose_baseline, "maya/{0}.json".format(_data)) for _data in ["3d_data", "2d_data"]] # replace it with abs path like "/path/to/2d_data.json"
+
+
+
+def load_data(data, threed):
+ suffix = "threed" if threed else "twod"
+ # jnts to ignore
+ to_pass = [5,4,9,10,12] if threed else []
+ # locator driver grp
+ if not cmds.objExists("drivers_{0}".format(suffix)):
+ cmds.group(n="drivers_{0}".format(suffix), em=True)
+ for frame, jnt in data.iteritems():
+ if not cmds.objExists("anim_joint"):
+ cmds.group(n="anim_joint", em=True)
+ anim_grp_prj = cmds.group(n="anim_joint_2d", em=True)
+ cmds.parent(anim_grp_prj, "anim_joint")
+ for jnt_id, trans in jnt.iteritems():
+ if not int(jnt_id) in to_pass:
+ if not cmds.objExists("anim_jnt_driver_{0}_{1}".format(jnt_id, suffix)):
+ cmds.select(clear=True)
+ jnt = cmds.joint(n="jnt_{0}_{1}".format(jnt_id, suffix), relative=True)
+ cmds.setAttr("{0}.radius".format(jnt), 10)
+ cmds.setAttr("{0}.displayLocalAxis".format(jnt), 1)
+ # match same pos for first frame
+ if threed:
+ cmds.move(trans["translate"][0],trans["translate"][1], trans["translate"][2], jnt)
+ else:
+ cmds.move(trans["translate"][0],trans["translate"][1], jnt)
+ anim_grp_child = cmds.listRelatives("anim_joint", children=True) or []
+ if not jnt in anim_grp_child:
+ cmds.parent(jnt, "anim_joint")
+
+ if threed:
+ #create 2d projection
+ jnt_proj = cmds.duplicate(jnt, n="jnt_prj_{0}".format(jnt_id))
+ cmds.pointConstraint(jnt, jnt_proj, mo=False, skip="z")
+ cmds.setAttr("{0}.translateZ".format(jnt_proj[0]), 0)
+ cmds.parent(jnt_proj, "anim_joint_2d")
+
+ # driver locator
+ driver = cmds.spaceLocator(n="anim_jnt_driver_{0}_{1}".format(jnt_id, suffix))
+ # drive jnt with animated locator frim frame 0
+ cmds.pointConstraint(driver, jnt)
+ #if not driver in cmds.listRelatives("drivers_{0}".format(suffix), children=True) or []:
+ cmds.parent(driver, "drivers_{0}".format(suffix))
+ # add trans anim values to driver locator
+ cmds.setKeyframe("anim_jnt_driver_{0}_{1}".format(jnt_id, suffix), t=frame, v=trans["translate"][0], at='translateX')
+ cmds.setKeyframe("anim_jnt_driver_{0}_{1}".format(jnt_id, suffix), t=frame, v=trans["translate"][1], at='translateY')
+ if threed:
+ cmds.setKeyframe("anim_jnt_driver_{0}_{1}".format(jnt_id, suffix), t=frame, v=trans["translate"][2], at='translateZ')
+ # hacking 3d-pose-baseline coord. to maya
+ cmds.setAttr("drivers_{0}.rotateX".format(suffix), -110 if threed else -180)
+
+def parent_skeleton(jnt_mapping):
+ if not isinstance(jnt_mapping, dict):
+ raise Exception("expected dict, {0}".format(type(jnt_mapping)))
+ #parent jnts based on jnt_mapping
+ for body_part, jnt_map in jnt_mapping.iteritems():
+ for map_dict in jnt_map:
+ for parent_jnt, child_jnt in map_dict.iteritems():
+ if isinstance(child_jnt, list):
+ for child in child_jnt:
+ cmds.parent(child, parent_jnt)
+ else:
+ cmds.parent(child_jnt,parent_jnt)
+
+def get_rotate(p1, p2):
+ #calc rot for 3d json
+ punkt_a = om.MPoint(p1[0], p1[1], p1[2])
+ punkt_b = om.MPoint(p2[0], p2[1], p2[2])
+ rot_vector = punkt_a - punkt_b
+ world = om.MVector(0, 1, 0)
+ quat = om.MQuaternion(world, rot_vector, 1)
+ mat = om.MTransformationMatrix()
+ util = om.MScriptUtil()
+ util.createFromDouble(0, 0, 0)
+ rot_i = util.asDoublePtr()
+ mat.setRotation(rot_i, om.MTransformationMatrix.kXYZ)
+ mat = mat.asMatrix() * quat.asMatrix()
+ quat = om.MTransformationMatrix(mat).rotation()
+ m_rotation = om.MVector(math.degrees(quat.asEulerRotation().x),
+ math.degrees(quat.asEulerRotation().y),
+ math.degrees(quat.asEulerRotation().z)
+ )
+
+ return (m_rotation[0],m_rotation[1],m_rotation[2])
+
+def set_orient(data, jnt_mapping):
+ #set orient
+ for frame, jnt in data.iteritems():
+ cmds.currentTime(int(frame))
+ for body_part, jnt_map in jnt_mapping.iteritems():
+ for map_dict in jnt_map:
+ for parent_jnt, child_jnt in map_dict.iteritems():
+ if not isinstance(child_jnt, list):
+ p1 = cmds.xform(parent_jnt, q=True, t=True, ws=True)
+ p2 = cmds.xform(child_jnt, q=True, t=True, ws=True)
+ rotation = get_rotate(p1,p2)
+ cmds.setKeyframe(parent_jnt, t=frame, v=rotation[0], at='rotateX')
+ cmds.setKeyframe(parent_jnt, t=frame, v=rotation[1], at='rotateY')
+ cmds.setKeyframe(parent_jnt, t=frame, v=rotation[2], at='rotateZ')
+
+
+def main():
+ threed_jnt_mapping = { 'root': [{'jnt_11_threed': ['jnt_1_threed','jnt_6_threed']},
+ {'jnt_13_threed': ['jnt_17_threed', 'jnt_25_threed']}],
+ #left leg from jnt 6 to 8
+ "left_leg":[{"jnt_{0}_threed".format(n):"jnt_{0}_threed".format(n+1)} for n in range(6,8)],
+ #right leg from jnt 1 to 3
+ "right_leg":[{"jnt_{0}_threed".format(n):"jnt_{0}_threed".format(n+1)} for n in range(1,3)],
+ #left arm from jnt 17 to 19
+ "left_arm":[{"jnt_{0}_threed".format(n):"jnt_{0}_threed".format(n+1)} for n in range(17,19)],
+ #right arm from jnt 25 to 27
+ "right_arm":[{"jnt_{0}_threed".format(n):"jnt_{0}_threed".format(n+1)} for n in range(25,27)]}
+
+ twod_jnt_mapping = { 'root': [{'jnt_1_twod': ['jnt_11_twod','jnt_8_twod','jnt_5_twod', 'jnt_2_twod']}],
+ #left leg from jnt 11 to 13
+ "left_leg":[{"jnt_{0}_twod".format(n):"jnt_{0}_twod".format(n+1)} for n in range(11,13)],
+ #right leg from jnt 8 to 10
+ "right_leg":[{"jnt_{0}_twod".format(n):"jnt_{0}_twod".format(n+1)} for n in range(8,10)],
+ #left arm from jnt 5 to 7
+ "left_arm":[{"jnt_{0}_twod".format(n):"jnt_{0}_twod".format(n+1)} for n in range(5,7)],
+ #right arm from jnt 2 to 4
+ "right_arm":[{"jnt_{0}_twod".format(n):"jnt_{0}_twod".format(n+1)} for n in range(2,4)]}
+
+ #read 2 or 3d json payload
+ for _data in input_json_path:
+ with open(_data) as json_data:
+ # loaded data format:
+ # frames x jnts x (x,y,z)
+ # {frame:[jnt:"translate":[x,y,z], jnt:"translate":[x,y,z], ...], frame:...}
+ data = json.load(json_data)
+ #set orient on 3d
+ if "3d_data" in _data:
+ #load data and build locs and jnts
+ load_data(data, True) #true for 3d data
+ #parent jnts
+ parent_skeleton(threed_jnt_mapping)
+ #set orientation fo jnts
+ set_orient(data, threed_jnt_mapping)
+ else:
+ #load data and build locs and jnts
+ load_data(data, False)
+ #parent jnts
+ parent_skeleton(twod_jnt_mapping)
+
+ #convert imageplane
+ convert_images = False
+ if convert_images:
+ #set imageplane
+ #convert openpose --write_images output to valid padding for maya
+ for image_file in os.listdir(openpose_images):
+ file_path = os.path.join(openpose_images, image_file)
+ frame_idx = int(re.findall("(\d+)", image_file)[-1])
+ os.rename(file_path, os.path.join(threed_pose_baseline, "maya/image_plane/image.{0}.png".format(frame_idx)))
+ print "use", os.path.join(threed_pose_baseline, "maya/image_plane/"), " for imageplane."
diff --git a/src/openpose_3dpose_sandbox.py b/src/openpose_3dpose_sandbox.py
new file mode 100644
index 0000000..6dfe7d1
--- /dev/null
+++ b/src/openpose_3dpose_sandbox.py
@@ -0,0 +1,447 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import tensorflow as tf
+import data_utils
+import viz
+import re
+import cameras
+import json
+import os
+import time
+from predict_3dpose import create_model
+import cv2
+import imageio
+import logging
+import scipy as sp
+from pprint import pprint
+from scipy.interpolate import interp1d
+from scipy.interpolate import UnivariateSpline
+
+FLAGS = tf.app.flags.FLAGS
+
+order = [15, 12, 25, 26, 27, 17, 18, 19, 1, 2, 3, 6, 7, 8]
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def show_anim_curves(anim_dict, _plt):
+ val = np.array(list(anim_dict.values()))
+ for o in range(0,36,2):
+ x = val[:,o]
+ y = val[:,o+1]
+ _plt.plot(x, 'r--', linewidth=0.2)
+ _plt.plot(y, 'g', linewidth=0.2)
+ return _plt
+
+def read_openpose_json(smooth=True, *args):
+ # openpose output format:
+ # [x1,y1,c1,x2,y2,c2,...]
+ # ignore confidence score, take x and y [x1,y1,x2,y2,...]
+
+ logger.info("start reading json files")
+ #load json files
+ json_files = os.listdir(openpose_output_dir)
+ # check for other file types
+ json_files = sorted([filename for filename in json_files if filename.endswith(".json")])
+ cache = {}
+ smoothed = {}
+ ### extract x,y and ignore confidence score
+ for file_name in json_files:
+ logger.debug("reading {0}".format(file_name))
+ _file = os.path.join(openpose_output_dir, file_name)
+ if not os.path.isfile(_file): raise Exception("No file found!!, {0}".format(_file))
+ data = json.load(open(_file))
+ #take first person
+ _data = data["people"][0]["pose_keypoints_2d"] if "pose_keypoints_2d" in data["people"][0] else data["people"][0]["pose_keypoints"]
+ xy = []
+ if len(_data)>=53:
+ #openpose incl. confidence score
+ #ignore confidence score
+ for o in range(0,len(_data),3):
+ xy.append(_data[o])
+ xy.append(_data[o+1])
+ else:
+ #tf-pose-estimation
+ xy = _data
+
+ # get frame index from openpose 12 padding
+ frame_indx = re.findall("(\d+)", file_name)
+ logger.debug("found {0} for frame {1}".format(xy, str(int(frame_indx[-1]))))
+
+ #body_25 support, convert body_25 output format to coco
+ if len(_data)>54:
+ _xy = xy[0:19*2]
+ for x in range(len(xy)):
+ #del jnt 8
+ if x==8*2:
+ del _xy[x]
+ if x==8*2+1:
+ del _xy[x]
+ #map jnt 9 to 8
+ if x==9*2:
+ _xy[16] = xy[x]
+ _xy[17] = xy[x+1]
+ #map jnt 10 to 9
+ if x==10*2:
+ _xy[18] = xy[x]
+ _xy[19] = xy[x+1]
+ #map jnt 11 to 10
+ if x==11*2:
+ _xy[20] = xy[x]
+ _xy[21] = xy[x+1]
+ #map jnt 12 to 11
+ if x==12*2:
+ _xy[22] = xy[x]
+ _xy[23] = xy[x+1]
+ #map jnt 13 to 12
+ if x==13*2:
+ _xy[24] = xy[x]
+ _xy[25] = xy[x+1]
+ #map jnt 14 to 13
+ if x==14*2:
+ _xy[26] = xy[x]
+ _xy[27] = xy[x+1]
+ #map jnt 15 to 14
+ if x==15*2:
+ _xy[28] = xy[x]
+ _xy[29] = xy[x+1]
+ #map jnt 16 to 15
+ if x==16*2:
+ _xy[30] = xy[x]
+ _xy[31] = xy[x+1]
+ #map jnt 17 to 16
+ if x==17*2:
+ _xy[32] = xy[x]
+ _xy[33] = xy[x+1]
+ #map jnt 18 to 17
+ if x==18*2:
+ _xy[34] = xy[x]
+ _xy[35] = xy[x+1]
+ #coco
+ xy = _xy
+
+ #add xy to frame
+ cache[int(frame_indx[-1])] = xy
+
+ plt.figure(1)
+ drop_curves_plot = show_anim_curves(cache, plt)
+ pngName = 'gif_output/dirty_plot.png'
+ drop_curves_plot.savefig(pngName)
+ logger.info('writing gif_output/dirty_plot.png')
+
+ # exit if no smoothing
+ if not smooth:
+ # return frames cache incl. 18 joints (x,y)
+ return cache
+
+ if len(json_files) == 1:
+ logger.info("found single json file")
+ # return frames cache incl. 18 joints (x,y) on single image\json
+ return cache
+
+ if len(json_files) <= 8:
+ raise Exception("need more frames, min 9 frames/json files for smoothing!!!")
+
+ logger.info("start smoothing")
+
+ # create frame blocks
+ head_frame_block = [int(re.findall("(\d+)", o)[-1]) for o in json_files[:4]]
+ tail_frame_block = [int(re.findall("(\d+)", o)[-1]) for o in json_files[-4:]]
+
+ ### smooth by median value, n frames
+ for frame, xy in cache.items():
+ # create neighbor array based on frame index
+ forward, back = ([] for _ in range(2))
+
+ # joints x,y array
+ _len = len(xy) # 36
+
+ # create array of parallel frames (-33)
+ for neighbor in range(1,4):
+ # first n frames, get value of xy in postive lookahead frames(current frame + 3)
+ if frame in head_frame_block:
+ forward += cache[frame+neighbor]
+ # last n frames, get value of xy in negative lookahead frames(current frame - 3)
+ elif frame in tail_frame_block:
+ back += cache[frame-neighbor]
+ else:
+ # between frames, get value of xy in bi-directional frames(current frame -+ 3)
+ forward += cache[frame+neighbor]
+ back += cache[frame-neighbor]
+
+ # build frame range vector
+ frames_joint_median = [0 for i in range(_len)]
+ # more info about mapping in src/data_utils.py
+ # for each 18joints*x,y (x1,y1,x2,y2,...)~36
+ for x in range(0,_len,2):
+ # set x and y
+ y = x+1
+ if frame in head_frame_block:
+ # get vector of n frames forward for x and y, incl. current frame
+ x_v = [xy[x], forward[x], forward[x+_len], forward[x+_len*2]]
+ y_v = [xy[y], forward[y], forward[y+_len], forward[y+_len*2]]
+ elif frame in tail_frame_block:
+ # get vector of n frames back for x and y, incl. current frame
+ x_v =[xy[x], back[x], back[x+_len], back[x+_len*2]]
+ y_v =[xy[y], back[y], back[y+_len], back[y+_len*2]]
+ else:
+ # get vector of n frames forward/back for x and y, incl. current frame
+ # median value calc: find neighbor frames joint value and sorted them, use numpy median module
+ # frame[x1,y1,[x2,y2],..]frame[x1,y1,[x2,y2],...], frame[x1,y1,[x2,y2],..]
+ # ^---------------------|-------------------------^
+ x_v =[xy[x], forward[x], forward[x+_len], forward[x+_len*2],
+ back[x], back[x+_len], back[x+_len*2]]
+ y_v =[xy[y], forward[y], forward[y+_len], forward[y+_len*2],
+ back[y], back[y+_len], back[y+_len*2]]
+
+ # get median of vector
+ x_med = np.median(sorted(x_v))
+ y_med = np.median(sorted(y_v))
+
+ # holding frame drops for joint
+ if not x_med:
+ # allow fix from first frame
+ if frame:
+ # get x from last frame
+ x_med = smoothed[frame-1][x]
+ # if joint is hidden y
+ if not y_med:
+ # allow fix from first frame
+ if frame:
+ # get y from last frame
+ y_med = smoothed[frame-1][y]
+
+ logger.debug("old X {0} sorted neighbor {1} new X {2}".format(xy[x],sorted(x_v), x_med))
+ logger.debug("old Y {0} sorted neighbor {1} new Y {2}".format(xy[y],sorted(y_v), y_med))
+
+ # build new array of joint x and y value
+ frames_joint_median[x] = x_med
+ frames_joint_median[x+1] = y_med
+
+
+ smoothed[frame] = frames_joint_median
+
+ return smoothed
+
+
+def main(_):
+
+ smoothed = read_openpose_json()
+ plt.figure(2)
+ smooth_curves_plot = show_anim_curves(smoothed, plt)
+ #return
+ pngName = 'gif_output/smooth_plot.png'
+ smooth_curves_plot.savefig(pngName)
+ logger.info('writing gif_output/smooth_plot.png')
+
+ if FLAGS.interpolation:
+ logger.info("start interpolation")
+
+ framerange = len( smoothed.keys() )
+ joint_rows = 36
+ array = np.concatenate(list(smoothed.values()))
+ array_reshaped = np.reshape(array, (framerange, joint_rows) )
+
+ multiplier = FLAGS.multiplier
+ multiplier_inv = 1/multiplier
+
+ out_array = np.array([])
+ for row in range(joint_rows):
+ x = []
+ for frame in range(framerange):
+ x.append( array_reshaped[frame, row] )
+
+ frame = range( framerange )
+ frame_resampled = np.arange(0, framerange, multiplier)
+ spl = UnivariateSpline(frame, x, k=3)
+ #relative smooth factor based on jnt anim curve
+ min_x, max_x = min(x), max(x)
+ smooth_fac = max_x - min_x
+ smooth_resamp = 125
+ smooth_fac = smooth_fac * smooth_resamp
+ spl.set_smoothing_factor( float(smooth_fac) )
+ xnew = spl(frame_resampled)
+
+ out_array = np.append(out_array, xnew)
+
+ logger.info("done interpolating. reshaping {0} frames, please wait!!".format(framerange))
+
+ a = np.array([])
+ for frame in range( int( framerange * multiplier_inv ) ):
+ jnt_array = []
+ for jnt in range(joint_rows):
+ jnt_array.append( out_array[ jnt * int(framerange * multiplier_inv) + frame] )
+ a = np.append(a, jnt_array)
+
+ a = np.reshape(a, (int(framerange * multiplier_inv), joint_rows))
+ out_array = a
+
+ interpolate_smoothed = {}
+ for frame in range( int(framerange * multiplier_inv) ):
+ interpolate_smoothed[frame] = list( out_array[frame] )
+
+ plt.figure(3)
+ smoothed = interpolate_smoothed
+ interpolate_curves_plot = show_anim_curves(smoothed, plt)
+ pngName = 'gif_output/interpolate_{0}.png'.format(smooth_resamp)
+ interpolate_curves_plot.savefig(pngName)
+ logger.info('writing gif_output/interpolate_plot.png')
+
+ enc_in = np.zeros((1, 64))
+ enc_in[0] = [0 for i in range(64)]
+
+ actions = data_utils.define_actions(FLAGS.action)
+
+ SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
+ rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
+ train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
+ actions, FLAGS.data_dir)
+ train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
+ actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
+
+ device_count = {"GPU": 1}
+ png_lib = []
+ before_pose = None
+ with tf.Session(config=tf.ConfigProto(
+ device_count=device_count,
+ allow_soft_placement=True)) as sess:
+ #plt.figure(3)
+ batch_size = 128
+ model = create_model(sess, actions, batch_size)
+ iter_range = len(smoothed.keys())
+ export_units = {}
+ twod_export_units = {}
+ for n, (frame, xy) in enumerate(smoothed.items()):
+ logger.info("calc frame {0}/{1}".format(frame, iter_range))
+ # map list into np array
+ joints_array = np.zeros((1, 36))
+ joints_array[0] = [0 for i in range(36)]
+ for o in range(len(joints_array[0])):
+ #feed array with xy array
+ joints_array[0][o] = float(xy[o])
+
+ twod_export_units[frame]={}
+ for abs_b, __n in enumerate(range(0, len(xy),2)):
+ twod_export_units[frame][abs_b] = {"translate": [xy[__n],xy[__n+1]]}
+
+ _data = joints_array[0]
+ # mapping all body parts or 3d-pose-baseline format
+ for i in range(len(order)):
+ for j in range(2):
+ # create encoder input
+ enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
+ for j in range(2):
+ # Hip
+ enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2
+ # Neck/Nose
+ enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2
+ # Thorax
+ enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j]
+
+ # set spine
+ spine_x = enc_in[0][24]
+ spine_y = enc_in[0][25]
+
+ enc_in = enc_in[:, dim_to_use_2d]
+ mu = data_mean_2d[dim_to_use_2d]
+ stddev = data_std_2d[dim_to_use_2d]
+ enc_in = np.divide((enc_in - mu), stddev)
+
+ dp = 1.0
+ dec_out = np.zeros((1, 48))
+ dec_out[0] = [0 for i in range(48)]
+ _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False)
+ all_poses_3d = []
+ enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d)
+ poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d)
+ gs1 = gridspec.GridSpec(1, 1)
+ gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes.
+ plt.axis('off')
+ all_poses_3d.append( poses3d )
+ enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] )
+ subplot_idx, exidx = 1, 1
+ _max = 0
+ _min = 10000
+
+ for i in range(poses3d.shape[0]):
+ for j in range(32):
+ tmp = poses3d[i][j * 3 + 2]
+ poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
+ poses3d[i][j * 3 + 1] = tmp
+ if poses3d[i][j * 3 + 2] > _max:
+ _max = poses3d[i][j * 3 + 2]
+ if poses3d[i][j * 3 + 2] < _min:
+ _min = poses3d[i][j * 3 + 2]
+
+ for i in range(poses3d.shape[0]):
+ for j in range(32):
+ poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
+ poses3d[i][j * 3] += (spine_x - 630)
+ poses3d[i][j * 3 + 2] += (500 - spine_y)
+
+ # Plot 3d predictions
+ ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
+ ax.view_init(18, -70)
+
+ if FLAGS.cache_on_fail:
+ if np.min(poses3d) < -1000:
+ poses3d = before_pose
+
+ p3d = poses3d
+ logger.info("frame score {0}".format(np.min(poses3d)))
+ x,y,z = [[] for _ in range(3)]
+ if not poses3d is None:
+ to_export = poses3d.tolist()[0]
+ else:
+ to_export = [0.0 for _ in range(96)]
+ logger.info("export {0}".format(to_export))
+ for o in range(0, len(to_export), 3):
+ x.append(to_export[o])
+ y.append(to_export[o+1])
+ z.append(to_export[o+2])
+
+ export_units[frame]={}
+ for jnt_index, (_x, _y, _z) in enumerate(zip(x,y,z)):
+ export_units[frame][jnt_index] = {"translate": [_x, _y, _z]}
+ viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
+
+ pngName = 'png/pose_frame_{0}.png'.format(str(frame).zfill(12))
+ plt.savefig(pngName)
+ if FLAGS.write_gif:
+ png_lib.append(imageio.imread(pngName))
+
+ if FLAGS.cache_on_fail:
+ before_pose = poses3d
+
+ if FLAGS.write_gif:
+ if FLAGS.interpolation:
+ #take every frame on gif_fps * multiplier_inv
+ png_lib = np.array([png_lib[png_image] for png_image in range(0,len(png_lib), int(multiplier_inv)) ])
+ logger.info("creating Gif gif_output/animation.gif, please Wait!")
+ imageio.mimsave('gif_output/animation.gif', png_lib, fps=FLAGS.gif_fps)
+
+ _out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'maya/3d_data.json')
+ twod_out_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'maya/2d_data.json')
+ with open(_out_file, 'w') as outfile:
+ logger.info("exported maya json to {0}".format(_out_file))
+ json.dump(export_units, outfile)
+ with open(twod_out_file, 'w') as outfile:
+ logger.info("exported maya json to {0}".format(twod_out_file))
+ json.dump(twod_export_units, outfile)
+
+ logger.info("Done!".format(pngName))
+
+if __name__ == "__main__":
+
+ openpose_output_dir = FLAGS.pose_estimation_json
+
+ level = {0:logging.ERROR,
+ 1:logging.WARNING,
+ 2:logging.INFO,
+ 3:logging.DEBUG}
+
+ logger.setLevel(level[FLAGS.verbose])
+
+
+ tf.app.run()
diff --git a/src/openpose_3dpose_sandbox_realtime.py b/src/openpose_3dpose_sandbox_realtime.py
new file mode 100644
index 0000000..48f870b
--- /dev/null
+++ b/src/openpose_3dpose_sandbox_realtime.py
@@ -0,0 +1,237 @@
+
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.gridspec as gridspec
+import tensorflow as tf
+import data_utils
+import viz
+import re
+import cameras
+import json
+import os
+from predict_3dpose import create_model
+import cv2
+import imageio
+import time
+import logging
+import glob
+FLAGS = tf.app.flags.FLAGS
+
+order = [15, 12, 25, 26, 27, 17, 18, 19, 1, 2, 3, 6, 7, 8]
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+
+def main(_):
+ done = []
+
+ enc_in = np.zeros((1, 64))
+ enc_in[0] = [0 for i in range(64)]
+
+ actions = data_utils.define_actions(FLAGS.action)
+
+ SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
+ rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
+ train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
+ actions, FLAGS.data_dir)
+ train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
+ actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
+
+ device_count = {"GPU": 0}
+ png_lib = []
+ with tf.Session(config=tf.ConfigProto(
+ device_count=device_count,
+ allow_soft_placement=True)) as sess:
+ #plt.figure(3)
+ batch_size = 128
+ model = create_model(sess, actions, batch_size)
+ while True:
+ key = cv2.waitKey(1) & 0xFF
+ #logger.info("start reading data")
+ # check for other file types
+ list_of_files = glob.iglob("{0}/*".format(openpose_output_dir)) # You may use iglob in Python3
+ latest_file = ""
+ try:
+ latest_file = max(list_of_files, key=os.path.getctime)
+ except ValueError:
+ #empthy dir
+ pass
+ if not latest_file:
+ continue
+ try:
+ _file = file_name = latest_file
+ print (latest_file)
+ if not os.path.isfile(_file): raise Exception("No file found!!, {0}".format(_file))
+ data = json.load(open(_file))
+ #take first person
+ _data = data["people"][0]["pose_keypoints_2d"]
+ xy = []
+ if len(_data)>=53:
+ #openpose incl. confidence score
+ #ignore confidence score
+ for o in range(0,len(_data),3):
+ xy.append(_data[o])
+ xy.append(_data[o+1])
+ else:
+ #tf-pose-estimation
+ xy = _data
+
+ frame_indx = re.findall("(\d+)", file_name)
+ frame = int(frame_indx[-1])
+ logger.debug("found {0} for frame {1}".format(xy, str(frame)))
+
+ #body_25 support, convert body_25 output format to coco
+ if len(xy)>54:
+ _xy = xy[0:19*2]
+ for x in range(len(xy)):
+ #del jnt 8
+ if x==8*2:
+ del _xy[x]
+ if x==8*2+1:
+ del _xy[x]
+ #map jnt 9 to 8
+ if x==9*2:
+ _xy[16] = xy[x]
+ _xy[17] = xy[x+1]
+ #map jnt 10 to 9
+ if x==10*2:
+ _xy[18] = xy[x]
+ _xy[19] = xy[x+1]
+ #map jnt 11 to 10
+ if x==11*2:
+ _xy[20] = xy[x]
+ _xy[21] = xy[x+1]
+ #map jnt 12 to 11
+ if x==12*2:
+ _xy[22] = xy[x]
+ _xy[23] = xy[x+1]
+ #map jnt 13 to 12
+ if x==13*2:
+ _xy[24] = xy[x]
+ _xy[25] = xy[x+1]
+ #map jnt 14 to 13
+ if x==14*2:
+ _xy[26] = xy[x]
+ _xy[27] = xy[x+1]
+ #map jnt 15 to 14
+ if x==15*2:
+ _xy[28] = xy[x]
+ _xy[29] = xy[x+1]
+ #map jnt 16 to 15
+ if x==16*2:
+ _xy[30] = xy[x]
+ _xy[31] = xy[x+1]
+ #map jnt 17 to 16
+ if x==17*2:
+ _xy[32] = xy[x]
+ _xy[33] = xy[x+1]
+ #map jnt 18 to 17
+ if x==18*2:
+ _xy[34] = xy[x]
+ _xy[35] = xy[x+1]
+ #coco
+ xy = _xy
+
+ joints_array = np.zeros((1, 36))
+ joints_array[0] = [0 for i in range(36)]
+ for o in range(len(joints_array[0])):
+ #feed array with xy array
+ joints_array[0][o] = xy[o]
+ _data = joints_array[0]
+ # mapping all body parts or 3d-pose-baseline format
+ for i in range(len(order)):
+ for j in range(2):
+ # create encoder input
+ enc_in[0][order[i] * 2 + j] = _data[i * 2 + j]
+ for j in range(2):
+ # Hip
+ enc_in[0][0 * 2 + j] = (enc_in[0][1 * 2 + j] + enc_in[0][6 * 2 + j]) / 2
+ # Neck/Nose
+ enc_in[0][14 * 2 + j] = (enc_in[0][15 * 2 + j] + enc_in[0][12 * 2 + j]) / 2
+ # Thorax
+ enc_in[0][13 * 2 + j] = 2 * enc_in[0][12 * 2 + j] - enc_in[0][14 * 2 + j]
+
+ # set spine
+ spine_x = enc_in[0][24]
+ spine_y = enc_in[0][25]
+
+ enc_in = enc_in[:, dim_to_use_2d]
+ mu = data_mean_2d[dim_to_use_2d]
+ stddev = data_std_2d[dim_to_use_2d]
+ enc_in = np.divide((enc_in - mu), stddev)
+
+ dp = 1.0
+ dec_out = np.zeros((1, 48))
+ dec_out[0] = [0 for i in range(48)]
+ _, _, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False)
+ all_poses_3d = []
+ enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d)
+ poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d)
+ gs1 = gridspec.GridSpec(1, 1)
+ gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes.
+ plt.axis('off')
+ all_poses_3d.append( poses3d )
+ enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] )
+ subplot_idx, exidx = 1, 1
+ _max = 0
+ _min = 10000
+
+ for i in range(poses3d.shape[0]):
+ for j in range(32):
+ tmp = poses3d[i][j * 3 + 2]
+ poses3d[i][j * 3 + 2] = poses3d[i][j * 3 + 1]
+ poses3d[i][j * 3 + 1] = tmp
+ if poses3d[i][j * 3 + 2] > _max:
+ _max = poses3d[i][j * 3 + 2]
+ if poses3d[i][j * 3 + 2] < _min:
+ _min = poses3d[i][j * 3 + 2]
+
+ for i in range(poses3d.shape[0]):
+ for j in range(32):
+ poses3d[i][j * 3 + 2] = _max - poses3d[i][j * 3 + 2] + _min
+ poses3d[i][j * 3] += (spine_x - 630)
+ poses3d[i][j * 3 + 2] += (500 - spine_y)
+
+ # Plot 3d predictions
+ ax = plt.subplot(gs1[subplot_idx - 1], projection='3d')
+ ax.view_init(18, -70)
+ logger.debug(np.min(poses3d))
+ if np.min(poses3d) < -1000 and frame != 0:
+ poses3d = before_pose
+
+ p3d = poses3d
+
+ viz.show3Dpose(p3d, ax, lcolor="#9b59b6", rcolor="#2ecc71")
+ before_pose = poses3d
+ pngName = 'png/test_{0}.png'.format(str(frame))
+ plt.savefig(pngName)
+
+ #plt.show()
+ img = cv2.imread(pngName,0)
+ rect_cpy = img.copy()
+ cv2.imshow('3d-pose-baseline', rect_cpy)
+ done.append(file_name)
+ if key == ord('q'):
+ break
+ except Exception as e:
+ print (e)
+
+ sess.close()
+
+
+
+if __name__ == "__main__":
+
+ openpose_output_dir = FLAGS.pose_estimation_json
+
+ level = {0:logging.ERROR,
+ 1:logging.WARNING,
+ 2:logging.INFO,
+ 3:logging.DEBUG}
+
+ logger.setLevel(level[FLAGS.verbose])
+
+
+ tf.app.run()
diff --git a/src/predict_3dpose.py b/src/predict_3dpose.py
index 7c1b23e..5e29144 100644
--- a/src/predict_3dpose.py
+++ b/src/predict_3dpose.py
@@ -51,6 +51,15 @@
tf.app.flags.DEFINE_string("data_dir", "data/h36m/", "Data directory")
tf.app.flags.DEFINE_string("train_dir", "experiments", "Training directory.")
+# openpose
+tf.app.flags.DEFINE_string("pose_estimation_json", "/tmp/", "pose estimation json output directory, openpose or tf-pose-estimation")
+tf.app.flags.DEFINE_boolean("interpolation", False, "interpolate openpose json")
+tf.app.flags.DEFINE_float("multiplier", 0.1, "interpolation frame range")
+tf.app.flags.DEFINE_boolean("write_gif", False, "write final anim gif")
+tf.app.flags.DEFINE_integer("gif_fps", 30, "output gif framerate")
+tf.app.flags.DEFINE_integer("verbose", 2, "0:Error, 1:Warning, 2:INFO*(default), 3:debug")
+tf.app.flags.DEFINE_boolean("cache_on_fail", True, "caching last valid frame on invalid frame")
+
# Train or load
tf.app.flags.DEFINE_boolean("sample", False, "Set to True for sampling.")
tf.app.flags.DEFINE_boolean("use_cpu", False, "Whether to use the CPU")
diff --git a/src/viz.py b/src/viz.py
index ea43a31..c470c53 100644
--- a/src/viz.py
+++ b/src/viz.py
@@ -28,11 +28,10 @@ def show3Dpose(channels, ax, lcolor="#3498db", rcolor="#e74c3c", add_labels=Fals
I = np.array([1,2,3,1,7,8,1, 13,14,15,14,18,19,14,26,27])-1 # start points
J = np.array([2,3,4,7,8,9,13,14,15,16,18,19,20,26,27,28])-1 # end points
LR = np.array([1,1,1,0,0,0,0, 0, 0, 0, 0, 0, 0, 1, 1, 1], dtype=bool)
-
# Make connection matrix
for i in np.arange( len(I) ):
x, y, z = [np.array( [vals[I[i], j], vals[J[i], j]] ) for j in range(3)]
- ax.plot(x, y, z, lw=2, c=lcolor if LR[i] else rcolor)
+ ax.plot(x, y, z, marker='o', markersize=2, lw=1, c=lcolor if LR[i] else rcolor)
RADIUS = 750 # space around the subject
xroot, yroot, zroot = vals[0,0], vals[0,1], vals[0,2]
@@ -56,7 +55,7 @@ def show3Dpose(channels, ax, lcolor="#3498db", rcolor="#e74c3c", add_labels=Fals
ax.set_aspect('equal')
# Get rid of the panes (actually, make them white)
- white = (1.0, 1.0, 1.0, 0.0)
+ white = (1.0, 1.0, 0.1, 0.0)
ax.w_xaxis.set_pane_color(white)
ax.w_yaxis.set_pane_color(white)
# Keep z pane