Support calibrating shape in mocap tracker. (#572)

Summary:
Pull Request resolved: #572

We want to experiment with calibration shape and markers simultaneously.

Reviewed By: jeongseok-meta

Differential Revision: D81521304

fbshipit-source-id: c46bde71708e760bcf5740b18ed254ebb98f6493

Author: cdtwigg

momentum/marker_tracking/marker_tracker.cpp

@@ -23,6 +23,7 @@
 #include "momentum/common/log.h"
 #include "momentum/common/progress_bar.h"
 #include "momentum/marker_tracking/tracker_utils.h"
+#include "momentum/math/mesh.h"
 #include "momentum/solver/gauss_newton_solver.h"
 #include "momentum/solver/solver.h"
 
@@ -210,7 +211,7 @@ Eigen::MatrixXf trackSequence(
   // (ie. not time varying). The input globalParams indicate which parameters within universalParams
   // we want to solve for. globalParams is either a subset or all of universalParams.
   ParameterSet poseParams = pt.getPoseParameters();
-  ParameterSet universalParams = pt.getScalingParameters();
+  ParameterSet universalParams = pt.getScalingParameters() | pt.getBlendShapeParameters();
   const auto locatorSet =
       pt.getParameterSet("locators", true) | pt.getParameterSet("skinnedLocators", true);
   poseParams &= ~locatorSet;
@@ -727,6 +728,11 @@ void calibrateModel(
   } else {
     calibBodySetExtended = transformExtended.getScalingParameters();
     calibBodySet = transform.getScalingParameters();
+
+    if (config.calibShape) {
+      calibBodySetExtended |= transformExtended.getBlendShapeParameters();
+      calibBodySet |= transform.getBlendShapeParameters();
+    }
   }
 
   // special trackingConfig for initialization: zero out smoothness and collision
@@ -852,6 +858,9 @@ void calibrateModel(
     // below.
     std::tie(identity.v, character.locators, character.skinnedLocators) =
         extractIdAndLocatorsFromParams(motion.col(0), solvingCharacter, character);
+    if (config.calibShape && solvingCharacter.blendShape && character.mesh) {
+      *character.mesh = extractBlendShapeFromParams(motion.col(0), solvingCharacter);
+    }
 
     // The sequence solve above could get stuck with euler singularity but per-frame solve could get
     // it out. Pass in the first frame from previous solve as a better initial guess than the zero

momentum/marker_tracking/marker_tracker.h

@@ -45,6 +45,8 @@ struct CalibrationConfig : public BaseConfig {
   bool enforceFloorInFirstFrame = false;
   /// Name of a pose constraint set to use for the first frame
   std::string firstFramePoseConstraintSet;
+  /// Calibrate the character's shape
+  bool calibShape = false;
 };
 
 /// Configuration for pose tracking given a calibrated body and locators

momentum/marker_tracking/tracker_utils.cpp

@@ -7,6 +7,8 @@
 
 #include "momentum/marker_tracking/tracker_utils.h"
 
+#include "momentum/character/blend_shape.h"
+#include "momentum/character/blend_shape_skinning.h"
 #include "momentum/character/character.h"
 #include "momentum/character/inverse_parameter_transform.h"
 #include "momentum/character/parameter_limits.h"
@@ -451,6 +453,15 @@ std::tuple<Eigen::VectorXf, LocatorList, SkinnedLocatorList> extractIdAndLocator
       skinnedLocators};
 }
 
+Mesh extractBlendShapeFromParams(
+    const momentum::ModelParameters& param,
+    const momentum::Character& sourceCharacter) {
+  Mesh result = *sourceCharacter.mesh;
+  auto blendWeights = extractBlendWeights(sourceCharacter.parameterTransform, param);
+  result.vertices = sourceCharacter.blendShape->computeShape<float>(blendWeights);
+  return result;
+}
+
 void fillIdentity(
     const ParameterSet& idSet,
     const ModelParameters& identity,

momentum/marker_tracking/tracker_utils.h

@@ -58,6 +58,10 @@ extractIdAndLocatorsFromParams(
     const momentum::Character& sourceCharacter,
     const momentum::Character& targetCharacter);
 
+Mesh extractBlendShapeFromParams(
+    const momentum::ModelParameters& param,
+    const momentum::Character& sourceCharacter);
+
 void fillIdentity(
     const momentum::ParameterSet& idSet,
     const momentum::ModelParameters& identity,

pymomentum/marker_tracking/marker_tracking_pybind.cpp

@@ -118,7 +118,7 @@ PYBIND11_MODULE(marker_tracking, m) {
           "__repr__",
           [](const momentum::CalibrationConfig& self) {
             return fmt::format(
-                "CalibrationConfig(min_vis_percent={}, loss_alpha={}, max_iter={}, regularization={}, debug={}, calib_frames={}, major_iter={}, global_scale_only={}, locators_only={}, greedy_sampling={}, enforce_floor_in_first_frame={}, first_frame_pose_constraint_set=\"{}\")",
+                "CalibrationConfig(min_vis_percent={}, loss_alpha={}, max_iter={}, regularization={}, debug={}, calib_frames={}, major_iter={}, global_scale_only={}, locators_only={}, greedy_sampling={}, enforce_floor_in_first_frame={}, first_frame_pose_constraint_set=\"{}\", calib_shape={})",
                 self.minVisPercent,
                 self.lossAlpha,
                 self.maxIter,
@@ -130,7 +130,8 @@ PYBIND11_MODULE(marker_tracking, m) {
                 boolToString(self.locatorsOnly),
                 self.greedySampling,
                 boolToString(self.enforceFloorInFirstFrame),
-                self.firstFramePoseConstraintSet);
+                self.firstFramePoseConstraintSet,
+                boolToString(self.calibShape));
           })
       .def(
           py::init<
@@ -145,7 +146,8 @@ PYBIND11_MODULE(marker_tracking, m) {
               bool,
               size_t,
               bool,
-              std::string>(),
+              std::string,
+              bool>(),
           R"(Create a CalibrationConfig with specified parameters.
 
           :param min_vis_percent: Minimum percentage of visible markers to be used
@@ -171,7 +173,8 @@ PYBIND11_MODULE(marker_tracking, m) {
           py::arg("locators_only") = false,
           py::arg("greedy_sampling") = 0,
           py::arg("enforce_floor_in_first_frame") = false,
-          py::arg("first_frame_pose_constraint_set") = "")
+          py::arg("first_frame_pose_constraint_set") = "",
+          py::arg("calib_shape") = false)
       .def_readwrite(
           "calib_frames",
           &momentum::CalibrationConfig::calibFrames,
@@ -199,7 +202,11 @@ PYBIND11_MODULE(marker_tracking, m) {
       .def_readwrite(
           "first_frame_pose_constraint_set",
           &momentum::CalibrationConfig::firstFramePoseConstraintSet,
-          "Name of pose constraint set to use in first frame");
+          "Name of pose constraint set to use in first frame")
+      .def_readwrite(
+          "calib_shape",
+          &momentum::CalibrationConfig::calibShape,
+          "Calibrate shape parameters");
 
   auto trackingConfig =
       py::class_<momentum::TrackingConfig, momentum::BaseConfig>(