From e94856e285e21a5f57235fa621a5e83acc7800d2 Mon Sep 17 00:00:00 2001 From: Chris Twigg Date: Mon, 27 Oct 2025 22:33:16 -0700 Subject: [PATCH] Switch SkinnedLocatorTriangleError to use MeshState (#713) Summary: Pull Request resolved: https://github.com/facebookresearch/momentum/pull/713 As title. Reviewed By: cstollmeta Differential Revision: D84999391 --- ...kinned_locator_triangle_error_function.cpp | 165 ++++++++---------- .../skinned_locator_triangle_error_function.h | 8 +- .../vertex_error_function.cpp | 3 + 3 files changed, 76 insertions(+), 100 deletions(-) diff --git a/momentum/character_solver/skinned_locator_triangle_error_function.cpp b/momentum/character_solver/skinned_locator_triangle_error_function.cpp index 4c91d23ada..8c8c166436 100644 --- a/momentum/character_solver/skinned_locator_triangle_error_function.cpp +++ b/momentum/character_solver/skinned_locator_triangle_error_function.cpp @@ -12,6 +12,7 @@ #include "momentum/character/blend_shape.h" #include "momentum/character/blend_shape_skinning.h" #include "momentum/character/linear_skinning.h" +#include "momentum/character/mesh_state.h" #include "momentum/character/skeleton_state.h" #include "momentum/character/skin_weights.h" #include "momentum/common/checks.h" @@ -33,9 +34,6 @@ SkinnedLocatorTriangleErrorFunctionT::SkinnedLocatorTriangleErrorFunctionT( MT_THROW_IF( type != VertexConstraintType::Position && type != VertexConstraintType::Plane, "SkinnedLocatorTriangleErrorFunction only supports Position and Plane constraint types"); - - this->neutralMesh_ = std::make_unique>(character_in.mesh->template cast()); - this->restMesh_ = std::make_unique>(character_in.mesh->template cast()); } template @@ -161,55 +159,52 @@ std::array, 3> compute_d_targetPos_d_vertexPos( template double SkinnedLocatorTriangleErrorFunctionT::getError( const ModelParametersT& modelParameters, - const SkeletonStateT& /*state*/, - const MeshStateT& /* meshState */) { + const SkeletonStateT& /* state */, + const MeshStateT& meshState) { MT_PROFILE_FUNCTION(); - updateMeshes(modelParameters); + MT_CHECK_NOTNULL(meshState.restMesh_); - // loop over all constraints and calculate the error double error = 0.0; if (constraintType_ == VertexConstraintType::Position) { - for (size_t i = 0; i < constraints_.size(); ++i) { - const SkinnedLocatorTriangleConstraintT& constr = constraints_[i]; - + for (const auto& constr : constraints_) { // Get the skinned locator position const Eigen::Vector3 locatorRestPos = getLocatorRestPosition(modelParameters, constr.locatorIndex); // Get the target position on the triangle - const Eigen::Vector3 tgtPoint = computeTargetPosition(*this->restMesh_, constr); + const Eigen::Vector3 tgtPoint = computeTargetPosition(*meshState.restMesh_, constr); // Calculate position error const Eigen::Vector3 diff = locatorRestPos - tgtPoint; error += constr.weight * diff.squaredNorm(); } - } else if (constraintType_ == VertexConstraintType::Plane) { - for (size_t i = 0; i < constraints_.size(); ++i) { - const SkinnedLocatorTriangleConstraintT& constr = constraints_[i]; - + } else { + MT_CHECK(constraintType_ == VertexConstraintType::Plane); + for (const auto& constr : constraints_) { // Get the skinned locator position const Eigen::Vector3 locatorRestPos = getLocatorRestPosition(modelParameters, constr.locatorIndex); // Get the target position and normal - const Eigen::Vector3 targetNormal = computeTargetTriangleNormal(*this->restMesh_, constr); + const Eigen::Vector3 targetNormal = + computeTargetTriangleNormal(*meshState.restMesh_, constr); const Eigen::Vector3 tgtPoint = - computeTargetBaryPosition(*this->restMesh_, constr) + constr.depth * targetNormal; + computeTargetBaryPosition(*meshState.restMesh_, constr) + constr.depth * targetNormal; // Calculate plane error (projection onto normal) const T dist = targetNormal.dot(locatorRestPos - tgtPoint); error += constr.weight * dist * dist; } } - // return error return error * this->weight_; } template double SkinnedLocatorTriangleErrorFunctionT::calculatePositionGradient( const ModelParametersT& modelParameters, + const MeshStateT& meshState, const SkinnedLocatorTriangleConstraintT& constr, Eigen::Ref> gradient) const { // Get the skinned locator position @@ -217,7 +212,7 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePositionGradient( getLocatorRestPosition(modelParameters, constr.locatorIndex); // Get the target position on the triangle - const Eigen::Vector3 tgtPoint = computeTargetPosition(*this->restMesh_, constr); + const Eigen::Vector3 tgtPoint = computeTargetPosition(*meshState.restMesh_, constr); // Calculate position error const Eigen::Vector3 diff = locatorRestPos - tgtPoint; @@ -227,7 +222,7 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePositionGradient( // Get the derivative of the target position with respect to the triangle vertices const auto d_targetPos_d_tgtTriVertexPos = - compute_d_targetPos_d_vertexPos(constr, *this->restMesh_); + compute_d_targetPos_d_vertexPos(constr, *meshState.restMesh_); // Apply gradient for locator parameter if it exists int paramIndex = getSkinnedLocatorParameterIndex(constr.locatorIndex); @@ -271,6 +266,7 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePositionGradient( template double SkinnedLocatorTriangleErrorFunctionT::calculatePlaneGradient( const ModelParametersT& modelParameters, + const MeshStateT& meshState, const SkinnedLocatorTriangleConstraintT& constr, Eigen::Ref> gradient) const { // Get the skinned locator position @@ -278,9 +274,9 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePlaneGradient( getLocatorRestPosition(modelParameters, constr.locatorIndex); // Get the target position and normal - const Eigen::Vector3 targetNormal = computeTargetTriangleNormal(*this->restMesh_, constr); + const Eigen::Vector3 targetNormal = computeTargetTriangleNormal(*meshState.restMesh_, constr); const Eigen::Vector3 tgtPoint = - computeTargetBaryPosition(*this->restMesh_, constr) + constr.depth * targetNormal; + computeTargetBaryPosition(*meshState.restMesh_, constr) + constr.depth * targetNormal; // Calculate plane error (projection onto normal) const Vector3 diff = locatorRestPos - tgtPoint; @@ -289,9 +285,9 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePlaneGradient( // Get the derivative of the target position with respect to the triangle vertices const auto d_targetPos_d_tgtTriVertexPos = - compute_d_targetPos_d_vertexPos(constr, *this->restMesh_); + compute_d_targetPos_d_vertexPos(constr, *meshState.restMesh_); const auto d_targetNormal_d_tgtTriVertexPos = - compute_d_targetNormal_d_vertexPos(constr, *this->restMesh_); + compute_d_targetNormal_d_vertexPos(constr, *meshState.restMesh_); // Apply gradient for locator parameter if it exists int paramIndex = getSkinnedLocatorParameterIndex(constr.locatorIndex); @@ -335,6 +331,7 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePlaneGradient( template double SkinnedLocatorTriangleErrorFunctionT::calculatePositionJacobian( const ModelParametersT& modelParameters, + const MeshStateT& meshState, const SkinnedLocatorTriangleConstraintT& constr, Ref> jac, Ref> res) const { @@ -343,7 +340,7 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePositionJacobian( getLocatorRestPosition(modelParameters, constr.locatorIndex); // Get the target position on the triangle - const Eigen::Vector3 tgtPoint = computeTargetPosition(*this->restMesh_, constr); + const Eigen::Vector3 tgtPoint = computeTargetPosition(*meshState.restMesh_, constr); // Calculate position error const Eigen::Vector3 diff = locatorRestPos - tgtPoint; @@ -354,7 +351,7 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePositionJacobian( // Get the derivative of the target position with respect to the triangle vertices const auto d_targetPos_d_tgtTriVertexPos = - compute_d_targetPos_d_vertexPos(constr, *this->restMesh_); + compute_d_targetPos_d_vertexPos(constr, *meshState.restMesh_); // Apply Jacobian for locator parameter if it exists int paramIndex = getSkinnedLocatorParameterIndex(constr.locatorIndex); @@ -396,6 +393,7 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePositionJacobian( template double SkinnedLocatorTriangleErrorFunctionT::calculatePlaneJacobian( const ModelParametersT& modelParameters, + const MeshStateT& meshState, const SkinnedLocatorTriangleConstraintT& constr, Ref> jac, T& res) const { @@ -404,23 +402,22 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePlaneJacobian( getLocatorRestPosition(modelParameters, constr.locatorIndex); // Get the target position and normal - const Eigen::Vector3 targetNormal = computeTargetTriangleNormal(*this->restMesh_, constr); + const Eigen::Vector3 targetNormal = computeTargetTriangleNormal(*meshState.restMesh_, constr); const Eigen::Vector3 tgtPoint = - computeTargetBaryPosition(*this->restMesh_, constr) + constr.depth * targetNormal; + computeTargetBaryPosition(*meshState.restMesh_, constr) + constr.depth * targetNormal; // Calculate plane error (projection onto normal) const Eigen::Vector3 diff = locatorRestPos - tgtPoint; const T dist = targetNormal.dot(diff); - // Set residual const T wgt = std::sqrt(constr.weight * this->weight_); res = wgt * dist; // Get the derivative of the target position with respect to the triangle vertices const auto d_targetPos_d_tgtTriVertexPos = - compute_d_targetPos_d_vertexPos(constr, *this->restMesh_); + compute_d_targetPos_d_vertexPos(constr, *meshState.restMesh_); const auto d_targetNormal_d_tgtTriVertexPos = - compute_d_targetNormal_d_vertexPos(constr, *this->restMesh_); + compute_d_targetNormal_d_vertexPos(constr, *meshState.restMesh_); // Apply Jacobian for locator parameter if it exists int paramIndex = getSkinnedLocatorParameterIndex(constr.locatorIndex); @@ -463,58 +460,68 @@ double SkinnedLocatorTriangleErrorFunctionT::calculatePlaneJacobian( template double SkinnedLocatorTriangleErrorFunctionT::getGradient( const ModelParametersT& modelParameters, - const SkeletonStateT& /*state*/, - const MeshStateT& /* meshState */, + const SkeletonStateT& /* state */, + const MeshStateT& meshState, Eigen::Ref> gradient) { - updateMeshes(modelParameters); + MT_CHECK_NOTNULL(meshState.restMesh_); - double error = 0.0; + double error = 0; - for (const auto& constraint : constraints_) { - if (constraintType_ == VertexConstraintType::Position) { - error += calculatePositionGradient(modelParameters, constraint, gradient); - } else if (constraintType_ == VertexConstraintType::Plane) { - error += calculatePlaneGradient(modelParameters, constraint, gradient); + if (constraintType_ == VertexConstraintType::Position) { + for (size_t iCons = 0; iCons < constraints_.size(); ++iCons) { + error += calculatePositionGradient(modelParameters, meshState, constraints_[iCons], gradient); + } + } else { + MT_CHECK(constraintType_ == VertexConstraintType::Plane); + for (size_t iCons = 0; iCons < constraints_.size(); ++iCons) { + error += calculatePlaneGradient(modelParameters, meshState, constraints_[iCons], gradient); } } - return error * this->weight_; + return this->weight_ * error; } template double SkinnedLocatorTriangleErrorFunctionT::getJacobian( const ModelParametersT& modelParameters, - const SkeletonStateT& /*state*/, - const MeshStateT& /* meshState */, + const SkeletonStateT& /* state */, + const MeshStateT& meshState, Eigen::Ref> jacobian, Eigen::Ref> residual, int& usedRows) { MT_CHECK( jacobian.cols() == static_cast(this->parameterTransform_.transform.cols())); + MT_CHECK(jacobian.rows() >= (Eigen::Index)(1 * constraints_.size())); + MT_CHECK(residual.rows() >= (Eigen::Index)(1 * constraints_.size())); - updateMeshes(modelParameters); + MT_CHECK( + meshState.restMesh_, "MeshState must have rest mesh for SkinnedLocatorTriangleErrorFunction"); - double error = 0.0; - usedRows = 0; + double error = 0; - for (size_t iCons = 0; iCons < constraints_.size(); ++iCons) { - if (constraintType_ == VertexConstraintType::Position) { + if (constraintType_ == VertexConstraintType::Position) { + for (size_t i = 0; i < constraints_.size(); ++i) { error += calculatePositionJacobian( modelParameters, - constraints_[iCons], - jacobian.block(usedRows, 0, 3, modelParameters.size()), - residual.middleRows(usedRows, 3)); - usedRows += 3; - } else if (constraintType_ == VertexConstraintType::Plane) { - T res; + meshState, + constraints_[i], + jacobian.block(3 * i, 0, 3, modelParameters.size()), + residual.middleRows(3 * i, 3)); + } + usedRows = 3 * constraints_.size(); + } else { + MT_CHECK(constraintType_ == VertexConstraintType::Plane); + for (size_t i = 0; i < constraints_.size(); ++i) { + T residualValue; error += calculatePlaneJacobian( modelParameters, - constraints_[iCons], - jacobian.block(usedRows, 0, 1, modelParameters.size()), - res); - residual[usedRows] = res; - usedRows += 1; + meshState, + constraints_[i], + jacobian.block(i, 0, 1, modelParameters.size()), + residualValue); + residual(i) = residualValue; } + usedRows = constraints_.size(); } return error; @@ -530,42 +537,8 @@ size_t SkinnedLocatorTriangleErrorFunctionT::getJacobianSize() const { return 0; } -template -void SkinnedLocatorTriangleErrorFunctionT::updateMeshes( - const ModelParametersT& modelParameters) { - MT_PROFILE_FUNCTION(); - - bool doUpdateNormals = false; - if (this->character_.blendShape) { - const BlendWeightsT blendWeights = - extractBlendWeights(this->parameterTransform_, modelParameters); - this->character_.blendShape->computeShape(blendWeights, this->restMesh_->vertices); - doUpdateNormals = true; - } - if (this->character_.faceExpressionBlendShape) { - if (!this->character_.blendShape) { - // Set restMesh back to neutral, removing potential previous expressions. - // Note that if the character comes with (shape) blendShape, the previous if block already - // takes care of this step. - Eigen::Map> outputVec( - &this->restMesh_->vertices[0][0], this->restMesh_->vertices.size() * 3); - const Eigen::Map> baseVec( - &this->neutralMesh_->vertices[0][0], this->neutralMesh_->vertices.size() * 3); - outputVec = baseVec.template cast(); - } - const BlendWeightsT faceExpressionBlendWeights = - extractFaceExpressionBlendWeights(this->parameterTransform_, modelParameters); - this->character_.faceExpressionBlendShape->applyDeltas( - faceExpressionBlendWeights, this->restMesh_->vertices); - doUpdateNormals = true; - } - if (doUpdateNormals) { - this->restMesh_->updateNormals(); - } -} +} // namespace momentum // Explicit template instantiations -template class SkinnedLocatorTriangleErrorFunctionT; -template class SkinnedLocatorTriangleErrorFunctionT; - -} // namespace momentum +template class momentum::SkinnedLocatorTriangleErrorFunctionT; +template class momentum::SkinnedLocatorTriangleErrorFunctionT; diff --git a/momentum/character_solver/skinned_locator_triangle_error_function.h b/momentum/character_solver/skinned_locator_triangle_error_function.h index 762cfe2d3a..4f21d2058b 100644 --- a/momentum/character_solver/skinned_locator_triangle_error_function.h +++ b/momentum/character_solver/skinned_locator_triangle_error_function.h @@ -109,23 +109,27 @@ class SkinnedLocatorTriangleErrorFunctionT : public SkeletonErrorFunctionT { double calculatePositionJacobian( const ModelParametersT& modelParameters, + const MeshStateT& meshState, const SkinnedLocatorTriangleConstraintT& constr, Ref> jac, Ref> res) const; double calculatePlaneJacobian( const ModelParametersT& modelParameters, + const MeshStateT& meshState, const SkinnedLocatorTriangleConstraintT& constr, Ref> jac, T& res) const; double calculatePositionGradient( const ModelParametersT& modelParameters, + const MeshStateT& meshState, const SkinnedLocatorTriangleConstraintT& constr, Eigen::Ref> gradient) const; double calculatePlaneGradient( const ModelParametersT& modelParameters, + const MeshStateT& meshState, const SkinnedLocatorTriangleConstraintT& constr, Eigen::Ref> gradient) const; @@ -136,11 +140,7 @@ class SkinnedLocatorTriangleErrorFunctionT : public SkeletonErrorFunctionT { std::vector> constraints_; - std::unique_ptr> neutralMesh_; // Rest mesh without facial expression basis - std::unique_ptr> restMesh_; // Rest positions after shape basis is applied - const VertexConstraintType constraintType_; - void updateMeshes(const ModelParametersT& modelParameters); }; } // namespace momentum diff --git a/momentum/character_solver/vertex_error_function.cpp b/momentum/character_solver/vertex_error_function.cpp index b2a95dc2bc..c62a1778ad 100644 --- a/momentum/character_solver/vertex_error_function.cpp +++ b/momentum/character_solver/vertex_error_function.cpp @@ -76,6 +76,7 @@ double VertexErrorFunctionT::getError( MT_PROFILE_FUNCTION(); MT_CHECK_NOTNULL(meshState.posedMesh_); + MT_CHECK_NOTNULL(meshState.restMesh_); // loop over all constraints and calculate the error double error = 0.0; @@ -584,6 +585,7 @@ double VertexErrorFunctionT::getGradient( const MeshStateT& meshState, Eigen::Ref> gradient) { MT_CHECK_NOTNULL(meshState.posedMesh_); + MT_CHECK_NOTNULL(meshState.restMesh_); double error = 0; std::vector>> errorGradThread; @@ -664,6 +666,7 @@ double VertexErrorFunctionT::getJacobian( MT_CHECK(residual.rows() >= (Eigen::Index)(1 * constraints_.size())); MT_CHECK_NOTNULL(meshState.posedMesh_); + MT_CHECK_NOTNULL(meshState.restMesh_); double error = 0; std::vector errorThread;