the joint order of the Shadow Hand in DFG Dataset

[tianyuwang01]

I delete the forearm and wrist in this file: DexFuncGrasp/Annotation/IsaacGym/assets/urdf/sr_description/robots/shadowhand_motor.urdf, and use it and urdfpy to visualize the grasps in DFG Dataset, but I can't find the correct order of shadowhand's joints.
How do you visualize the grasps in your dataset? Do I need to perform additional processing on these rjt values?
What is the joint order of the Shadow Hand? Or could you provide your URDF file used for visualization?

[tianyuwang01]

I contacted the author personally, and thanks to his help, I got the correct order of hand joints in the DFGdataset.

DFG_joints_name = ['rh_LFJ5', 'rh_LFJ4', 'rh_LFJ3', 'rh_LFJ2', 'rh_LFJ1', 
                   'rh_RFJ4', 'rh_RFJ3', 'rh_RFJ2', 'rh_RFJ1',
                   'rh_MFJ4', 'rh_MFJ3', 'rh_MFJ2', 'rh_MFJ1',
                   'rh_FFJ4', 'rh_FFJ3', 'rh_FFJ2', 'rh_FFJ1', 
                   'rh_THJ5', 'rh_THJ4', 'rh_THJ3', 'rh_THJ2', 'rh_THJ1']

The order is different from that in the URDF file /DexFuncGrasp/Annotation/dexterous_hands/data/urdf/sr_description/robots/shadowhand_motor.urdf, so I use this function to convert the hand pose.

def modify_hand_pose(hand_pose, ori_joint_order, target_joint_order):
    """
    Change the order of hand pose from original hand to target hand
    
    Parameters
    ----------
    hand_pose:(B, `n_dofs`) torch.FloatTensor
        hand joints in original order
    ori_joint_order: list of str
        original joint order
    target_joint_order: list of str
        target joint order
    
    Returns
    -------
    target_hand_pose: (B, `n_dofs`) torch.FloatTensor
    """
    assert (len(ori_joint_order) == hand_pose.shape[1] and len(target_joint_order) == hand_pose.shape[1]), "the shape is different"
    if isinstance(hand_pose, np.ndarray):
        target_hand_pose = np.zeros_like(hand_pose)
    else:
        target_hand_pose = torch.zeros_like(hand_pose, device=hand_pose.device)
    for i in range(len(target_joint_order)):
        flag = False
        for j, joint_name in enumerate(ori_joint_order):
            if target_joint_order[i][-4:] == joint_name[-4:]:
                target_hand_pose[:, i] = hand_pose[:, j]
                flag = True
                break
        assert flag, f"joint name {target_joint_order[i]} not found"
    return target_hand_pose

[tianyuwang01]

Later, I found that I still couldn't successfully visualize it. I checked the code in the project and discovered that the shadow hand mesh is assigned a view_mat_mesh rotation each time it is used.

view_mat_mesh = np.array([[0.0, 0.0, 1.0],
                         [-1.0, 0, 0.0],
                         [0.0, -1.0, 0]])

Here is the code I use for visualizing the grasp poses in the dataset with the HandModel.

import numpy as np
import trimesh as tm
import os
import sys
sys.path.append('/home/wty/wangtianyu/data/DexFuncGrasp/Annotation/dexterous_hands')
from HandModel import HandModel
import torch
from utils_hand.rot6d import compute_rotation_matrix_from_quaternion, robust_compute_rotation_matrix_from_ortho6d
import random
from termcolor import cprint

def modify_hand_pose(hand_pose, ori_joint_order, target_joint_order):
    """
    Change the order of hand pose from original hand to target hand
    
    Parameters
    ----------
    hand_pose:(B, `n_dofs`) torch.FloatTensor
        hand joints in original order
    ori_joint_order: list of str
        original joint order
    target_joint_order: list of str
        target joint order
    
    Returns
    -------
    target_hand_pose: (B, `n_dofs`) torch.FloatTensor
    """
    assert (len(ori_joint_order) == hand_pose.shape[1] and len(target_joint_order) == hand_pose.shape[1]), "the shape is different"
    if isinstance(hand_pose, np.ndarray):
        target_hand_pose = np.zeros_like(hand_pose)
    else:
        target_hand_pose = torch.zeros_like(hand_pose, device=hand_pose.device)
    for i in range(len(target_joint_order)):
        flag = False
        for j, joint_name in enumerate(ori_joint_order):
            if target_joint_order[i][-4:] == joint_name[-4:]:
                target_hand_pose[:, i] = hand_pose[:, j]
                flag = True
                break
        assert flag, f"joint name {target_joint_order[i]} not found"
    return target_hand_pose

DFG_joints_name = ['rh_LFJ5', 'rh_LFJ4', 'rh_LFJ3', 'rh_LFJ2', 'rh_LFJ1', 
                   'rh_RFJ4', 'rh_RFJ3', 'rh_RFJ2', 'rh_RFJ1',
                   'rh_MFJ4', 'rh_MFJ3', 'rh_MFJ2', 'rh_MFJ1',
                   'rh_FFJ4', 'rh_FFJ3', 'rh_FFJ2', 'rh_FFJ1', 
                   'rh_THJ5', 'rh_THJ4', 'rh_THJ3', 'rh_THJ2', 'rh_THJ1']

view_mat_mesh = torch.tensor([[0.0, 0.0, 1.0],
                                [-1.0, 0, 0.0],
                                [0.0, -1.0, 0]], device='cuda:0')

shadow_urdf_path = '/home/wty/wangtianyu/data/DexFuncGrasp/Annotation/dexterous_hands/data/urdf/sr_description/robots/shadowhand_motor.urdf'
mesh_path = '/home/wty/wangtianyu/data/DexFuncGrasp/Annotation/dexterous_hands/data/urdf/sr_description/meshes/hand'
hand_model = HandModel('shadowhand', shadow_urdf_path, mesh_path, hand_scale=1.)

dataset_path = '/home/wty/wangtianyu/data/DexFuncGrasp/DexFuncGraspNet/Grasps_Dataset/train' 
objects = os.listdir(dataset_path)
object = objects[random.randint(0, len(objects) - 1)]
npy_file_list = os.listdir(os.path.join(dataset_path, object))
npy_file = npy_file_list[random.randint(0, len(npy_file_list) - 1)]
npy_file = os.path.join(dataset_path, object, npy_file)
npy_file = '/home/wty/wangtianyu/data/DexFuncGrasp/DexFuncGraspNet/Grasps_Dataset/test/9_unit_teapot_0/unit_teapot_4.npy'
cprint(npy_file, 'red')

npy = np.load(npy_file, allow_pickle=True)
obj_class = npy[0]['obj_class']
obj_name = npy[0]['obj_name']
grasps = npy[1]['rtj']
print(obj_class, obj_name, grasps.shape[0])

obj_root = '/home/wty/wangtianyu/data/DexFuncGrasp/Annotation/IsaacGym/assets/'
obj_mesh_path = os.path.join(obj_root, f'urdf/off/Obj_Data/{obj_class}', f'{obj_name}.obj')
cprint(obj_mesh_path, 'red')
obj_mesh = tm.load_mesh(obj_mesh_path)

for index in range(grasps.shape[0]):
    grasp = grasps[index]
    cprint(grasp, 'green')
    quat = grasp[0:4] #wxyz
    pos = grasp[4:7] * 0.001
    joints = grasp[7:]
    quat = torch.from_numpy(quat).float().to('cuda:0')
    pos = torch.from_numpy(pos).float().to('cuda:0')
    joints = torch.from_numpy(joints).float().to('cuda:0')
    joints = modify_hand_pose(joints.unsqueeze(0), DFG_joints_name, hand_model.robot.get_joint_parameter_names()).squeeze(0)

    rot_matrix = compute_rotation_matrix_from_quaternion(quat.unsqueeze(0)).squeeze(0)
    rot_matrix = rot_matrix.matmul(view_mat_mesh)
    rot = rot_matrix.T.flatten()[:6]
    hand_pose = torch.cat([pos, rot, joints], dim=0).unsqueeze(0)
    print(hand_pose)

    hand_mesh = hand_model.get_meshes_from_q(hand_pose)
    for mesh in hand_mesh:
        mesh.visual.vertex_colors = np.array([0.0, 0.0, 255.0])
    obj_mesh.visual.vertex_colors = np.array([100.0, 100.0, 100.0])
    axis = tm.creation.axis(axis_length=0.2, axis_radius=0.001)
    tm.Scene([hand_mesh, obj_mesh]).show()

cprint(hand_model.robot.get_joint_parameter_names(), 'red')

[cxcAxxy]

请问怎么联系作者，有一些问题想要咨询一下作者

[tianyuwang01]

请问怎么联系作者，有一些问题想要咨询一下作者

我看到作者回复了你的issue

[haochen611]

您好，您能可视化了嘛，我在可视化的过程中遇到了点问题，想请教一下您

[haochen611]

Later, I found that I still couldn't successfully visualize it. I checked the code in the project and discovered that the shadow hand mesh is assigned a view_mat_mesh rotation each time it is used.

view_mat_mesh = np.array([[0.0, 0.0, 1.0],
                         [-1.0, 0, 0.0],
                         [0.0, -1.0, 0]])

Here is the code I use for visualizing the grasp poses in the dataset with the HandModel.

import numpy as np
import trimesh as tm
import os
import sys
sys.path.append('/home/wty/wangtianyu/data/DexFuncGrasp/Annotation/dexterous_hands')
from HandModel import HandModel
import torch
from utils_hand.rot6d import compute_rotation_matrix_from_quaternion, robust_compute_rotation_matrix_from_ortho6d
import random
from termcolor import cprint

def modify_hand_pose(hand_pose, ori_joint_order, target_joint_order):
    """
    Change the order of hand pose from original hand to target hand
    
    Parameters
    ----------
    hand_pose:(B, `n_dofs`) torch.FloatTensor
        hand joints in original order
    ori_joint_order: list of str
        original joint order
    target_joint_order: list of str
        target joint order
    
    Returns
    -------
    target_hand_pose: (B, `n_dofs`) torch.FloatTensor
    """
    assert (len(ori_joint_order) == hand_pose.shape[1] and len(target_joint_order) == hand_pose.shape[1]), "the shape is different"
    if isinstance(hand_pose, np.ndarray):
        target_hand_pose = np.zeros_like(hand_pose)
    else:
        target_hand_pose = torch.zeros_like(hand_pose, device=hand_pose.device)
    for i in range(len(target_joint_order)):
        flag = False
        for j, joint_name in enumerate(ori_joint_order):
            if target_joint_order[i][-4:] == joint_name[-4:]:
                target_hand_pose[:, i] = hand_pose[:, j]
                flag = True
                break
        assert flag, f"joint name {target_joint_order[i]} not found"
    return target_hand_pose

DFG_joints_name = ['rh_LFJ5', 'rh_LFJ4', 'rh_LFJ3', 'rh_LFJ2', 'rh_LFJ1', 
                   'rh_RFJ4', 'rh_RFJ3', 'rh_RFJ2', 'rh_RFJ1',
                   'rh_MFJ4', 'rh_MFJ3', 'rh_MFJ2', 'rh_MFJ1',
                   'rh_FFJ4', 'rh_FFJ3', 'rh_FFJ2', 'rh_FFJ1', 
                   'rh_THJ5', 'rh_THJ4', 'rh_THJ3', 'rh_THJ2', 'rh_THJ1']

view_mat_mesh = torch.tensor([[0.0, 0.0, 1.0],
                                [-1.0, 0, 0.0],
                                [0.0, -1.0, 0]], device='cuda:0')

shadow_urdf_path = '/home/wty/wangtianyu/data/DexFuncGrasp/Annotation/dexterous_hands/data/urdf/sr_description/robots/shadowhand_motor.urdf'
mesh_path = '/home/wty/wangtianyu/data/DexFuncGrasp/Annotation/dexterous_hands/data/urdf/sr_description/meshes/hand'
hand_model = HandModel('shadowhand', shadow_urdf_path, mesh_path, hand_scale=1.)

dataset_path = '/home/wty/wangtianyu/data/DexFuncGrasp/DexFuncGraspNet/Grasps_Dataset/train' 
objects = os.listdir(dataset_path)
object = objects[random.randint(0, len(objects) - 1)]
npy_file_list = os.listdir(os.path.join(dataset_path, object))
npy_file = npy_file_list[random.randint(0, len(npy_file_list) - 1)]
npy_file = os.path.join(dataset_path, object, npy_file)
npy_file = '/home/wty/wangtianyu/data/DexFuncGrasp/DexFuncGraspNet/Grasps_Dataset/test/9_unit_teapot_0/unit_teapot_4.npy'
cprint(npy_file, 'red')

npy = np.load(npy_file, allow_pickle=True)
obj_class = npy[0]['obj_class']
obj_name = npy[0]['obj_name']
grasps = npy[1]['rtj']
print(obj_class, obj_name, grasps.shape[0])

obj_root = '/home/wty/wangtianyu/data/DexFuncGrasp/Annotation/IsaacGym/assets/'
obj_mesh_path = os.path.join(obj_root, f'urdf/off/Obj_Data/{obj_class}', f'{obj_name}.obj')
cprint(obj_mesh_path, 'red')
obj_mesh = tm.load_mesh(obj_mesh_path)

for index in range(grasps.shape[0]):
    grasp = grasps[index]
    cprint(grasp, 'green')
    quat = grasp[0:4] #wxyz
    pos = grasp[4:7] * 0.001
    joints = grasp[7:]
    quat = torch.from_numpy(quat).float().to('cuda:0')
    pos = torch.from_numpy(pos).float().to('cuda:0')
    joints = torch.from_numpy(joints).float().to('cuda:0')
    joints = modify_hand_pose(joints.unsqueeze(0), DFG_joints_name, hand_model.robot.get_joint_parameter_names()).squeeze(0)

    rot_matrix = compute_rotation_matrix_from_quaternion(quat.unsqueeze(0)).squeeze(0)
    rot_matrix = rot_matrix.matmul(view_mat_mesh)
    rot = rot_matrix.T.flatten()[:6]
    hand_pose = torch.cat([pos, rot, joints], dim=0).unsqueeze(0)
    print(hand_pose)

    hand_mesh = hand_model.get_meshes_from_q(hand_pose)
    for mesh in hand_mesh:
        mesh.visual.vertex_colors = np.array([0.0, 0.0, 255.0])
    obj_mesh.visual.vertex_colors = np.array([100.0, 100.0, 100.0])
    axis = tm.creation.axis(axis_length=0.2, axis_radius=0.001)
    tm.Scene([hand_mesh, obj_mesh]).show()

cprint(hand_model.robot.get_joint_parameter_names(), 'red')

您好，我使用您的代码可视化成功了，这个抓取手势j是对的，但是作者的rt数据可视化不是很理想的效果，这个问题您有遇到嘛，恳请您帮忙解答，如有打扰，实在不好意思。

[tianyuwang01]

您好，我使用您的代码可视化成功了，这个抓取手势j是对的，但是作者的rt数据可视化不是很理想的效果，这个问题您有遇到嘛，恳请您帮忙解答，如有打扰，实在不好意思。

你用我的代码改了哪个部分吗

[haochen611]

您好，我使用您的代码可视化成功了，这个抓取手势j是对的，但是作者的rt数据可视化不是很理想的效果，这个问题您有遇到嘛，恳请您帮忙解答，如有打扰，实在不好意思。

你用我的代码改了哪个部分吗

没有改动，手势是对的，但是有的数据手势跟物体之间有一定距离，这个如何解决呢，请教一下您

[tianyuwang01]

您好，我使用您的代码可视化成功了，这个抓取手势j是对的，但是作者的rt数据可视化不是很理想的效果，这个问题您有遇到嘛，恳请您帮忙解答，如有打扰，实在不好意思。

你用我的代码改了哪个部分吗

没有改动，手势是对的，但是有的数据手势跟物体之间有一定距离，这个如何解决呢，请教一下您

我的微信w1T5y3，我们交流一下看看吧

[haochen611]

您好，我使用您的代码可视化成功了，这个抓取手势j是对的，但是作者的rt数据可视化不是很理想的效果，这个问题您有遇到嘛，恳请您帮忙解答，如有打扰，实在不好意思。

你用我的代码改了哪个部分吗

没有改动，手势是对的，但是有的数据手势跟物体之间有一定距离，这个如何解决呢，请教一下您

我的微信w1T5y3，我们交流一下看看吧

非常感谢！