mmdetection3d - CRN 분석 3 - mats 파일 분석

openxlab

mmdetection3d - CRN 분석 3 - mats 파일 분석

wandering developer 2024. 7. 20. 05:48

mats 파일 분석

(sweep_imgs, mats, img_metas, _, _, _, _, pts_pv) = batch

preds = self(sweep_imgs, mats,
            pts_pv=pts_pv,
            is_train=False)

총 5개 변수를 가지고 있는것을 확인 할 수 있음.

그리고 각각 변수의 차원은 아래 와 같음.

torch.stack(sensor2ego_mats_batch).size()  
torch.Size(\[1, 4, 6, 4, 4\])  
torch.stack(intrin_mats_batch).size()  
torch.Size(\[1, 4, 6, 4, 4\])  
torch.stack(ida_mats_batch).size()  
torch.Size(\[1, 4, 6, 4, 4\])  
torch.stack(sensor2sensor_mats_batch).size()  
torch.Size(\[1, 4, 6, 4, 4\])  
torch.stack(bda_mat_batch).size()  
torch.Size(\[1, 4, 4\])

def collate_fn(data,

mats_dict = dict()
mats_dict['sensor2ego_mats'] = torch.stack(sensor2ego_mats_batch)
mats_dict['intrin_mats'] = torch.stack(intrin_mats_batch)
mats_dict['ida_mats'] = torch.stack(ida_mats_batch)
mats_dict['sensor2sensor_mats'] = torch.stack(sensor2sensor_mats_batch)
mats_dict['bda_mat'] = torch.stack(bda_mat_batch)
ret_list = [
    torch.stack(imgs_batch),
    mats_dict,
    img_metas_batch,
    gt_boxes_3d_batch,
    gt_labels_3d_batch,
    None,  # reserve for segmentation
]

'sensor2ego_mats_batch' 아래 값이 저장되어 있다.

cam_info[cam] : calibrated_sensor  
cam_info[cam] : ego_pose  
key_info[cam] : ego_pose

w, x, y, z = cam_info[cam]['calibrated_sensor']['rotation']
# sweep sensor to sweep ego
sweepsensor2sweepego_rot = torch.Tensor(
    Quaternion(w, x, y, z).rotation_matrix)
sweepsensor2sweepego_tran = torch.Tensor(
    cam_info[cam]['calibrated_sensor']['translation'])
sweepsensor2sweepego = sweepsensor2sweepego_rot.new_zeros(
    (4, 4))
sweepsensor2sweepego[3, 3] = 1
sweepsensor2sweepego[:3, :3] = sweepsensor2sweepego_rot
sweepsensor2sweepego[:3, -1] = sweepsensor2sweepego_tran

# sweep ego to global
w, x, y, z = cam_info[cam]['ego_pose']['rotation']
sweepego2global_rot = torch.Tensor(
    Quaternion(w, x, y, z).rotation_matrix)
sweepego2global_tran = torch.Tensor(
    cam_info[cam]['ego_pose']['translation'])
sweepego2global = sweepego2global_rot.new_zeros((4, 4))
sweepego2global[3, 3] = 1
sweepego2global[:3, :3] = sweepego2global_rot
sweepego2global[:3, -1] = sweepego2global_tran

# global sensor to cur ego
w, x, y, z = key_info[cam]['ego_pose']['rotation']
keyego2global_rot = torch.Tensor(
    Quaternion(w, x, y, z).rotation_matrix)

keyego2global_tran = torch.Tensor(
    key_info[cam]['ego_pose']['translation'])

keyego2global = keyego2global_rot.new_zeros((4, 4))
keyego2global[3, 3] = 1
keyego2global[:3, :3] = keyego2global_rot
keyego2global[:3, -1] = keyego2global_tran
global2keyego = keyego2global.inverse()

sweepsensor2keyego = global2keyego @ sweepego2global @\
                    sweepsensor2sweepego

sensor2ego_mats.append(sweepsensor2keyego)

sweep_sensor2ego_mats.append(torch.stack(sensor2ego_mats))

torch.stack(sweep_sensor2ego_mats).permute(1, 0, 2, 3),

'intrin_mats_batch'

intrin_mat = torch.zeros((4, 4))
intrin_mat[3, 3] = 1
intrin_mat[:3, :3] = torch.Tensor(
    cam_info[cam]['calibrated_sensor']['camera_intrinsic'])

intrin_mats.append(intrin_mat)

sweep_intrin_mats.append(torch.stack(intrin_mats))

torch.stack(sweep_intrin_mats).permute(1, 0, 2, 3),

'ida_mats_batch'

img, ida_mat = img_transform(
    img,
    resize=resize,
    resize_dims=resize_dims,
    crop=crop,
    flip=flip,
    rotate=rotate_ida,
)

ida_mats.append(ida_mat)

sweep_ida_mats.append(torch.stack(ida_mats))

torch.stack(sweep_ida_mats).permute(1, 0, 2, 3)

'sensor2sensor_mats_batch'

sweep sensor to sweep ego

cam_info[cam]['calibrated_sensor']

sweep ego to global

cam_info[cam]['ego_pose']

global sensor to cur ego

key_info[cam]['ego_pose']

cur ego to sensor

key_info[cam]['calibrated_sensor']

# sweep sensor to sweep ego
sweepsensor2sweepego_rot = torch.Tensor(
    Quaternion(w, x, y, z).rotation_matrix)
sweepsensor2sweepego_tran = torch.Tensor(
    cam_info[cam]['calibrated_sensor']['translation'])
sweepsensor2sweepego = sweepsensor2sweepego_rot.new_zeros(
    (4, 4))
sweepsensor2sweepego[3, 3] = 1
sweepsensor2sweepego[:3, :3] = sweepsensor2sweepego_rot
sweepsensor2sweepego[:3, -1] = sweepsensor2sweepego_tran

# sweep ego to global
w, x, y, z = cam_info[cam]['ego_pose']['rotation']
sweepego2global_rot = torch.Tensor(
    Quaternion(w, x, y, z).rotation_matrix)
sweepego2global_tran = torch.Tensor(
    cam_info[cam]['ego_pose']['translation'])
sweepego2global = sweepego2global_rot.new_zeros((4, 4))
sweepego2global[3, 3] = 1
sweepego2global[:3, :3] = sweepego2global_rot
sweepego2global[:3, -1] = sweepego2global_tran

# global sensor to cur ego
w, x, y, z = key_info[cam]['ego_pose']['rotation']
keyego2global_rot = torch.Tensor(
    Quaternion(w, x, y, z).rotation_matrix)

keyego2global_tran = torch.Tensor(
    key_info[cam]['ego_pose']['translation'])

keyego2global = keyego2global_rot.new_zeros((4, 4))
keyego2global[3, 3] = 1
keyego2global[:3, :3] = keyego2global_rot
keyego2global[:3, -1] = keyego2global_tran
global2keyego = keyego2global.inverse()


# cur ego to sensor
w, x, y, z = key_info[cam]['calibrated_sensor']['rotation']
keysensor2keyego_rot = torch.Tensor(
    Quaternion(w, x, y, z).rotation_matrix)
keysensor2keyego_tran = torch.Tensor(
    key_info[cam]['calibrated_sensor']['translation'])
keysensor2keyego = keysensor2keyego_rot.new_zeros((4, 4))
keysensor2keyego[3, 3] = 1
keysensor2keyego[:3, :3] = keysensor2keyego_rot
keysensor2keyego[:3, -1] = keysensor2keyego_tran
keyego2keysensor = keysensor2keyego.inverse()

keysensor2sweepsensor = (
    keyego2keysensor @ global2keyego @ sweepego2global
    @ sweepsensor2sweepego).inverse()

sensor2sensor_mats.append(keysensor2sweepsensor)

sweep_sensor2sensor_mats.append(torch.stack(sensor2sensor_mats))

torch.stack(sweep_sensor2sensor_mats).permute(1, 0, 2, 3),

bda_mat_batch

# def __getitem__(self, idx): 에서

gt_boxes_3d, gt_labels_3d, gt_corners = self.get_gt(self.infos[idx], cams, return_corners=False)

rotate_bda, scale_bda, flip_dx, flip_dy = self.sample_bda_augmentation()
gt_boxes_3d, bda_rot = bev_det_transform(gt_boxes_3d, rotate_bda, scale_bda, flip_dx, flip_dy)

bda_mat = torch.zeros(4, 4, dtype=torch.float32)
bda_mat[:3, :3] = bda_rot
bda_mat[3, 3] = 1    

bda_mat

bda_mat_batch.append(bda_mat)

mats_dict['bda_mat'] = torch.stack(bda_mat_batch)

기타

get_image 함수의 2번째 return 값을 분석하면 됨.

결국 카메라에 있는 정보를 이용해서

아래 코드를 이해하고 적용하면 됨.

ida_mats_batch는 아래 값을 이용해서 작성됨.

self.ida_aug_conf = {  
'resize_lim': (0.386, 0.55),  
'final_dim': (256, 704),  
'rot_lim': (0., 0.),  
'H': 900,  
'W': 1600,  
'rand_flip': True,  
'bot_pct_lim': (0.0, 0.0),  
'cams': [  
'CAM_FRONT_LEFT', 'CAM\_FRONT', 'CAM\_FRONT\_RIGHT',  
'CAM_BACK_LEFT', 'CAM\_BACK', 'CAM\_BACK\_RIGHT'  
],  
'Ncams': 6,  
}

bda_mat_batch 는 아래 정보를 참조해서 작성됨.

self.bda_aug_conf = {  
'rot_ratio': 1.0,  
'rot_lim': (-22.5, 22.5),  
'scale_lim': (0.9, 1.1),  
'flip_dx_ratio': 0.5,  
'flip_dy_ratio': 0.5  
}