{"kind":"task","effective_mode":"full","benchmark":{"kind":"benchmark","effective_mode":"full","slug":"longbench-v2","formal_name":"LongBench v2","introduction":"LongBench v2 evaluates deep understanding and reasoning over long contexts through multiple-choice questions. Its official description lists 503 questions spanning tasks such as single-document and multi-document QA and code-repository understanding.","introduction_ja":"","introduction_en":"","category":"Category not supplied","task_count":null,"acquisition_status":"Acquisition status not supplied","official_url":"https://huggingface.co/datasets/zai-org/LongBench-v2","indexing_mode":"noindex","profile":{"resources":[],"task_format":"","scoring":"","metric":"","size":"","answer_access":"","license":"","citation":"","maintainer":"","released":"","why_hard":"","related":[]}},"task_id":"f0ee614c-6d51-5554-87f4-fbed9893d65e","task_key":"train--66fa50acbb02136c067c6827","task_revision_id":"3","upstream_id":"66fa50acbb02136c067c6827","short_description":"Which realistic factor in collaborative perception does this algorithm model…","config":"","split":"train","body":"{\"choice_A\":\"This algorithm model takes into account the realistic factors of communication overload and solves the problem of excessive communication pressure.\",\"choice_B\":\"This model takes into account real-world problems, which are time asynchrony and posture errors, and solves the problem of spatial alignment.\",\"choice_C\":\"This algorithm model takes into account real-world issues such as time asynchrony and sensor heterogeneity, and solves the problem of time and spatial alignment.\",\"choice_D\":\"The algorithm model takes into account realistic issues such as communication pressure overload and solves the problem of communication strategy\",\"context\":\"\\\"\\\"\\\"Specifies the current version number of v2xvit.\\\"\\\"\\\"\\n\\n__version__ = \\\"0.1.0\\\"\\n\\n\\n\\n\\nimport torch\\nimport torch.nn as nn\\n\\nfrom v2xvit.models.sub_modules.pillar_vfe import PillarVFE\\nfrom v2xvit.models.sub_modules.point_pillar_scatter import PointPillarScatter\\nfrom v2xvit.models.sub_modules.base_bev_backbone import BaseBEVBackbone\\nfrom v2xvit.models.sub_modules.fuse_utils import regroup\\nfrom v2xvit.models.sub_modules.downsample_conv import DownsampleConv\\nfrom v2xvit.models.sub_modules.naive_compress import NaiveCompressor\\nfrom v2xvit.models.sub_modules.v2xvit_basic import V2XTransformer\\n\\n\\nclass PointPillarTransformer(nn.Module):\\n def __init__(self, args):\\n super(PointPillarTransformer, self).__init__()\\n\\n self.max_cav = args['max_cav']\\n # PIllar VFE\\n self.pillar_vfe = PillarVFE(args['pillar_vfe'],\\n num_point_features=4,\\n voxel_size=args['voxel_size'],\\n point_cloud_range=args['lidar_range'])\\n self.scatter = PointPillarScatter(args['point_pillar_scatter'])\\n self.backbone = BaseBEVBackbone(args['base_bev_backbone'], 64)\\n # used to downsample the feature map for efficient computation\\n self.shrink_flag = False\\n if 'shrink_header' in args:\\n self.shrink_flag = True\\n self.shrink_conv = DownsampleConv(args['shrink_header'])\\n self.compression = False\\n\\n if args['compression'] > 0:\\n self.compression = True\\n self.naive_compressor = NaiveCompressor(256, args['compression'])\\n\\n self.fusion_net = V2XTransformer(args['transformer'])\\n\\n self.cls_head = nn.Conv2d(128 * 2, args['anchor_number'],\\n kernel_size=1)\\n self.reg_head = nn.Conv2d(128 * 2, 7 * args['anchor_number'],\\n kernel_size=1)\\n\\n if args['backbone_fix']:\\n self.backbone_fix()\\n\\n def backbone_fix(self):\\n \\\"\\\"\\\"\\n Fix the parameters of backbone during finetune on timedelay。\\n \\\"\\\"\\\"\\n for p in self.pillar_vfe.parameters():\\n p.requires_grad = False\\n\\n for p in self.scatter.parameters():\\n p.requires_grad = False\\n\\n for p in self.backbone.parameters():\\n p.requires_grad = False\\n\\n if self.compression:\\n for p in self.naive_compressor.parameters():\\n p.requires_grad = False\\n if self.shrink_flag:\\n for p in self.shrink_conv.parameters():\\n p.requires_grad = False\\n\\n for p in self.cls_head.parameters():\\n p.requires_grad = False\\n for p in self.reg_head.parameters():\\n p.requires_grad = False\\n\\n def forward(self, data_dict):\\n voxel_features = data_dict['processed_lidar']['voxel_features']\\n voxel_coords = data_dict['processed_lidar']['voxel_coords']\\n voxel_num_points = data_dict['processed_lidar']['voxel_num_points']\\n record_len = data_dict['record_len']\\n spatial_correction_matrix = data_dict['spatial_correction_matrix']\\n\\n # B, max_cav, 3(dt dv infra), 1, 1\\n prior_encoding =\\\\\\n data_dict['prior_encoding'].unsqueeze(-1).unsqueeze(-1)\\n\\n batch_dict = {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords,\\n 'voxel_num_points': voxel_num_points,\\n 'record_len': record_len}\\n # n, 4 -> n, c\\n batch_dict = self.pillar_vfe(batch_dict)\\n # n, c -> N, C, H, W\\n batch_dict = self.scatter(batch_dict)\\n batch_dict = self.backbone(batch_dict)\\n\\n spatial_features_2d = batch_dict['spatial_features_2d']\\n # downsample feature to reduce memory\\n if self.shrink_flag:\\n spatial_features_2d = self.shrink_conv(spatial_features_2d)\\n # compressor\\n if self.compression:\\n spatial_features_2d = self.naive_compressor(spatial_features_2d)\\n # N, C, H, W -> B, L, C, H, W\\n regroup_feature, mask = regroup(spatial_features_2d,\\n record_len,\\n self.max_cav)\\n # prior encoding added\\n prior_encoding = prior_encoding.repeat(1, 1, 1,\\n regroup_feature.shape[3],\\n regroup_feature.shape[4])\\n regroup_feature = torch.cat([regroup_feature, prior_encoding], dim=2)\\n\\n # b l c h w -> b l h w c\\n regroup_feature = regroup_feature.permute(0, 1, 3, 4, 2)\\n # transformer fusion\\n fused_feature = self.fusion_net(regroup_feature, mask, spatial_correction_matrix)\\n # b h w c -> b c h w\\n fused_feature = fused_feature.permute(0, 3, 1, 2)\\n\\n psm = self.cls_head(fused_feature)\\n rm = self.reg_head(fused_feature)\\n\\n output_dict = {'psm': psm,\\n 'rm': rm}\\n\\n return output_dict\\n\\n\\nimport torch.nn as nn\\n\\nfrom v2xvit.models.sub_modules.pillar_vfe import PillarVFE\\nfrom v2xvit.models.sub_modules.point_pillar_scatter import PointPillarScatter\\nfrom v2xvit.models.sub_modules.base_bev_backbone import BaseBEVBackbone\\nfrom v2xvit.models.sub_modules.downsample_conv import DownsampleConv\\nfrom v2xvit.models.sub_modules.naive_compress import NaiveCompressor\\nfrom v2xvit.models.sub_modules.f_cooper_fuse import SpatialFusion\\n\\n\\nclass PointPillarFCooper(nn.Module):\\n def __init__(self, args):\\n super(PointPillarFCooper, self).__init__()\\n\\n self.max_cav = args['max_cav']\\n # PIllar VFE\\n self.pillar_vfe = PillarVFE(args['pillar_vfe'],\\n num_point_features=4,\\n voxel_size=args['voxel_size'],\\n point_cloud_range=args['lidar_range'])\\n self.scatter = PointPillarScatter(args['point_pillar_scatter'])\\n self.backbone = BaseBEVBackbone(args['base_bev_backbone'], 64)\\n # used to downsample the feature map for efficient computation\\n self.shrink_flag = False\\n if 'shrink_header' in args:\\n self.shrink_flag = True\\n self.shrink_conv = DownsampleConv(args['shrink_header'])\\n self.compression = False\\n\\n if args['compression'] > 0:\\n self.compression = True\\n self.naive_compressor = NaiveCompressor(256, args['compression'])\\n\\n self.fusion_net = SpatialFusion()\\n\\n self.cls_head = nn.Conv2d(128 * 2, args['anchor_number'],\\n kernel_size=1)\\n self.reg_head = nn.Conv2d(128 * 2, 7 * args['anchor_number'],\\n kernel_size=1)\\n\\n if args['backbone_fix']:\\n self.backbone_fix()\\n\\n def backbone_fix(self):\\n \\\"\\\"\\\"\\n Fix the parameters of backbone during finetune on timedelay。\\n \\\"\\\"\\\"\\n for p in self.pillar_vfe.parameters():\\n p.requires_grad = False\\n\\n for p in self.scatter.parameters():\\n p.requires_grad = False\\n\\n for p in self.backbone.parameters():\\n p.requires_grad = False\\n\\n if self.compression:\\n for p in self.naive_compressor.parameters():\\n p.requires_grad = False\\n if self.shrink_flag:\\n for p in self.shrink_conv.parameters():\\n p.requires_grad = False\\n\\n for p in self.cls_head.parameters():\\n p.requires_grad = False\\n for p in self.reg_head.parameters():\\n p.requires_grad = False\\n\\n def forward(self, data_dict):\\n voxel_features = data_dict['processed_lidar']['voxel_features']\\n voxel_coords = data_dict['processed_lidar']['voxel_coords']\\n voxel_num_points = data_dict['processed_lidar']['voxel_num_points']\\n record_len = data_dict['record_len']\\n spatial_correction_matrix = data_dict['spatial_correction_matrix']\\n\\n batch_dict = {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords,\\n 'voxel_num_points': voxel_num_points,\\n 'record_len': record_len}\\n # n, 4 -> n, c\\n batch_dict = self.pillar_vfe(batch_dict)\\n # n, c -> N, C, H, W\\n batch_dict = self.scatter(batch_dict)\\n batch_dict = self.backbone(batch_dict)\\n\\n spatial_features_2d = batch_dict['spatial_features_2d']\\n # downsample feature to reduce memory\\n if self.shrink_flag:\\n spatial_features_2d = self.shrink_conv(spatial_features_2d)\\n # compressor\\n if self.compression:\\n spatial_features_2d = self.naive_compressor(spatial_features_2d)\\n\\n fused_feature = self.fusion_net(spatial_features_2d, record_len)\\n\\n psm = self.cls_head(fused_feature)\\n rm = self.reg_head(fused_feature)\\n\\n output_dict = {'psm': psm,\\n 'rm': rm}\\n\\n return output_dict\\n\\n\\nimport torch.nn as nn\\n\\nfrom v2xvit.models.sub_modules.pillar_vfe import PillarVFE\\nfrom v2xvit.models.sub_modules.point_pillar_scatter import PointPillarScatter\\nfrom v2xvit.models.sub_modules.base_bev_backbone import BaseBEVBackbone\\nfrom v2xvit.models.sub_modules.downsample_conv import DownsampleConv\\nfrom v2xvit.models.sub_modules.naive_compress import NaiveCompressor\\nfrom v2xvit.models.sub_modules.self_attn import AttFusion\\n\\n\\nclass PointPillarOPV2V(nn.Module):\\n def __init__(self, args):\\n super(PointPillarOPV2V, self).__init__()\\n\\n self.max_cav = args['max_cav']\\n # PIllar VFE\\n self.pillar_vfe = PillarVFE(args['pillar_vfe'],\\n num_point_features=4,\\n voxel_size=args['voxel_size'],\\n point_cloud_range=args['lidar_range'])\\n self.scatter = PointPillarScatter(args['point_pillar_scatter'])\\n self.backbone = BaseBEVBackbone(args['base_bev_backbone'], 64)\\n # used to downsample the feature map for efficient computation\\n self.shrink_flag = False\\n if 'shrink_header' in args:\\n self.shrink_flag = True\\n self.shrink_conv = DownsampleConv(args['shrink_header'])\\n self.compression = False\\n\\n if args['compression'] > 0:\\n self.compression = True\\n self.naive_compressor = NaiveCompressor(256, args['compression'])\\n\\n self.fusion_net = AttFusion(256)\\n\\n self.cls_head = nn.Conv2d(128 * 2, args['anchor_number'],\\n kernel_size=1)\\n self.reg_head = nn.Conv2d(128 * 2, 7 * args['anchor_number'],\\n kernel_size=1)\\n\\n if args['backbone_fix']:\\n self.backbone_fix()\\n\\n def backbone_fix(self):\\n \\\"\\\"\\\"\\n Fix the parameters of backbone during finetune on timedelay。\\n \\\"\\\"\\\"\\n for p in self.pillar_vfe.parameters():\\n p.requires_grad = False\\n\\n for p in self.scatter.parameters():\\n p.requires_grad = False\\n\\n for p in self.backbone.parameters():\\n p.requires_grad = False\\n\\n if self.compression:\\n for p in self.naive_compressor.parameters():\\n p.requires_grad = False\\n if self.shrink_flag:\\n for p in self.shrink_conv.parameters():\\n p.requires_grad = False\\n\\n for p in self.cls_head.parameters():\\n p.requires_grad = False\\n for p in self.reg_head.parameters():\\n p.requires_grad = False\\n\\n def forward(self, data_dict):\\n voxel_features = data_dict['processed_lidar']['voxel_features']\\n voxel_coords = data_dict['processed_lidar']['voxel_coords']\\n voxel_num_points = data_dict['processed_lidar']['voxel_num_points']\\n record_len = data_dict['record_len']\\n spatial_correction_matrix = data_dict['spatial_correction_matrix']\\n\\n # B, max_cav, 3(dt dv infra), 1, 1\\n prior_encoding =\\\\\\n data_dict['prior_encoding'].unsqueeze(-1).unsqueeze(-1)\\n\\n batch_dict = {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords,\\n 'voxel_num_points': voxel_num_points,\\n 'record_len': record_len}\\n # n, 4 -> n, c\\n batch_dict = self.pillar_vfe(batch_dict)\\n # n, c -> N, C, H, W\\n batch_dict = self.scatter(batch_dict)\\n batch_dict = self.backbone(batch_dict)\\n\\n spatial_features_2d = batch_dict['spatial_features_2d']\\n # downsample feature to reduce memory\\n if self.shrink_flag:\\n spatial_features_2d = self.shrink_conv(spatial_features_2d)\\n # compressor\\n if self.compression:\\n spatial_features_2d = self.naive_compressor(spatial_features_2d)\\n\\n fused_feature = self.fusion_net(spatial_features_2d, record_len)\\n\\n psm = self.cls_head(fused_feature)\\n rm = self.reg_head(fused_feature)\\n\\n output_dict = {'psm': psm,\\n 'rm': rm}\\n\\n return output_dict\\n\\n\\n\\n\\n\\\"\\\"\\\"\\nVanilla pointpillar for early and late fusion.\\n\\\"\\\"\\\"\\nimport torch.nn as nn\\n\\nfrom v2xvit.models.sub_modules.pillar_vfe import PillarVFE\\nfrom v2xvit.models.sub_modules.point_pillar_scatter import PointPillarScatter\\nfrom v2xvit.models.sub_modules.base_bev_backbone import BaseBEVBackbone\\nfrom v2xvit.models.sub_modules.downsample_conv import DownsampleConv\\n\\n\\nclass PointPillar(nn.Module):\\n def __init__(self, args):\\n super(PointPillar, self).__init__()\\n\\n # PIllar VFE\\n self.pillar_vfe = PillarVFE(args['pillar_vfe'],\\n num_point_features=4,\\n voxel_size=args['voxel_size'],\\n point_cloud_range=args['lidar_range'])\\n self.scatter = PointPillarScatter(args['point_pillar_scatter'])\\n self.backbone = BaseBEVBackbone(args['base_bev_backbone'], 64)\\n # used to downsample the feature map for efficient computation\\n self.shrink_flag = False\\n if 'shrink_header' in args:\\n self.shrink_flag = True\\n self.shrink_conv = DownsampleConv(args['shrink_header'])\\n\\n self.cls_head = nn.Conv2d(args['cls_head_dim'], args['anchor_number'],\\n kernel_size=1)\\n self.reg_head = nn.Conv2d(args['cls_head_dim'],\\n 7 * args['anchor_number'],\\n kernel_size=1)\\n\\n def forward(self, data_dict):\\n\\n voxel_features = data_dict['processed_lidar']['voxel_features']\\n voxel_coords = data_dict['processed_lidar']['voxel_coords']\\n voxel_num_points = data_dict['processed_lidar']['voxel_num_points']\\n\\n batch_dict = {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords,\\n 'voxel_num_points': voxel_num_points}\\n\\n batch_dict = self.pillar_vfe(batch_dict)\\n batch_dict = self.scatter(batch_dict)\\n batch_dict = self.backbone(batch_dict)\\n\\n spatial_features_2d = batch_dict['spatial_features_2d']\\n if self.shrink_flag:\\n spatial_features_2d = self.shrink_conv(spatial_features_2d)\\n\\n psm = self.cls_head(spatial_features_2d)\\n rm = self.reg_head(spatial_features_2d)\\n\\n output_dict = {'psm': psm,\\n 'rm': rm}\\n\\n return output_dict\\n\\nimport torch\\nimport torch.nn as nn\\n\\nfrom v2xvit.models.sub_modules.pillar_vfe import PillarVFE\\nfrom v2xvit.models.sub_modules.point_pillar_scatter import PointPillarScatter\\nfrom v2xvit.models.sub_modules.base_bev_backbone import BaseBEVBackbone\\nfrom v2xvit.models.sub_modules.downsample_conv import DownsampleConv\\nfrom v2xvit.models.sub_modules.naive_compress import NaiveCompressor\\nfrom v2xvit.models.sub_modules.v2v_fuse import V2VNetFusion\\n\\n\\nclass PointPillarV2VNet(nn.Module):\\n def __init__(self, args):\\n super(PointPillarV2VNet, self).__init__()\\n\\n self.max_cav = args['max_cav']\\n # PIllar VFE\\n self.pillar_vfe = PillarVFE(args['pillar_vfe'],\\n num_point_features=4,\\n voxel_size=args['voxel_size'],\\n point_cloud_range=args['lidar_range'])\\n self.scatter = PointPillarScatter(args['point_pillar_scatter'])\\n self.backbone = BaseBEVBackbone(args['base_bev_backbone'], 64)\\n # used to downsample the feature map for efficient computation\\n self.shrink_flag = False\\n if 'shrink_header' in args:\\n self.shrink_flag = True\\n self.shrink_conv = DownsampleConv(args['shrink_header'])\\n self.compression = False\\n\\n if args['compression'] > 0:\\n self.compression = True\\n self.naive_compressor = NaiveCompressor(256, args['compression'])\\n\\n self.fusion_net = V2VNetFusion(args['v2vfusion'])\\n\\n self.cls_head = nn.Conv2d(128 * 2, args['anchor_number'],\\n kernel_size=1)\\n self.reg_head = nn.Conv2d(128 * 2, 7 * args['anchor_number'],\\n kernel_size=1)\\n\\n if args['backbone_fix']:\\n self.backbone_fix()\\n\\n def backbone_fix(self):\\n \\\"\\\"\\\"\\n Fix the parameters of backbone during finetune on timedelay。\\n \\\"\\\"\\\"\\n for p in self.pillar_vfe.parameters():\\n p.requires_grad = False\\n\\n for p in self.scatter.parameters():\\n p.requires_grad = False\\n\\n for p in self.backbone.parameters():\\n p.requires_grad = False\\n\\n if self.compression:\\n for p in self.naive_compressor.parameters():\\n p.requires_grad = False\\n if self.shrink_flag:\\n for p in self.shrink_conv.parameters():\\n p.requires_grad = False\\n\\n for p in self.cls_head.parameters():\\n p.requires_grad = False\\n for p in self.reg_head.parameters():\\n p.requires_grad = False\\n\\n def unpad_prior_encoding(self, x, record_len):\\n # remove padded zeros to form tensor with shape (N, 3)\\n # x: (B, L, 3); record_len: (B)\\n B = x.shape[0]\\n out = []\\n for i in range(B):\\n # (valid_len, 3)\\n out.append(x[i, :record_len[i], :])\\n out = torch.cat(out, dim=0)\\n # (N, 3)\\n return out\\n\\n def forward(self, data_dict):\\n voxel_features = data_dict['processed_lidar']['voxel_features']\\n voxel_coords = data_dict['processed_lidar']['voxel_coords']\\n voxel_num_points = data_dict['processed_lidar']['voxel_num_points']\\n record_len = data_dict['record_len']\\n spatial_correction_matrix = data_dict['spatial_correction_matrix']\\n pairwise_t_matrix = data_dict['pairwise_t_matrix']\\n prior_encoding = data_dict['prior_encoding']\\n prior_encoding = self.unpad_prior_encoding(prior_encoding, record_len)\\n\\n batch_dict = {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords,\\n 'voxel_num_points': voxel_num_points,\\n 'record_len': record_len}\\n # n, 4 -> n, c\\n batch_dict = self.pillar_vfe(batch_dict)\\n # n, c -> N, C, H, W\\n batch_dict = self.scatter(batch_dict)\\n batch_dict = self.backbone(batch_dict)\\n\\n spatial_features_2d = batch_dict['spatial_features_2d']\\n # downsample feature to reduce memory\\n if self.shrink_flag:\\n spatial_features_2d = self.shrink_conv(spatial_features_2d)\\n # compressor\\n if self.compression:\\n spatial_features_2d = self.naive_compressor(spatial_features_2d)\\n fused_feature = self.fusion_net(spatial_features_2d,\\n record_len,\\n pairwise_t_matrix,\\n prior_encoding)\\n\\n psm = self.cls_head(fused_feature)\\n rm = self.reg_head(fused_feature)\\n\\n output_dict = {'psm': psm,\\n 'rm': rm}\\n\\n return output_dict\\n\\n\\nimport numpy as np\\nimport torch\\nimport torch.nn as nn\\nimport torch.nn.functional as F\\n\\n\\nclass ScaledDotProductAttention(nn.Module):\\n \\\"\\\"\\\"\\n Scaled Dot-Product Attention proposed in \\\"Attention Is All You Need\\\"\\n Compute the dot products of the query with all keys, divide each by sqrt(dim),\\n and apply a softmax function to obtain the weights on the values\\n Args: dim, mask\\n dim (int): dimention of attention\\n mask (torch.Tensor): tensor containing indices to be masked\\n Inputs: query, key, value, mask\\n - **query** (batch, q_len, d_model): tensor containing projection vector for decoder.\\n - **key** (batch, k_len, d_model): tensor containing projection vector for encoder.\\n - **value** (batch, v_len, d_model): tensor containing features of the encoded input sequence.\\n - **mask** (-): tensor containing indices to be masked\\n Returns: context, attn\\n - **context**: tensor containing the context vector from attention mechanism.\\n - **attn**: tensor containing the attention (alignment) from the encoder outputs.\\n \\\"\\\"\\\"\\n\\n def __init__(self, dim):\\n super(ScaledDotProductAttention, self).__init__()\\n self.sqrt_dim = np.sqrt(dim)\\n\\n def forward(self, query, key, value):\\n score = torch.bmm(query, key.transpose(1, 2)) / self.sqrt_dim\\n attn = F.softmax(score, -1)\\n context = torch.bmm(attn, value)\\n return context\\n\\n\\nclass AttFusion(nn.Module):\\n def __init__(self, feature_dim):\\n super(AttFusion, self).__init__()\\n self.att = ScaledDotProductAttention(feature_dim)\\n\\n def forward(self, x, record_len):\\n split_x = self.regroup(x, record_len)\\n batch_size = len(record_len)\\n C, W, H = split_x[0].shape[1:]\\n out = []\\n for xx in split_x:\\n cav_num = xx.shape[0]\\n xx = xx.view(cav_num, C, -1).permute(2, 0, 1)\\n h = self.att(xx, xx, xx)\\n h = h.permute(1, 2, 0).view(cav_num, C, W, H)[0, ...].unsqueeze(0)\\n out.append(h)\\n return torch.cat(out, dim=0)\\n\\n def regroup(self, x, record_len):\\n cum_sum_len = torch.cumsum(record_len, dim=0)\\n split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())\\n return split_x\\n\\n\\nimport os\\nimport torch\\nfrom torch import nn\\nfrom torch.autograd import Variable\\n\\n\\nclass ConvGRUCell(nn.Module):\\n def __init__(self, input_size, input_dim, hidden_dim, kernel_size, bias):\\n \\\"\\\"\\\"\\n Initialize the ConvLSTM cell\\n :param input_size: (int, int)\\n Height and width of input tensor as (height, width).\\n :param input_dim: int\\n Number of channels of input tensor.\\n :param hidden_dim: int\\n Number of channels of hidden state.\\n :param kernel_size: (int, int)\\n Size of the convolutional kernel.\\n :param bias: bool\\n Whether or not to add the bias.\\n :param dtype: torch.cuda.FloatTensor or torch.FloatTensor\\n Whether or not to use cuda.\\n \\\"\\\"\\\"\\n super(ConvGRUCell, self).__init__()\\n self.height, self.width = input_size\\n self.padding = kernel_size[0] // 2, kernel_size[1] // 2\\n self.hidden_dim = hidden_dim\\n self.bias = bias\\n\\n self.conv_gates = nn.Conv2d(in_channels=input_dim + hidden_dim,\\n out_channels=2 * self.hidden_dim,\\n # for update_gate,reset_gate respectively\\n kernel_size=kernel_size,\\n padding=self.padding,\\n bias=self.bias)\\n\\n self.conv_can = nn.Conv2d(in_channels=input_dim + hidden_dim,\\n out_channels=self.hidden_dim,\\n # for candidate neural memory\\n kernel_size=kernel_size,\\n padding=self.padding,\\n bias=self.bias)\\n\\n def init_hidden(self, batch_size):\\n return (Variable(\\n torch.zeros(batch_size, self.hidden_dim, self.height, self.width)))\\n\\n def forward(self, input_tensor, h_cur):\\n \\\"\\\"\\\"\\n :param self:\\n :param input_tensor: (b, c, h, w)\\n input is actually the target_model\\n :param h_cur: (b, c_hidden, h, w)\\n current hidden and cell states respectively\\n :return: h_next,\\n next hidden state\\n \\\"\\\"\\\"\\n combined = torch.cat([input_tensor, h_cur], dim=1)\\n combined_conv = self.conv_gates(combined)\\n\\n gamma, beta = torch.split(combined_conv, self.hidden_dim, dim=1)\\n reset_gate = torch.sigmoid(gamma)\\n update_gate = torch.sigmoid(beta)\\n\\n combined = torch.cat([input_tensor, reset_gate * h_cur], dim=1)\\n cc_cnm = self.conv_can(combined)\\n cnm = torch.tanh(cc_cnm)\\n\\n h_next = (1 - update_gate) * h_cur + update_gate * cnm\\n return h_next\\n\\n\\nclass ConvGRU(nn.Module):\\n def __init__(self, input_size, input_dim, hidden_dim, kernel_size,\\n num_layers,\\n batch_first=False, bias=True, return_all_layers=False):\\n \\\"\\\"\\\"\\n :param input_size: (int, int)\\n Height and width of input tensor as (height, width).\\n :param input_dim: int e.g. 256\\n Number of channels of input tensor.\\n :param hidden_dim: int e.g. 1024\\n Number of channels of hidden state.\\n :param kernel_size: (int, int)\\n Size of the convolutional kernel.\\n :param num_layers: int\\n Number of ConvLSTM layers\\n :param dtype: torch.cuda.FloatTensor or torch.FloatTensor\\n Whether or not to use cuda.\\n :param alexnet_path: str\\n pretrained alexnet parameters\\n :param batch_first: bool\\n if the first position of array is batch or not\\n :param bias: bool\\n Whether or not to add the bias.\\n :param return_all_layers: bool\\n if return hidden and cell states for all layers\\n \\\"\\\"\\\"\\n super(ConvGRU, self).__init__()\\n\\n # Make sure that both `kernel_size` and\\n # `hidden_dim` are lists having len == num_layers\\n kernel_size = self._extend_for_multilayer(kernel_size, num_layers)\\n hidden_dim = self._extend_for_multilayer(hidden_dim, num_layers)\\n if not len(kernel_size) == len(hidden_dim) == num_layers:\\n raise ValueError('Inconsistent list length.')\\n\\n self.height, self.width = input_size\\n self.input_dim = input_dim\\n self.hidden_dim = hidden_dim\\n self.kernel_size = kernel_size\\n self.num_layers = num_layers\\n self.batch_first = batch_first\\n self.bias = bias\\n self.return_all_layers = return_all_layers\\n\\n cell_list = []\\n for i in range(0, self.num_layers):\\n cur_input_dim = input_dim if i == 0 else hidden_dim[i - 1]\\n cell_list.append(ConvGRUCell(input_size=(self.height, self.width),\\n input_dim=cur_input_dim,\\n hidden_dim=self.hidden_dim[i],\\n kernel_size=self.kernel_size[i],\\n bias=self.bias))\\n\\n # convert python list to pytorch module\\n self.cell_list = nn.ModuleList(cell_list)\\n\\n def forward(self, input_tensor, hidden_state=None):\\n \\\"\\\"\\\"\\n :param input_tensor: (b, t, c, h, w) or (t,b,c,h,w)\\n depends on if batch first or not extracted features from alexnet\\n :param hidden_state:\\n :return: layer_output_list, last_state_list\\n \\\"\\\"\\\"\\n if not self.batch_first:\\n # (t, b, c, h, w) -> (b, t, c, h, w)\\n input_tensor = input_tensor.permute(1, 0, 2, 3, 4)\\n\\n # Implement stateful ConvLSTM\\n if hidden_state is not None:\\n raise NotImplementedError()\\n else:\\n hidden_state = self._init_hidden(batch_size=input_tensor.size(0),\\n device=input_tensor.device,\\n dtype=input_tensor.dtype)\\n\\n layer_output_list = []\\n last_state_list = []\\n\\n seq_len = input_tensor.size(1)\\n cur_layer_input = input_tensor\\n\\n for layer_idx in range(self.num_layers):\\n h = hidden_state[layer_idx]\\n output_inner = []\\n for t in range(seq_len):\\n # input current hidden and cell state\\n # then compute the next hidden\\n # and cell state through ConvLSTMCell forward function\\n h = self.cell_list[layer_idx](\\n input_tensor=cur_layer_input[:, t, :, :, :], # (b,t,c,h,w)\\n h_cur=h)\\n output_inner.append(h)\\n\\n layer_output = torch.stack(output_inner, dim=1)\\n cur_layer_input = layer_output\\n\\n layer_output_list.append(layer_output)\\n last_state_list.append([h])\\n\\n if not self.return_all_layers:\\n layer_output_list = layer_output_list[-1:]\\n last_state_list = last_state_list[-1:]\\n\\n return layer_output_list, last_state_list\\n\\n def _init_hidden(self, batch_size, device=None, dtype=None):\\n init_states = []\\n for i in range(self.num_layers):\\n init_states.append(\\n self.cell_list[i].init_hidden(batch_size).to(device).to(dtype))\\n return init_states\\n\\n @staticmethod\\n def _check_kernel_size_consistency(kernel_size):\\n if not (isinstance(kernel_size, tuple) or\\n (isinstance(kernel_size, list) and all(\\n [isinstance(elem, tuple) for elem in kernel_size]))):\\n raise ValueError('`kernel_size` must be tuple or list of tuples')\\n\\n @staticmethod\\n def _extend_for_multilayer(param, num_layers):\\n if not isinstance(param, list):\\n param = [param] * num_layers\\n return param\\n\\n\\nif __name__ == '__main__':\\n # set CUDA device\\n os.environ[\\\"CUDA_VISIBLE_DEVICES\\\"] = \\\"3\\\"\\n\\n # detect if CUDA is available or not\\n use_gpu = torch.cuda.is_available()\\n # if use_gpu:\\n # dtype = torch.cuda.FloatTensor # computation in GPU\\n # else:\\n # dtype = torch.FloatTensor\\n\\n height = width = 6\\n channels = 256\\n hidden_dim = [32, 64]\\n kernel_size = (3, 3) # kernel size for two stacked hidden layer\\n num_layers = 2 # number of stacked hidden layer\\n model = ConvGRU(input_size=(height, width),\\n input_dim=channels,\\n hidden_dim=hidden_dim,\\n kernel_size=kernel_size,\\n num_layers=num_layers,\\n batch_first=True,\\n bias=True,\\n return_all_layers=False)\\n\\n batch_size = 1\\n time_steps = 1\\n input_tensor = torch.rand(batch_size, time_steps, channels, height,\\n width) # (b,t,c,h,w)\\n layer_output_list, last_state_list = model(input_tensor)\\n\\n\\n\\\"\\\"\\\"\\nPillar VFE, credits to OpenPCDet.\\n\\\"\\\"\\\"\\n\\nimport torch\\nimport torch.nn as nn\\nimport torch.nn.functional as F\\n\\n\\nclass PFNLayer(nn.Module):\\n def __init__(self,\\n in_channels,\\n out_channels,\\n use_norm=True,\\n last_layer=False):\\n super().__init__()\\n\\n self.last_vfe = last_layer\\n self.use_norm = use_norm\\n if not self.last_vfe:\\n out_channels = out_channels // 2\\n\\n if self.use_norm:\\n self.linear = nn.Linear(in_channels, out_channels, bias=False)\\n self.norm = nn.BatchNorm1d(out_channels, eps=1e-3, momentum=0.01)\\n else:\\n self.linear = nn.Linear(in_channels, out_channels, bias=True)\\n\\n self.part = 50000\\n\\n def forward(self, inputs):\\n if inputs.shape[0] > self.part:\\n # nn.Linear performs randomly when batch size is too large\\n num_parts = inputs.shape[0] // self.part\\n part_linear_out = [self.linear(\\n inputs[num_part * self.part:(num_part + 1) * self.part])\\n for num_part in range(num_parts + 1)]\\n x = torch.cat(part_linear_out, dim=0)\\n else:\\n x = self.linear(inputs)\\n torch.backends.cudnn.enabled = False\\n x = self.norm(x.permute(0, 2, 1)).permute(0, 2,\\n 1) if self.use_norm else x\\n torch.backends.cudnn.enabled = True\\n x = F.relu(x)\\n x_max = torch.max(x, dim=1, keepdim=True)[0]\\n\\n if self.last_vfe:\\n return x_max\\n else:\\n x_repeat = x_max.repeat(1, inputs.shape[1], 1)\\n x_concatenated = torch.cat([x, x_repeat], dim=2)\\n return x_concatenated\\n\\n\\nclass PillarVFE(nn.Module):\\n def __init__(self, model_cfg, num_point_features, voxel_size,\\n point_cloud_range):\\n super().__init__()\\n self.model_cfg = model_cfg\\n\\n self.use_norm = self.model_cfg['use_norm']\\n self.with_distance = self.model_cfg['with_distance']\\n\\n self.use_absolute_xyz = self.model_cfg['use_absolute_xyz']\\n num_point_features += 6 if self.use_absolute_xyz else 3\\n if self.with_distance:\\n num_point_features += 1\\n\\n self.num_filters = self.model_cfg['num_filters']\\n assert len(self.num_filters) > 0\\n num_filters = [num_point_features] + list(self.num_filters)\\n\\n pfn_layers = []\\n for i in range(len(num_filters) - 1):\\n in_filters = num_filters[i]\\n out_filters = num_filters[i + 1]\\n pfn_layers.append(\\n PFNLayer(in_filters, out_filters, self.use_norm,\\n last_layer=(i >= len(num_filters) - 2))\\n )\\n self.pfn_layers = nn.ModuleList(pfn_layers)\\n\\n self.voxel_x = voxel_size[0]\\n self.voxel_y = voxel_size[1]\\n self.voxel_z = voxel_size[2]\\n self.x_offset = self.voxel_x / 2 + point_cloud_range[0]\\n self.y_offset = self.voxel_y / 2 + point_cloud_range[1]\\n self.z_offset = self.voxel_z / 2 + point_cloud_range[2]\\n\\n def get_output_feature_dim(self):\\n return self.num_filters[-1]\\n\\n @staticmethod\\n def get_paddings_indicator(actual_num, max_num, axis=0):\\n actual_num = torch.unsqueeze(actual_num, axis + 1)\\n max_num_shape = [1] * len(actual_num.shape)\\n max_num_shape[axis + 1] = -1\\n max_num = torch.arange(max_num,\\n dtype=torch.int,\\n device=actual_num.device).view(max_num_shape)\\n paddings_indicator = actual_num.int() > max_num\\n return paddings_indicator\\n\\n def forward(self, batch_dict):\\n\\n voxel_features, voxel_num_points, coords = \\\\\\n batch_dict['voxel_features'], batch_dict['voxel_num_points'], \\\\\\n batch_dict['voxel_coords']\\n points_mean = \\\\\\n voxel_features[:, :, :3].sum(dim=1, keepdim=True) / \\\\\\n voxel_num_points.type_as(voxel_features).view(-1, 1, 1)\\n f_cluster = voxel_features[:, :, :3] - points_mean\\n\\n f_center = torch.zeros_like(voxel_features[:, :, :3])\\n f_center[:, :, 0] = voxel_features[:, :, 0] - (\\n coords[:, 3].to(voxel_features.dtype).unsqueeze(\\n 1) * self.voxel_x + self.x_offset)\\n f_center[:, :, 1] = voxel_features[:, :, 1] - (\\n coords[:, 2].to(voxel_features.dtype).unsqueeze(\\n 1) * self.voxel_y + self.y_offset)\\n f_center[:, :, 2] = voxel_features[:, :, 2] - (\\n coords[:, 1].to(voxel_features.dtype).unsqueeze(\\n 1) * self.voxel_z + self.z_offset)\\n\\n if self.use_absolute_xyz:\\n features = [voxel_features, f_cluster, f_center]\\n else:\\n features = [voxel_features[..., 3:], f_cluster, f_center]\\n\\n if self.with_distance:\\n points_dist = torch.norm(voxel_features[:, :, :3], 2, 2,\\n keepdim=True)\\n features.append(points_dist)\\n features = torch.cat(features, dim=-1)\\n\\n voxel_count = features.shape[1]\\n mask = self.get_paddings_indicator(voxel_num_points, voxel_count,\\n axis=0)\\n mask = torch.unsqueeze(mask, -1).type_as(voxel_features)\\n features *= mask\\n for pfn in self.pfn_layers:\\n features = pfn(features)\\n features = features.squeeze()\\n batch_dict['pillar_features'] = features\\n return batch_dict\\n\\n\\n\\\"\\\"\\\"\\ntorch_transformation_utils.py\\n\\\"\\\"\\\"\\nimport os\\n\\nimport torch\\nimport torch.nn.functional as F\\nimport numpy as np\\nimport matplotlib.pyplot as plt\\n\\n\\ndef get_roi_and_cav_mask(shape, cav_mask, spatial_correction_matrix,\\n discrete_ratio, downsample_rate):\\n \\\"\\\"\\\"\\n Get mask for the combination of cav_mask and rorated ROI mask.\\n Parameters\\n ----------\\n shape : tuple\\n Shape of (B, L, H, W, C).\\n cav_mask : torch.Tensor\\n Shape of (B, L).\\n spatial_correction_matrix : torch.Tensor\\n Shape of (B, L, 4, 4)\\n discrete_ratio : float\\n Discrete ratio.\\n downsample_rate : float\\n Downsample rate.\\n\\n Returns\\n -------\\n com_mask : torch.Tensor\\n Combined mask with shape (B, H, W, L, 1).\\n\\n \\\"\\\"\\\"\\n B, L, H, W, C = shape\\n C = 1\\n # (B,L,4,4)\\n dist_correction_matrix = get_discretized_transformation_matrix(\\n spatial_correction_matrix, discrete_ratio,\\n downsample_rate)\\n # (B*L,2,3)\\n T = get_transformation_matrix(\\n dist_correction_matrix.reshape(-1, 2, 3), (H, W))\\n # (B,L,1,H,W)\\n roi_mask = get_rotated_roi((B, L, C, H, W), T)\\n # (B,L,1,H,W)\\n com_mask = combine_roi_and_cav_mask(roi_mask, cav_mask)\\n # (B,H,W,1,L)\\n com_mask = com_mask.permute(0, 3, 4, 2, 1)\\n return com_mask\\n\\n\\ndef combine_roi_and_cav_mask(roi_mask, cav_mask):\\n \\\"\\\"\\\"\\n Combine ROI mask and CAV mask\\n\\n Parameters\\n ----------\\n roi_mask : torch.Tensor\\n Mask for ROI region after considering the spatial transformation/correction.\\n cav_mask : torch.Tensor\\n Mask for CAV to remove padded 0.\\n\\n Returns\\n -------\\n com_mask : torch.Tensor\\n Combined mask.\\n \\\"\\\"\\\"\\n # (B, L, 1, 1, 1)\\n cav_mask = cav_mask.unsqueeze(2).unsqueeze(3).unsqueeze(4)\\n # (B, L, C, H, W)\\n cav_mask = cav_mask.expand(roi_mask.shape)\\n # (B, L, C, H, W)\\n com_mask = roi_mask * cav_mask\\n return com_mask\\n\\n\\ndef get_rotated_roi(shape, correction_matrix):\\n \\\"\\\"\\\"\\n Get rorated ROI mask.\\n\\n Parameters\\n ----------\\n shape : tuple\\n Shape of (B,L,C,H,W).\\n correction_matrix : torch.Tensor\\n Correction matrix with shape (N,2,3).\\n\\n Returns\\n -------\\n roi_mask : torch.Tensor\\n Roated ROI mask with shape (N,2,3).\\n\\n \\\"\\\"\\\"\\n B, L, C, H, W = shape\\n # To reduce the computation, we only need to calculate the mask for the first channel.\\n # (B,L,1,H,W)\\n x = torch.ones((B, L, 1, H, W)).to(correction_matrix.dtype).to(\\n correction_matrix.device)\\n # (B*L,1,H,W)\\n roi_mask = warp_affine(x.reshape(-1, 1, H, W), correction_matrix,\\n dsize=(H, W), mode=\\\"nearest\\\")\\n # (B,L,C,H,W)\\n roi_mask = torch.repeat_interleave(roi_mask, C, dim=1).reshape(B, L, C, H,\\n W)\\n return roi_mask\\n\\n\\ndef get_discretized_transformation_matrix(matrix, discrete_ratio,\\n downsample_rate):\\n \\\"\\\"\\\"\\n Get disretized transformation matrix.\\n Parameters\\n ----------\\n matrix : torch.Tensor\\n Shape -- (B, L, 4, 4) where B is the batch size, L is the max cav\\n number.\\n discrete_ratio : float\\n Discrete ratio.\\n downsample_rate : float/int\\n downsample_rate\\n\\n Returns\\n -------\\n matrix : torch.Tensor\\n Output transformation matrix in 2D with shape (B, L, 2, 3),\\n including 2D transformation and 2D rotation.\\n\\n \\\"\\\"\\\"\\n matrix = matrix[:, :, [0, 1], :][:, :, :, [0, 1, 3]]\\n # normalize the x,y transformation\\n matrix[:, :, :, -1] = matrix[:, :, :, -1] \\\\\\n / (discrete_ratio * downsample_rate)\\n\\n return matrix.type(dtype=torch.float)\\n\\n\\ndef _torch_inverse_cast(input):\\n r\\\"\\\"\\\"\\n Helper function to make torch.inverse work with other than fp32/64.\\n The function torch.inverse is only implemented for fp32/64 which makes\\n impossible to be used by fp16 or others. What this function does,\\n is cast input data type to fp32, apply torch.inverse,\\n and cast back to the input dtype.\\n Args:\\n input : torch.Tensor\\n Tensor to be inversed.\\n\\n Returns:\\n out : torch.Tensor\\n Inversed Tensor.\\n\\n \\\"\\\"\\\"\\n dtype = input.dtype\\n if dtype not in (torch.float32, torch.float64):\\n dtype = torch.float32\\n out = torch.inverse(input.to(dtype)).to(input.dtype)\\n return out\\n\\n\\ndef normal_transform_pixel(\\n height, width, device, dtype, eps=1e-14):\\n r\\\"\\\"\\\"\\n Compute the normalization matrix from image size in pixels to [-1, 1].\\n Args:\\n height : int\\n Image height.\\n width : int\\n Image width.\\n device : torch.device\\n Output tensor devices.\\n dtype : torch.dtype\\n Output tensor data type.\\n eps : float\\n Epsilon to prevent divide-by-zero errors.\\n\\n Returns:\\n tr_mat : torch.Tensor\\n Normalized transform with shape :math:`(1, 3, 3)`.\\n \\\"\\\"\\\"\\n tr_mat = torch.tensor(\\n [[1.0, 0.0, -1.0], [0.0, 1.0, -1.0], [0.0, 0.0, 1.0]], device=device,\\n dtype=dtype) # 3x3\\n\\n # prevent divide by zero bugs\\n width_denom = eps if width == 1 else width - 1.0\\n height_denom = eps if height == 1 else height - 1.0\\n\\n tr_mat[0, 0] = tr_mat[0, 0] * 2.0 / width_denom\\n tr_mat[1, 1] = tr_mat[1, 1] * 2.0 / height_denom\\n\\n return tr_mat.unsqueeze(0) # 1x3x3\\n\\n\\ndef eye_like(n, B, device, dtype):\\n r\\\"\\\"\\\"\\n Return a 2-D tensor with ones on the diagonal and\\n zeros elsewhere with the same batch size as the input.\\n Args:\\n n : int\\n The number of rows :math:`(n)`.\\n B : int\\n Btach size.\\n device : torch.device\\n Devices of the output tensor.\\n dtype : torch.dtype\\n Data type of the output tensor.\\n\\n Returns:\\n The identity matrix with the shape :math:`(B, n, n)`.\\n \\\"\\\"\\\"\\n\\n identity = torch.eye(n, device=device, dtype=dtype)\\n return identity[None].repeat(B, 1, 1)\\n\\n\\ndef normalize_homography(dst_pix_trans_src_pix, dsize_src, dsize_dst=None):\\n r\\\"\\\"\\\"\\n Normalize a given homography in pixels to [-1, 1].\\n Args:\\n dst_pix_trans_src_pix : torch.Tensor\\n Homography/ies from source to destination to be normalized with\\n shape :math:`(B, 3, 3)`.\\n dsize_src : Tuple[int, int]\\n Size of the source image (height, width).\\n dsize_dst : Tuple[int, int]\\n Size of the destination image (height, width).\\n\\n Returns:\\n dst_norm_trans_src_norm : torch.Tensor\\n The normalized homography of shape :math:`(B, 3, 3)`.\\n \\\"\\\"\\\"\\n if dsize_dst is None:\\n dsize_dst = dsize_src\\n # source and destination sizes\\n src_h, src_w = dsize_src\\n dst_h, dst_w = dsize_dst\\n device = dst_pix_trans_src_pix.device\\n dtype = dst_pix_trans_src_pix.dtype\\n # compute the transformation pixel/norm for src/dst\\n src_norm_trans_src_pix = normal_transform_pixel(src_h, src_w, device,\\n dtype).to(\\n dst_pix_trans_src_pix)\\n\\n src_pix_trans_src_norm = _torch_inverse_cast(src_norm_trans_src_pix)\\n dst_norm_trans_dst_pix = normal_transform_pixel(dst_h, dst_w, device,\\n dtype).to(\\n dst_pix_trans_src_pix)\\n # compute chain transformations\\n dst_norm_trans_src_norm: torch.Tensor = dst_norm_trans_dst_pix @ (\\n dst_pix_trans_src_pix @ src_pix_trans_src_norm)\\n return dst_norm_trans_src_norm\\n\\n\\ndef get_rotation_matrix2d(M, dsize):\\n r\\\"\\\"\\\"\\n Return rotation matrix for torch.affine_grid based on transformation matrix.\\n Args:\\n M : torch.Tensor\\n Transformation matrix with shape :math:`(B, 2, 3)`.\\n dsize : Tuple[int, int]\\n Size of the source image (height, width).\\n\\n Returns:\\n R : torch.Tensor\\n Rotation matrix with shape :math:`(B, 2, 3)`.\\n \\\"\\\"\\\"\\n H, W = dsize\\n B = M.shape[0]\\n center = torch.Tensor([W / 2, H / 2]).to(M.dtype).to(M.device).unsqueeze(0)\\n shift_m = eye_like(3, B, M.device, M.dtype)\\n shift_m[:, :2, 2] = center\\n\\n shift_m_inv = eye_like(3, B, M.device, M.dtype)\\n shift_m_inv[:, :2, 2] = -center\\n\\n rotat_m = eye_like(3, B, M.device, M.dtype)\\n rotat_m[:, :2, :2] = M[:, :2, :2]\\n affine_m = shift_m @ rotat_m @ shift_m_inv\\n return affine_m[:, :2, :] # Bx2x3\\n\\n\\ndef get_transformation_matrix(M, dsize):\\n r\\\"\\\"\\\"\\n Return transformation matrix for torch.affine_grid.\\n Args:\\n M : torch.Tensor\\n Transformation matrix with shape :math:`(N, 2, 3)`.\\n dsize : Tuple[int, int]\\n Size of the source image (height, width).\\n\\n Returns:\\n T : torch.Tensor\\n Transformation matrix with shape :math:`(N, 2, 3)`.\\n \\\"\\\"\\\"\\n T = get_rotation_matrix2d(M, dsize)\\n T[..., 2] += M[..., 2]\\n return T\\n\\n\\ndef convert_affinematrix_to_homography(A):\\n r\\\"\\\"\\\"\\n Convert to homography coordinates\\n Args:\\n A : torch.Tensor\\n The affine matrix with shape :math:`(B,2,3)`.\\n\\n Returns:\\n H : torch.Tensor\\n The homography matrix with shape of :math:`(B,3,3)`.\\n \\\"\\\"\\\"\\n H: torch.Tensor = torch.nn.functional.pad(A, [0, 0, 0, 1], \\\"constant\\\",\\n value=0.0)\\n H[..., -1, -1] += 1.0\\n return H\\n\\n\\ndef warp_affine(\\n src, M, dsize,\\n mode='bilinear',\\n padding_mode='zeros',\\n align_corners=True):\\n r\\\"\\\"\\\"\\n Transform the src based on transformation matrix M.\\n Args:\\n src : torch.Tensor\\n Input feature map with shape :math:`(B,C,H,W)`.\\n M : torch.Tensor\\n Transformation matrix with shape :math:`(B,2,3)`.\\n dsize : tuple\\n Tuple of output image H_out and W_out.\\n mode : str\\n Interpolation methods for F.grid_sample.\\n padding_mode : str\\n Padding methods for F.grid_sample.\\n align_corners : boolean\\n Parameter of F.affine_grid.\\n\\n Returns:\\n Transformed features with shape :math:`(B,C,H,W)`.\\n \\\"\\\"\\\"\\n\\n B, C, H, W = src.size()\\n\\n # we generate a 3x3 transformation matrix from 2x3 affine\\n M_3x3 = convert_affinematrix_to_homography(M)\\n dst_norm_trans_src_norm = normalize_homography(M_3x3, (H, W), dsize)\\n\\n # src_norm_trans_dst_norm = torch.inverse(dst_norm_trans_src_norm)\\n src_norm_trans_dst_norm = _torch_inverse_cast(dst_norm_trans_src_norm)\\n grid = F.affine_grid(src_norm_trans_dst_norm[:, :2, :],\\n [B, C, dsize[0], dsize[1]],\\n align_corners=align_corners)\\n\\n return F.grid_sample(src.half() if grid.dtype == torch.half else src, grid,\\n align_corners=align_corners, mode=mode,\\n padding_mode=padding_mode)\\n\\n\\nclass Test:\\n \\\"\\\"\\\"\\n Test the transformation in this file.\\n The methods in this class are not supposed to be used outside of this file.\\n \\\"\\\"\\\"\\n\\n def __init__(self):\\n pass\\n\\n @staticmethod\\n def load_img():\\n torch.manual_seed(0)\\n x = torch.randn(1, 5, 16, 400, 200) * 100\\n # x = torch.ones(1, 5, 16, 400, 200)\\n return x\\n\\n @staticmethod\\n def load_raw_transformation_matrix(N):\\n a = 90 / 180 * np.pi\\n matrix = torch.Tensor([[np.cos(a), -np.sin(a), 10],\\n [np.sin(a), np.cos(a), 10]])\\n matrix = torch.repeat_interleave(matrix.unsqueeze(0).unsqueeze(0), N,\\n dim=1)\\n return matrix\\n\\n @staticmethod\\n def load_raw_transformation_matrix2(N, alpha):\\n a = alpha / 180 * np.pi\\n matrix = torch.Tensor([[np.cos(a), -np.sin(a), 0, 0],\\n [np.sin(a), np.cos(a), 0, 0]])\\n matrix = torch.repeat_interleave(matrix.unsqueeze(0).unsqueeze(0), N,\\n dim=1)\\n return matrix\\n\\n @staticmethod\\n def test():\\n img = Test.load_img()\\n B, L, C, H, W = img.shape\\n raw_T = Test.load_raw_transformation_matrix(5)\\n T = get_transformation_matrix(raw_T.reshape(-1, 2, 3), (H, W))\\n img_rot = warp_affine(img.reshape(-1, C, H, W), T, (H, W))\\n print(img_rot[0, 0, :, :])\\n plt.matshow(img_rot[0, 0, :, :])\\n plt.show()\\n\\n @staticmethod\\n def test_combine_roi_and_cav_mask():\\n B = 2\\n L = 5\\n C = 16\\n H = 300\\n W = 400\\n # 2, 5\\n cav_mask = torch.Tensor([[1, 1, 1, 0, 0], [1, 0, 0, 0, 0]])\\n x = torch.zeros(B, L, C, H, W)\\n correction_matrix = Test.load_raw_transformation_matrix2(5, 10)\\n correction_matrix = torch.cat([correction_matrix, correction_matrix],\\n dim=0)\\n mask = get_roi_and_cav_mask((B, L, H, W, C), cav_mask,\\n correction_matrix, 0.4, 4)\\n plt.matshow(mask[0, :, :, 0, 0])\\n plt.show()\\n\\n\\nif __name__ == \\\"__main__\\\":\\n os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'\\n Test.test_combine_roi_and_cav_mask()\\n\\n\\nimport torch\\nimport torch.nn as nn\\nimport torch.nn.functional as F\\n\\n\\nclass RadixSoftmax(nn.Module):\\n def __init__(self, radix, cardinality):\\n super(RadixSoftmax, self).__init__()\\n self.radix = radix\\n self.cardinality = cardinality\\n\\n def forward(self, x):\\n # x: (B, L, 1, 1, 3C)\\n batch = x.size(0)\\n cav_num = x.size(1)\\n\\n if self.radix > 1:\\n # x: (B, L, 1, 3, C)\\n x = x.view(batch,\\n cav_num,\\n self.cardinality, self.radix, -1)\\n x = F.softmax(x, dim=3)\\n # B, 3LC\\n x = x.reshape(batch, -1)\\n else:\\n x = torch.sigmoid(x)\\n return x\\n\\n\\nclass SplitAttn(nn.Module):\\n def __init__(self, input_dim):\\n super(SplitAttn, self).__init__()\\n self.input_dim = input_dim\\n\\n self.fc1 = nn.Linear(input_dim, input_dim, bias=False)\\n self.bn1 = nn.LayerNorm(input_dim)\\n self.act1 = nn.ReLU()\\n self.fc2 = nn.Linear(input_dim, input_dim * 3, bias=False)\\n\\n self.rsoftmax = RadixSoftmax(3, 1)\\n\\n def forward(self, window_list):\\n # window list: [(B, L, H, W, C) * 3]\\n assert len(window_list) == 3, 'only 3 windows are supported'\\n\\n sw, mw, bw = window_list[0], window_list[1], window_list[2]\\n B, L = sw.shape[0], sw.shape[1]\\n\\n # global average pooling, B, L, H, W, C\\n x_gap = sw + mw + bw\\n # B, L, 1, 1, C\\n x_gap = x_gap.mean((2, 3), keepdim=True)\\n x_gap = self.act1(self.bn1(self.fc1(x_gap)))\\n # B, L, 1, 1, 3C\\n x_attn = self.fc2(x_gap)\\n # B L 1 1 3C\\n x_attn = self.rsoftmax(x_attn).view(B, L, 1, 1, -1)\\n\\n out = sw * x_attn[:, :, :, :, 0:self.input_dim] + \\\\\\n mw * x_attn[:, :, :, :, self.input_dim:2*self.input_dim] +\\\\\\n bw * x_attn[:, :, :, :, self.input_dim*2:]\\n\\n return out\\n\\n\\nimport numpy as np\\nimport torch\\nimport torch.nn as nn\\n\\n\\nclass BaseBEVBackbone(nn.Module):\\n def __init__(self, model_cfg, input_channels):\\n super().__init__()\\n self.model_cfg = model_cfg\\n\\n if 'layer_nums' in self.model_cfg:\\n\\n assert len(self.model_cfg['layer_nums']) == \\\\\\n len(self.model_cfg['layer_strides']) == \\\\\\n len(self.model_cfg['num_filters'])\\n\\n layer_nums = self.model_cfg['layer_nums']\\n layer_strides = self.model_cfg['layer_strides']\\n num_filters = self.model_cfg['num_filters']\\n else:\\n layer_nums = layer_strides = num_filters = []\\n\\n if 'upsample_strides' in self.model_cfg:\\n assert len(self.model_cfg['upsample_strides']) \\\\\\n == len(self.model_cfg['num_upsample_filter'])\\n\\n num_upsample_filters = self.model_cfg['num_upsample_filter']\\n upsample_strides = self.model_cfg['upsample_strides']\\n\\n else:\\n upsample_strides = num_upsample_filters = []\\n\\n num_levels = len(layer_nums)\\n c_in_list = [input_channels, *num_filters[:-1]]\\n\\n self.blocks = nn.ModuleList()\\n self.deblocks = nn.ModuleList()\\n\\n for idx in range(num_levels):\\n cur_layers = [\\n nn.ZeroPad2d(1),\\n nn.Conv2d(\\n c_in_list[idx], num_filters[idx], kernel_size=3,\\n stride=layer_strides[idx], padding=0, bias=False\\n ),\\n nn.BatchNorm2d(num_filters[idx], eps=1e-3, momentum=0.01),\\n nn.ReLU()\\n ]\\n for k in range(layer_nums[idx]):\\n cur_layers.extend([\\n nn.Conv2d(num_filters[idx], num_filters[idx],\\n kernel_size=3, padding=1, bias=False),\\n nn.BatchNorm2d(num_filters[idx], eps=1e-3, momentum=0.01),\\n nn.ReLU()\\n ])\\n\\n self.blocks.append(nn.Sequential(*cur_layers))\\n if len(upsample_strides) > 0:\\n stride = upsample_strides[idx]\\n if stride >= 1:\\n self.deblocks.append(nn.Sequential(\\n nn.ConvTranspose2d(\\n num_filters[idx], num_upsample_filters[idx],\\n upsample_strides[idx],\\n stride=upsample_strides[idx], bias=False\\n ),\\n nn.BatchNorm2d(num_upsample_filters[idx],\\n eps=1e-3, momentum=0.01),\\n nn.ReLU()\\n ))\\n else:\\n stride = np.round(1 / stride).astype(np.int)\\n self.deblocks.append(nn.Sequential(\\n nn.Conv2d(\\n num_filters[idx], num_upsample_filters[idx],\\n stride,\\n stride=stride, bias=False\\n ),\\n nn.BatchNorm2d(num_upsample_filters[idx], eps=1e-3,\\n momentum=0.01),\\n nn.ReLU()\\n ))\\n\\n c_in = sum(num_upsample_filters)\\n if len(upsample_strides) > num_levels:\\n self.deblocks.append(nn.Sequential(\\n nn.ConvTranspose2d(c_in, c_in, upsample_strides[-1],\\n stride=upsample_strides[-1], bias=False),\\n nn.BatchNorm2d(c_in, eps=1e-3, momentum=0.01),\\n nn.ReLU(),\\n ))\\n\\n self.num_bev_features = c_in\\n\\n def forward(self, data_dict):\\n spatial_features = data_dict['spatial_features']\\n\\n ups = []\\n ret_dict = {}\\n x = spatial_features\\n\\n for i in range(len(self.blocks)):\\n x = self.blocks[i](x)\\n\\n stride = int(spatial_features.shape[2] / x.shape[2])\\n ret_dict['spatial_features_%dx' % stride] = x\\n\\n if len(self.deblocks) > 0:\\n ups.append(self.deblocks[i](x))\\n else:\\n ups.append(x)\\n\\n if len(ups) > 1:\\n x = torch.cat(ups, dim=1)\\n elif len(ups) == 1:\\n x = ups[0]\\n\\n if len(self.deblocks) > len(self.blocks):\\n x = self.deblocks[-1](x)\\n\\n data_dict['spatial_features_2d'] = x\\n return data_dict\\n\\n\\nimport torch\\nimport numpy as np\\n\\nfrom einops import rearrange\\nfrom v2xvit.utils.common_utils import torch_tensor_to_numpy\\n\\n\\ndef regroup(dense_feature, record_len, max_len):\\n \\\"\\\"\\\"\\n Regroup the data based on the record_len.\\n\\n Parameters\\n ----------\\n dense_feature : torch.Tensor\\n N, C, H, W\\n record_len : list\\n [sample1_len, sample2_len, ...]\\n max_len : int\\n Maximum cav number\\n\\n Returns\\n -------\\n regroup_feature : torch.Tensor\\n B, L, C, H, W\\n \\\"\\\"\\\"\\n cum_sum_len = list(np.cumsum(torch_tensor_to_numpy(record_len)))\\n split_features = torch.tensor_split(dense_feature,\\n cum_sum_len[:-1])\\n regroup_features = []\\n mask = []\\n\\n for split_feature in split_features:\\n # M, C, H, W\\n feature_shape = split_feature.shape\\n\\n # the maximum M is 5 as most 5 cavs\\n padding_len = max_len - feature_shape[0]\\n mask.append([1] * feature_shape[0] + [0] * padding_len)\\n\\n padding_tensor = torch.zeros(padding_len, feature_shape[1],\\n feature_shape[2], feature_shape[3])\\n padding_tensor = padding_tensor.to(split_feature.device)\\n\\n split_feature = torch.cat([split_feature, padding_tensor],\\n dim=0)\\n\\n # 1, 5C, H, W\\n split_feature = split_feature.view(-1,\\n feature_shape[2],\\n feature_shape[3]).unsqueeze(0)\\n regroup_features.append(split_feature)\\n\\n # B, 5C, H, W\\n regroup_features = torch.cat(regroup_features, dim=0)\\n # B, L, C, H, W\\n regroup_features = rearrange(regroup_features,\\n 'b (l c) h w -> b l c h w',\\n l=max_len)\\n mask = torch.from_numpy(np.array(mask)).to(regroup_features.device)\\n\\n return regroup_features, mask\\n\\n\\nimport torch\\nimport torch.nn as nn\\n\\n\\nclass PointPillarScatter(nn.Module):\\n def __init__(self, model_cfg):\\n super().__init__()\\n\\n self.model_cfg = model_cfg\\n self.num_bev_features = self.model_cfg['num_features']\\n self.nx, self.ny, self.nz = model_cfg['grid_size']\\n assert self.nz == 1\\n\\n def forward(self, batch_dict):\\n pillar_features, coords = batch_dict['pillar_features'], batch_dict[\\n 'voxel_coords']\\n batch_spatial_features = []\\n batch_size = coords[:, 0].max().int().item() + 1\\n\\n for batch_idx in range(batch_size):\\n spatial_feature = torch.zeros(\\n self.num_bev_features,\\n self.nz * self.nx * self.ny,\\n dtype=pillar_features.dtype,\\n device=pillar_features.device)\\n\\n batch_mask = coords[:, 0] == batch_idx\\n this_coords = coords[batch_mask, :]\\n\\n indices = this_coords[:, 1] + \\\\\\n this_coords[:, 2] * self.nx + \\\\\\n this_coords[:, 3]\\n indices = indices.type(torch.long)\\n\\n pillars = pillar_features[batch_mask, :]\\n pillars = pillars.t()\\n spatial_feature[:, indices] = pillars\\n batch_spatial_features.append(spatial_feature)\\n\\n batch_spatial_features = \\\\\\n torch.stack(batch_spatial_features, 0)\\n batch_spatial_features = \\\\\\n batch_spatial_features.view(batch_size, self.num_bev_features *\\n self.nz, self.ny, self.nx)\\n batch_dict['spatial_features'] = batch_spatial_features\\n\\n return batch_dict\\n\\n\\n\\nimport torch\\nfrom torch import nn\\n\\nfrom einops import rearrange\\n\\n\\nclass PreNorm(nn.Module):\\n def __init__(self, dim, fn):\\n super().__init__()\\n self.norm = nn.LayerNorm(dim)\\n self.fn = fn\\n\\n def forward(self, x, **kwargs):\\n return self.fn(self.norm(x), **kwargs)\\n\\n\\nclass FeedForward(nn.Module):\\n def __init__(self, dim, hidden_dim, dropout=0.):\\n super().__init__()\\n self.net = nn.Sequential(\\n nn.Linear(dim, hidden_dim),\\n nn.GELU(),\\n nn.Dropout(dropout),\\n nn.Linear(hidden_dim, dim),\\n nn.Dropout(dropout)\\n )\\n\\n def forward(self, x):\\n return self.net(x)\\n\\n\\nclass CavAttention(nn.Module):\\n \\\"\\\"\\\"\\n Vanilla CAV attention.\\n \\\"\\\"\\\"\\n def __init__(self, dim, heads, dim_head=64, dropout=0.1):\\n super().__init__()\\n inner_dim = heads * dim_head\\n\\n self.heads = heads\\n self.scale = dim_head ** -0.5\\n\\n self.attend = nn.Softmax(dim=-1)\\n self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)\\n\\n self.to_out = nn.Sequential(\\n nn.Linear(inner_dim, dim),\\n nn.Dropout(dropout)\\n )\\n\\n def forward(self, x, mask, prior_encoding):\\n # x: (B, L, H, W, C) -> (B, H, W, L, C)\\n # mask: (B, L)\\n x = x.permute(0, 2, 3, 1, 4)\\n # mask: (B, 1, H, W, L, 1)\\n mask = mask.unsqueeze(1)\\n\\n # qkv: [(B, H, W, L, C_inner) *3]\\n qkv = self.to_qkv(x).chunk(3, dim=-1)\\n # q: (B, M, H, W, L, C)\\n q, k, v = map(lambda t: rearrange(t, 'b h w l (m c) -> b m h w l c',\\n m=self.heads), qkv)\\n\\n # attention, (B, M, H, W, L, L)\\n att_map = torch.einsum('b m h w i c, b m h w j c -> b m h w i j',\\n q, k) * self.scale\\n # add mask\\n att_map = att_map.masked_fill(mask == 0, -float('inf'))\\n # softmax\\n att_map = self.attend(att_map)\\n\\n # out:(B, M, H, W, L, C_head)\\n out = torch.einsum('b m h w i j, b m h w j c -> b m h w i c', att_map,\\n v)\\n out = rearrange(out, 'b m h w l c -> b h w l (m c)',\\n m=self.heads)\\n out = self.to_out(out)\\n # (B L H W C)\\n out = out.permute(0, 3, 1, 2, 4)\\n return out\\n\\n\\nclass BaseEncoder(nn.Module):\\n def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout=0.):\\n super().__init__()\\n self.layers = nn.ModuleList([])\\n for _ in range(depth):\\n self.layers.append(nn.ModuleList([\\n PreNorm(dim, CavAttention(dim,\\n heads=heads,\\n dim_head=dim_head,\\n dropout=dropout)),\\n PreNorm(dim, FeedForward(dim, mlp_dim, dropout=dropout))\\n ]))\\n\\n def forward(self, x, mask):\\n for attn, ff in self.layers:\\n x = attn(x, mask=mask) + x\\n x = ff(x) + x\\n return x\\n\\n\\nclass BaseTransformer(nn.Module):\\n def __init__(self, args):\\n super().__init__()\\n\\n dim = args['dim']\\n depth = args['depth']\\n heads = args['heads']\\n dim_head = args['dim_head']\\n mlp_dim = args['mlp_dim']\\n dropout = args['dropout']\\n max_cav = args['max_cav']\\n\\n self.encoder = BaseEncoder(dim, depth, heads, dim_head, mlp_dim,\\n dropout)\\n\\n def forward(self, x, mask):\\n # B, L, H, W, C\\n output = self.encoder(x, mask)\\n # B, H, W, C\\n output = output[:, 0]\\n\\n return output\\n\\n\\\"\\\"\\\"\\nImplementation of F-cooper maxout fusing.\\n\\\"\\\"\\\"\\nimport torch\\nimport torch.nn as nn\\n\\n\\nclass SpatialFusion(nn.Module):\\n def __init__(self):\\n super(SpatialFusion, self).__init__()\\n\\n def regroup(self, x, record_len):\\n cum_sum_len = torch.cumsum(record_len, dim=0)\\n split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())\\n return split_x\\n\\n def forward(self, x, record_len):\\n # x: B, C, H, W, split x:[(B1, C, W, H), (B2, C, W, H)]\\n split_x = self.regroup(x, record_len)\\n out = []\\n\\n for xx in split_x:\\n xx = torch.max(xx, dim=0, keepdim=True)[0]\\n out.append(xx)\\n return torch.cat(out, dim=0)\\n\\nimport torch\\nimport torch.nn as nn\\n\\n\\nclass NaiveCompressor(nn.Module):\\n def __init__(self, input_dim, compress_raito):\\n super().__init__()\\n self.encoder = nn.Sequential(\\n nn.Conv2d(input_dim, input_dim//compress_raito, kernel_size=3,\\n stride=1, padding=1),\\n nn.BatchNorm2d(input_dim//compress_raito, eps=1e-3, momentum=0.01),\\n nn.ReLU()\\n )\\n self.decoder = nn.Sequential(\\n nn.Conv2d(input_dim//compress_raito, input_dim, kernel_size=3,\\n stride=1, padding=1),\\n nn.BatchNorm2d(input_dim, eps=1e-3, momentum=0.01),\\n nn.ReLU(),\\n nn.Conv2d(input_dim, input_dim, kernel_size=3, stride=1, padding=1),\\n nn.BatchNorm2d(input_dim, eps=1e-3,\\n momentum=0.01),\\n nn.ReLU()\\n )\\n\\n def forward(self, x):\\n x = self.encoder(x)\\n x = self.decoder(x)\\n\\n return x\\n\\nimport math\\n\\nfrom v2xvit.models.sub_modules.base_transformer import *\\nfrom v2xvit.models.sub_modules.hmsa import *\\nfrom v2xvit.models.sub_modules.mswin import *\\nfrom v2xvit.models.sub_modules.torch_transformation_utils import \\\\\\n get_transformation_matrix, warp_affine, get_roi_and_cav_mask, \\\\\\n get_discretized_transformation_matrix\\n\\n\\nclass STTF(nn.Module):\\n def __init__(self, args):\\n super(STTF, self).__init__()\\n self.discrete_ratio = args['voxel_size'][0]\\n self.downsample_rate = args['downsample_rate']\\n\\n def forward(self, x, mask, spatial_correction_matrix):\\n x = x.permute(0, 1, 4, 2, 3)\\n dist_correction_matrix = get_discretized_transformation_matrix(\\n spatial_correction_matrix, self.discrete_ratio,\\n self.downsample_rate)\\n # Only compensate non-ego vehicles\\n B, L, C, H, W = x.shape\\n\\n T = get_transformation_matrix(\\n dist_correction_matrix[:, 1:, :, :].reshape(-1, 2, 3), (H, W))\\n cav_features = warp_affine(x[:, 1:, :, :, :].reshape(-1, C, H, W), T,\\n (H, W))\\n cav_features = cav_features.reshape(B, -1, C, H, W)\\n x = torch.cat([x[:, 0, :, :, :].unsqueeze(1), cav_features], dim=1)\\n x = x.permute(0, 1, 3, 4, 2)\\n return x\\n\\n\\nclass RelTemporalEncoding(nn.Module):\\n \\\"\\\"\\\"\\n Implement the Temporal Encoding (Sinusoid) function.\\n \\\"\\\"\\\"\\n\\n def __init__(self, n_hid, RTE_ratio, max_len=100, dropout=0.2):\\n super(RelTemporalEncoding, self).__init__()\\n position = torch.arange(0., max_len).unsqueeze(1)\\n div_term = torch.exp(torch.arange(0, n_hid, 2) *\\n -(math.log(10000.0) / n_hid))\\n emb = nn.Embedding(max_len, n_hid)\\n emb.weight.data[:, 0::2] = torch.sin(position * div_term) / math.sqrt(\\n n_hid)\\n emb.weight.data[:, 1::2] = torch.cos(position * div_term) / math.sqrt(\\n n_hid)\\n emb.requires_grad = False\\n self.RTE_ratio = RTE_ratio\\n self.emb = emb\\n self.lin = nn.Linear(n_hid, n_hid)\\n\\n def forward(self, x, t):\\n # When t has unit of 50ms, rte_ratio=1.\\n # So we can train on 100ms but test on 50ms\\n return x + self.lin(self.emb(t * self.RTE_ratio)).unsqueeze(\\n 0).unsqueeze(1)\\n\\n\\nclass RTE(nn.Module):\\n def __init__(self, dim, RTE_ratio=2):\\n super(RTE, self).__init__()\\n self.RTE_ratio = RTE_ratio\\n\\n self.emb = RelTemporalEncoding(dim, RTE_ratio=self.RTE_ratio)\\n\\n def forward(self, x, dts):\\n # x: (B,L,H,W,C)\\n # dts: (B,L)\\n rte_batch = []\\n for b in range(x.shape[0]):\\n rte_list = []\\n for i in range(x.shape[1]):\\n rte_list.append(\\n self.emb(x[b, i, :, :, :], dts[b, i]).unsqueeze(0))\\n rte_batch.append(torch.cat(rte_list, dim=0).unsqueeze(0))\\n return torch.cat(rte_batch, dim=0)\\n\\n\\nclass V2XFusionBlock(nn.Module):\\n def __init__(self, num_blocks, cav_att_config, pwindow_config):\\n super().__init__()\\n # first multi-agent attention and then multi-window attention\\n self.layers = nn.ModuleList([])\\n self.num_blocks = num_blocks\\n\\n for _ in range(num_blocks):\\n att = HGTCavAttention(cav_att_config['dim'],\\n heads=cav_att_config['heads'],\\n dim_head=cav_att_config['dim_head'],\\n dropout=cav_att_config['dropout']) if \\\\\\n cav_att_config['use_hetero'] else \\\\\\n CavAttention(cav_att_config['dim'],\\n heads=cav_att_config['heads'],\\n dim_head=cav_att_config['dim_head'],\\n dropout=cav_att_config['dropout'])\\n self.layers.append(nn.ModuleList([\\n PreNorm(cav_att_config['dim'], att),\\n PreNorm(cav_att_config['dim'],\\n PyramidWindowAttention(pwindow_config['dim'],\\n heads=pwindow_config['heads'],\\n dim_heads=pwindow_config[\\n 'dim_head'],\\n drop_out=pwindow_config[\\n 'dropout'],\\n window_size=pwindow_config[\\n 'window_size'],\\n relative_pos_embedding=\\n pwindow_config[\\n 'relative_pos_embedding'],\\n fuse_method=pwindow_config[\\n 'fusion_method']))]))\\n\\n def forward(self, x, mask, prior_encoding):\\n for cav_attn, pwindow_attn in self.layers:\\n x = cav_attn(x, mask=mask, prior_encoding=prior_encoding) + x\\n x = pwindow_attn(x) + x\\n return x\\n\\n\\nclass V2XTEncoder(nn.Module):\\n def __init__(self, args):\\n super().__init__()\\n\\n cav_att_config = args['cav_att_config']\\n pwindow_att_config = args['pwindow_att_config']\\n feed_config = args['feed_forward']\\n\\n num_blocks = args['num_blocks']\\n depth = args['depth']\\n mlp_dim = feed_config['mlp_dim']\\n dropout = feed_config['dropout']\\n\\n self.downsample_rate = args['sttf']['downsample_rate']\\n self.discrete_ratio = args['sttf']['voxel_size'][0]\\n self.use_roi_mask = args['use_roi_mask']\\n self.use_RTE = cav_att_config['use_RTE']\\n self.RTE_ratio = cav_att_config['RTE_ratio']\\n self.sttf = STTF(args['sttf'])\\n # adjust the channel numbers from 256+3 -> 256\\n self.prior_feed = nn.Linear(cav_att_config['dim'] + 3,\\n cav_att_config['dim'])\\n self.layers = nn.ModuleList([])\\n if self.use_RTE:\\n self.rte = RTE(cav_att_config['dim'], self.RTE_ratio)\\n for _ in range(depth):\\n self.layers.append(nn.ModuleList([\\n V2XFusionBlock(num_blocks, cav_att_config, pwindow_att_config),\\n PreNorm(cav_att_config['dim'],\\n FeedForward(cav_att_config['dim'], mlp_dim,\\n dropout=dropout))\\n ]))\\n\\n def forward(self, x, mask, spatial_correction_matrix):\\n\\n # transform the features to the current timestamp\\n # velocity, time_delay, infra\\n # (B,L,H,W,3)\\n prior_encoding = x[..., -3:]\\n # (B,L,H,W,C)\\n x = x[..., :-3]\\n if self.use_RTE:\\n # dt: (B,L)\\n dt = prior_encoding[:, :, 0, 0, 1].to(torch.int)\\n x = self.rte(x, dt)\\n x = self.sttf(x, mask, spatial_correction_matrix)\\n com_mask = mask.unsqueeze(1).unsqueeze(2).unsqueeze(\\n 3) if not self.use_roi_mask else get_roi_and_cav_mask(x.shape,\\n mask,\\n spatial_correction_matrix,\\n self.discrete_ratio,\\n self.downsample_rate)\\n for attn, ff in self.layers:\\n x = attn(x, mask=com_mask, prior_encoding=prior_encoding)\\n x = ff(x) + x\\n return x\\n\\n\\nclass V2XTransformer(nn.Module):\\n def __init__(self, args):\\n super(V2XTransformer, self).__init__()\\n\\n encoder_args = args['encoder']\\n self.encoder = V2XTEncoder(encoder_args)\\n\\n def forward(self, x, mask, spatial_correction_matrix):\\n output = self.encoder(x, mask, spatial_correction_matrix)\\n output = output[:, 0]\\n return output\\n\\n\\n\\\"\\\"\\\"\\nImplementation of V2VNet Fusion\\n\\\"\\\"\\\"\\n\\nimport torch\\nimport torch.nn as nn\\n\\nfrom v2xvit.models.sub_modules.torch_transformation_utils import \\\\\\n get_discretized_transformation_matrix, get_transformation_matrix, \\\\\\n warp_affine, get_rotated_roi\\nfrom v2xvit.models.sub_modules.convgru import ConvGRU\\n\\n\\nclass V2VNetFusion(nn.Module):\\n def __init__(self, args):\\n super(V2VNetFusion, self).__init__()\\n in_channels = args['in_channels']\\n H, W = args['conv_gru']['H'], args['conv_gru']['W']\\n kernel_size = args['conv_gru']['kernel_size']\\n num_gru_layers = args['conv_gru']['num_layers']\\n\\n self.use_temporal_encoding = args['use_temporal_encoding']\\n self.discrete_ratio = args['voxel_size'][0]\\n self.downsample_rate = args['downsample_rate']\\n self.num_iteration = args['num_iteration']\\n self.gru_flag = args['gru_flag']\\n self.agg_operator = args['agg_operator']\\n\\n self.cnn = nn.Conv2d(in_channels + 1, in_channels, kernel_size=3,\\n stride=1, padding=1)\\n self.msg_cnn = nn.Conv2d(in_channels * 2, in_channels, kernel_size=3,\\n stride=1, padding=1)\\n self.conv_gru = ConvGRU(input_size=(H, W),\\n input_dim=in_channels * 2,\\n hidden_dim=[in_channels],\\n kernel_size=kernel_size,\\n num_layers=num_gru_layers,\\n batch_first=True,\\n bias=True,\\n return_all_layers=False)\\n self.mlp = nn.Linear(in_channels, in_channels)\\n\\n def regroup(self, x, record_len):\\n cum_sum_len = torch.cumsum(record_len, dim=0)\\n split_x = torch.tensor_split(x, cum_sum_len[:-1].cpu())\\n return split_x\\n\\n def forward(self, x, record_len, pairwise_t_matrix, prior_encoding):\\n # x: (B,C,H,W)\\n # record_len: (B)\\n # pairwise_t_matrix: (B,L,L,4,4)\\n # prior_encoding: (B,3)\\n _, C, H, W = x.shape\\n B, L = pairwise_t_matrix.shape[:2]\\n\\n if self.use_temporal_encoding:\\n # (B,1,1,1)\\n dt = prior_encoding[:, 1].to(torch.int).unsqueeze(1).unsqueeze(\\n 2).unsqueeze(3)\\n x = torch.cat([x, dt.repeat(1, 1, H, W)], dim=1)\\n x = self.cnn(x)\\n\\n # split x:[(L1, C, H, W), (L2, C, H, W)]\\n split_x = self.regroup(x, record_len)\\n # (B,L,L,2,3)\\n pairwise_t_matrix = get_discretized_transformation_matrix(\\n pairwise_t_matrix.reshape(-1, L, 4, 4), self.discrete_ratio,\\n self.downsample_rate).reshape(B, L, L, 2, 3)\\n # (B*L,L,1,H,W)\\n roi_mask = get_rotated_roi((B * L, L, 1, H, W),\\n pairwise_t_matrix.reshape(B * L * L, 2, 3))\\n roi_mask = roi_mask.reshape(B, L, L, 1, H, W)\\n\\n batch_node_features = split_x\\n # iteratively update the features for num_iteration times\\n for l in range(self.num_iteration):\\n\\n batch_updated_node_features = []\\n # iterate each batch\\n for b in range(B):\\n\\n # number of valid agent\\n N = record_len[b]\\n # (N,N,4,4)\\n # t_matrix[i, j]-> from i to j\\n t_matrix = pairwise_t_matrix[b][:N, :N, :, :]\\n updated_node_features = []\\n # update each node i\\n for i in range(N):\\n # (N,1,H,W)\\n mask = roi_mask[b, :N, i, ...]\\n\\n current_t_matrix = t_matrix[:, i, :, :]\\n current_t_matrix = get_transformation_matrix(\\n current_t_matrix, (H, W))\\n\\n # (N,C,H,W)\\n neighbor_feature = warp_affine(batch_node_features[b],\\n current_t_matrix,\\n (H, W))\\n # (N,C,H,W)\\n ego_agent_feature = batch_node_features[b][i].unsqueeze(\\n 0).repeat(N, 1, 1, 1)\\n #(N,2C,H,W)\\n neighbor_feature = torch.cat(\\n [neighbor_feature, ego_agent_feature], dim=1)\\n # (N,C,H,W)\\n message = self.msg_cnn(neighbor_feature) * mask\\n\\n # (C,H,W)\\n if self.agg_operator==\\\"avg\\\":\\n agg_feature = torch.mean(message, dim=0)\\n elif self.agg_operator==\\\"max\\\":\\n agg_feature = torch.max(message, dim=0)[0]\\n else:\\n raise ValueError(\\\"agg_operator has wrong value\\\")\\n # (2C, H, W)\\n cat_feature = torch.cat(\\n [batch_node_features[b][i, ...], agg_feature], dim=0)\\n # (C,H,W)\\n if self.gru_flag:\\n gru_out = \\\\\\n self.conv_gru(cat_feature.unsqueeze(0).unsqueeze(0))[\\n 0][\\n 0].squeeze(0).squeeze(0)\\n else:\\n gru_out = batch_node_features[b][i, ...] + agg_feature\\n updated_node_features.append(gru_out.unsqueeze(0))\\n # (N,C,H,W)\\n batch_updated_node_features.append(\\n torch.cat(updated_node_features, dim=0))\\n batch_node_features = batch_updated_node_features\\n # (B,C,H,W)\\n out = torch.cat(\\n [itm[0, ...].unsqueeze(0) for itm in batch_node_features], dim=0)\\n # (B,C,H,W)\\n out = self.mlp(out.permute(0, 2, 3, 1)).permute(0, 3, 1, 2)\\n\\n return out\\n\\n\\n\\\"\\\"\\\"\\nClass used to downsample features by 3*3 conv\\n\\\"\\\"\\\"\\n\\nimport torch\\nimport torch.nn as nn\\n\\n\\nclass DoubleConv(nn.Module):\\n \\\"\\\"\\\"\\n Double convoltuion\\n Args:\\n in_channels: input channel num\\n out_channels: output channel num\\n \\\"\\\"\\\"\\n\\n def __init__(self, in_channels, out_channels, kernel_size,\\n stride, padding):\\n super().__init__()\\n self.double_conv = nn.Sequential(\\n nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size,\\n stride=stride, padding=padding),\\n nn.ReLU(inplace=True),\\n nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),\\n nn.ReLU(inplace=True)\\n )\\n\\n def forward(self, x):\\n return self.double_conv(x)\\n\\n\\nclass DownsampleConv(nn.Module):\\n def __init__(self, config):\\n super(DownsampleConv, self).__init__()\\n self.layers = nn.ModuleList([])\\n input_dim = config['input_dim']\\n\\n for (ksize, dim, stride, padding) in zip(config['kernal_size'],\\n config['dim'],\\n config['stride'],\\n config['padding']):\\n self.layers.append(DoubleConv(input_dim,\\n dim,\\n kernel_size=ksize,\\n stride=stride,\\n padding=padding))\\n input_dim = dim\\n\\n def forward(self, x):\\n for i in range(len(self.layers)):\\n x = self.layers[i](x)\\n return x\\n\\nimport torch\\nfrom torch import nn\\n\\nfrom einops import rearrange\\n\\n\\nclass HGTCavAttention(nn.Module):\\n def __init__(self, dim, heads, num_types=2,\\n num_relations=4, dim_head=64, dropout=0.1):\\n super().__init__()\\n inner_dim = heads * dim_head\\n\\n self.heads = heads\\n self.scale = dim_head ** -0.5\\n self.num_types = num_types\\n\\n self.attend = nn.Softmax(dim=-1)\\n self.drop_out = nn.Dropout(dropout)\\n self.k_linears = nn.ModuleList()\\n self.q_linears = nn.ModuleList()\\n self.v_linears = nn.ModuleList()\\n self.a_linears = nn.ModuleList()\\n self.norms = nn.ModuleList()\\n for t in range(num_types):\\n self.k_linears.append(nn.Linear(dim, inner_dim))\\n self.q_linears.append(nn.Linear(dim, inner_dim))\\n self.v_linears.append(nn.Linear(dim, inner_dim))\\n self.a_linears.append(nn.Linear(inner_dim, dim))\\n\\n self.relation_att = nn.Parameter(\\n torch.Tensor(num_relations, heads, dim_head, dim_head))\\n self.relation_msg = nn.Parameter(\\n torch.Tensor(num_relations, heads, dim_head, dim_head))\\n\\n torch.nn.init.xavier_uniform(self.relation_att)\\n torch.nn.init.xavier_uniform(self.relation_msg)\\n\\n def to_qkv(self, x, types):\\n # x: (B,H,W,L,C)\\n # types: (B,L)\\n q_batch = []\\n k_batch = []\\n v_batch = []\\n\\n for b in range(x.shape[0]):\\n q_list = []\\n k_list = []\\n v_list = []\\n\\n for i in range(x.shape[-2]):\\n # (H,W,1,C)\\n q_list.append(\\n self.q_linears[types[b, i]](x[b, :, :, i, :].unsqueeze(2)))\\n k_list.append(\\n self.k_linears[types[b, i]](x[b, :, :, i, :].unsqueeze(2)))\\n v_list.append(\\n self.v_linears[types[b, i]](x[b, :, :, i, :].unsqueeze(2)))\\n # (1,H,W,L,C)\\n q_batch.append(torch.cat(q_list, dim=2).unsqueeze(0))\\n k_batch.append(torch.cat(k_list, dim=2).unsqueeze(0))\\n v_batch.append(torch.cat(v_list, dim=2).unsqueeze(0))\\n # (B,H,W,L,C)\\n q = torch.cat(q_batch, dim=0)\\n k = torch.cat(k_batch, dim=0)\\n v = torch.cat(v_batch, dim=0)\\n return q, k, v\\n\\n def get_relation_type_index(self, type1, type2):\\n return type1 * self.num_types + type2\\n\\n def get_hetero_edge_weights(self, x, types):\\n w_att_batch = []\\n w_msg_batch = []\\n\\n for b in range(x.shape[0]):\\n w_att_list = []\\n w_msg_list = []\\n\\n for i in range(x.shape[-2]):\\n w_att_i_list = []\\n w_msg_i_list = []\\n\\n for j in range(x.shape[-2]):\\n e_type = self.get_relation_type_index(types[b, i],\\n types[b, j])\\n w_att_i_list.append(self.relation_att[e_type].unsqueeze(0))\\n w_msg_i_list.append(self.relation_msg[e_type].unsqueeze(0))\\n w_att_list.append(torch.cat(w_att_i_list, dim=0).unsqueeze(0))\\n w_msg_list.append(torch.cat(w_msg_i_list, dim=0).unsqueeze(0))\\n\\n w_att_batch.append(torch.cat(w_att_list, dim=0).unsqueeze(0))\\n w_msg_batch.append(torch.cat(w_msg_list, dim=0).unsqueeze(0))\\n\\n # (B,M,L,L,C_head,C_head)\\n w_att = torch.cat(w_att_batch, dim=0).permute(0, 3, 1, 2, 4, 5)\\n w_msg = torch.cat(w_msg_batch, dim=0).permute(0, 3, 1, 2, 4, 5)\\n return w_att, w_msg\\n\\n def to_out(self, x, types):\\n out_batch = []\\n for b in range(x.shape[0]):\\n out_list = []\\n for i in range(x.shape[-2]):\\n out_list.append(\\n self.a_linears[types[b, i]](x[b, :, :, i, :].unsqueeze(2)))\\n out_batch.append(torch.cat(out_list, dim=2).unsqueeze(0))\\n out = torch.cat(out_batch, dim=0)\\n return out\\n\\n def forward(self, x, mask, prior_encoding):\\n # x: (B, L, H, W, C) -> (B, H, W, L, C)\\n # mask: (B, H, W, L, 1)\\n # prior_encoding: (B,L,H,W,3)\\n x = x.permute(0, 2, 3, 1, 4)\\n # mask: (B, 1, H, W, L, 1)\\n mask = mask.unsqueeze(1)\\n # (B,L)\\n velocities, dts, types = [itm.squeeze(-1) for itm in\\n prior_encoding[:, :, 0, 0, :].split(\\n [1, 1, 1], dim=-1)]\\n types = types.to(torch.int)\\n dts = dts.to(torch.int)\\n qkv = self.to_qkv(x, types)\\n # (B,M,L,L,C_head,C_head)\\n w_att, w_msg = self.get_hetero_edge_weights(x, types)\\n\\n # q: (B, M, H, W, L, C)\\n q, k, v = map(lambda t: rearrange(t, 'b h w l (m c) -> b m h w l c',\\n m=self.heads), (qkv))\\n # attention, (B, M, H, W, L, L)\\n att_map = torch.einsum(\\n 'b m h w i p, b m i j p q, bm h w j q -> b m h w i j',\\n [q, w_att, k]) * self.scale\\n # add mask\\n att_map = att_map.masked_fill(mask == 0, -float('inf'))\\n # softmax\\n att_map = self.attend(att_map)\\n\\n # out:(B, M, H, W, L, C_head)\\n v_msg = torch.einsum('b m i j p c, b m h w j p -> b m h w i j c',\\n w_msg, v)\\n out = torch.einsum('b m h w i j, b m h w i j c -> b m h w i c',\\n att_map, v_msg)\\n\\n out = rearrange(out, 'b m h w l c -> b h w l (m c)',\\n m=self.heads)\\n out = self.to_out(out, types)\\n out = self.drop_out(out)\\n # (B L H W C)\\n out = out.permute(0, 3, 1, 2, 4)\\n return out\\n\\n\\n\\n\\\"\\\"\\\"\\nMulti-scale window transformer\\n\\\"\\\"\\\"\\nimport torch\\nimport torch.nn as nn\\nimport numpy as np\\n\\nfrom einops import rearrange\\nfrom v2xvit.models.sub_modules.split_attn import SplitAttn\\n\\n\\ndef get_relative_distances(window_size):\\n indices = torch.tensor(np.array(\\n [[x, y] for x in range(window_size) for y in range(window_size)]))\\n distances = indices[None, :, :] - indices[:, None, :]\\n return distances\\n\\n\\nclass BaseWindowAttention(nn.Module):\\n def __init__(self, dim, heads, dim_head, drop_out, window_size,\\n relative_pos_embedding):\\n super().__init__()\\n inner_dim = dim_head * heads\\n\\n self.heads = heads\\n self.scale = dim_head ** -0.5\\n self.window_size = window_size\\n self.relative_pos_embedding = relative_pos_embedding\\n\\n self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)\\n\\n if self.relative_pos_embedding:\\n self.relative_indices = get_relative_distances(window_size) + \\\\\\n window_size - 1\\n self.pos_embedding = nn.Parameter(torch.randn(2 * window_size - 1,\\n 2 * window_size - 1))\\n else:\\n self.pos_embedding = nn.Parameter(torch.randn(window_size ** 2,\\n window_size ** 2))\\n\\n self.to_out = nn.Sequential(\\n nn.Linear(inner_dim, dim),\\n nn.Dropout(drop_out)\\n )\\n\\n def forward(self, x):\\n b, l, h, w, c, m = *x.shape, self.heads\\n\\n qkv = self.to_qkv(x).chunk(3, dim=-1)\\n new_h = h // self.window_size\\n new_w = w // self.window_size\\n\\n # q : (b, l, m, new_h*new_w, window_size^2, c_head)\\n q, k, v = map(\\n lambda t: rearrange(t,\\n 'b l (new_h w_h) (new_w w_w) (m c) -> b l m (new_h new_w) (w_h w_w) c',\\n m=m, w_h=self.window_size,\\n w_w=self.window_size), qkv)\\n # b l m h window_size window_size\\n dots = torch.einsum('b l m h i c, b l m h j c -> b l m h i j',\\n q, k, ) * self.scale\\n # consider prior knowledge of the local window\\n if self.relative_pos_embedding:\\n dots += self.pos_embedding[self.relative_indices[:, :, 0],\\n self.relative_indices[:, :, 1]]\\n else:\\n dots += self.pos_embedding\\n\\n attn = dots.softmax(dim=-1)\\n\\n out = torch.einsum('b l m h i j, b l m h j c -> b l m h i c', attn, v)\\n # b l h w c\\n out = rearrange(out,\\n 'b l m (new_h new_w) (w_h w_w) c -> b l (new_h w_h) (new_w w_w) (m c)',\\n m=self.heads, w_h=self.window_size,\\n w_w=self.window_size,\\n new_w=new_w, new_h=new_h)\\n out = self.to_out(out)\\n\\n return out\\n\\n\\nclass PyramidWindowAttention(nn.Module):\\n def __init__(self, dim, heads, dim_heads, drop_out, window_size,\\n relative_pos_embedding, fuse_method='naive'):\\n super().__init__()\\n\\n assert isinstance(window_size, list)\\n assert isinstance(heads, list)\\n assert isinstance(dim_heads, list)\\n assert len(dim_heads) == len(heads)\\n\\n self.pwmsa = nn.ModuleList([])\\n\\n for (head, dim_head, ws) in zip(heads, dim_heads, window_size):\\n self.pwmsa.append(BaseWindowAttention(dim,\\n head,\\n dim_head,\\n drop_out,\\n ws,\\n relative_pos_embedding))\\n self.fuse_mehod = fuse_method\\n if fuse_method == 'split_attn':\\n self.split_attn = SplitAttn(256)\\n\\n def forward(self, x):\\n output = None\\n # naive fusion will just sum up all window attention output and do a\\n # mean\\n if self.fuse_mehod == 'naive':\\n for wmsa in self.pwmsa:\\n output = wmsa(x) if output is None else output + wmsa(x)\\n return output / len(self.pwmsa)\\n\\n elif self.fuse_mehod == 'split_attn':\\n window_list = []\\n for wmsa in self.pwmsa:\\n window_list.append(wmsa(x))\\n return self.split_attn(window_list)\\n\\nimport os\\nfrom collections import OrderedDict\\n\\nimport numpy as np\\nimport torch\\n\\nfrom v2xvit.utils.common_utils import torch_tensor_to_numpy\\n\\n\\ndef inference_late_fusion(batch_data, model, dataset):\\n \\\"\\\"\\\"\\n Model inference for late fusion.\\n\\n Parameters\\n ----------\\n batch_data : dict\\n model : opencood.object\\n dataset : opencood.LateFusionDataset\\n\\n Returns\\n -------\\n pred_box_tensor : torch.Tensor\\n The tensor of prediction bounding box after NMS.\\n gt_box_tensor : torch.Tensor\\n The tensor of gt bounding box.\\n \\\"\\\"\\\"\\n output_dict = OrderedDict()\\n\\n for cav_id, cav_content in batch_data.items():\\n output_dict[cav_id] = model(cav_content)\\n\\n pred_box_tensor, pred_score, gt_box_tensor = \\\\\\n dataset.post_process(batch_data,\\n output_dict)\\n\\n return pred_box_tensor, pred_score, gt_box_tensor\\n\\n\\ndef inference_early_fusion(batch_data, model, dataset):\\n \\\"\\\"\\\"\\n Model inference for early fusion.\\n\\n Parameters\\n ----------\\n batch_data : dict\\n model : opencood.object\\n dataset : opencood.EarlyFusionDataset\\n\\n Returns\\n -------\\n pred_box_tensor : torch.Tensor\\n The tensor of prediction bounding box after NMS.\\n gt_box_tensor : torch.Tensor\\n The tensor of gt bounding box.\\n \\\"\\\"\\\"\\n output_dict = OrderedDict()\\n cav_content = batch_data['ego']\\n\\n output_dict['ego'] = model(cav_content)\\n\\n pred_box_tensor, pred_score, gt_box_tensor = \\\\\\n dataset.post_process(batch_data,\\n output_dict)\\n\\n return pred_box_tensor, pred_score, gt_box_tensor\\n\\n\\ndef inference_intermediate_fusion(batch_data, model, dataset):\\n \\\"\\\"\\\"\\n Model inference for early fusion.\\n\\n Parameters\\n ----------\\n batch_data : dict\\n model : opencood.object\\n dataset : opencood.EarlyFusionDataset\\n\\n Returns\\n -------\\n pred_box_tensor : torch.Tensor\\n The tensor of prediction bounding box after NMS.\\n gt_box_tensor : torch.Tensor\\n The tensor of gt bounding box.\\n \\\"\\\"\\\"\\n return inference_early_fusion(batch_data, model, dataset)\\n\\n\\ndef save_prediction_gt(pred_tensor, gt_tensor, pcd, timestamp, save_path):\\n \\\"\\\"\\\"\\n Save prediction and gt tensor to txt file.\\n \\\"\\\"\\\"\\n pred_np = torch_tensor_to_numpy(pred_tensor)\\n gt_np = torch_tensor_to_numpy(gt_tensor)\\n pcd_np = torch_tensor_to_numpy(pcd)\\n\\n np.save(os.path.join(save_path, '%04d_pcd.npy' % timestamp), pcd_np)\\n np.save(os.path.join(save_path, '%04d_pred.npy' % timestamp), pred_np)\\n np.save(os.path.join(save_path, '%04d_gt.npy' % timestamp), gt_np)\\n\\n\\nimport argparse\\nimport os\\nimport time\\n\\nimport torch\\nimport open3d as o3d\\nfrom torch.utils.data import DataLoader\\n\\nimport v2xvit.hypes_yaml.yaml_utils as yaml_utils\\nfrom v2xvit.tools import train_utils, infrence_utils\\nfrom v2xvit.data_utils.datasets import build_dataset\\nfrom v2xvit.visualization import vis_utils\\nfrom v2xvit.utils import eval_utils\\n\\n\\ndef test_parser():\\n parser = argparse.ArgumentParser(description=\\\"synthetic data generation\\\")\\n parser.add_argument('--model_dir', type=str, required=True,\\n help='Continued training path')\\n parser.add_argument('--fusion_method', required=True, type=str,\\n default='late',\\n help='late, early or intermediate')\\n parser.add_argument('--show_vis', action='store_true',\\n help='whether to show image visualization result')\\n parser.add_argument('--show_sequence', action='store_true',\\n help='whether to show video visualization result.'\\n 'it can note be set true with show_vis together ')\\n parser.add_argument('--save_vis', action='store_true',\\n help='whether to save visualization result')\\n parser.add_argument('--save_npy', action='store_true',\\n help='whether to save prediction and gt result'\\n 'in npy file')\\n opt = parser.parse_args()\\n return opt\\n\\n\\ndef main():\\n opt = test_parser()\\n assert opt.fusion_method in ['late', 'early', 'intermediate']\\n assert not (opt.show_vis and opt.show_sequence), \\\\\\n 'you can only visualize ' \\\\\\n 'the results in single ' \\\\\\n 'image mode or video mode'\\n\\n hypes = yaml_utils.load_yaml(None, opt)\\n\\n print('Dataset Building')\\n opencood_dataset = build_dataset(hypes, visualize=True, train=False)\\n data_loader = DataLoader(opencood_dataset,\\n batch_size=1,\\n num_workers=10,\\n collate_fn=opencood_dataset.collate_batch_test,\\n shuffle=False,\\n pin_memory=False,\\n drop_last=False)\\n\\n print('Creating Model')\\n model = train_utils.create_model(hypes)\\n # we assume gpu is necessary\\n if torch.cuda.is_available():\\n model.cuda()\\n device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\\n\\n print('Loading Model from checkpoint')\\n saved_path = opt.model_dir\\n _, model = train_utils.load_saved_model(saved_path, model)\\n model.eval()\\n\\n # Create the dictionary for evaluation\\n result_stat = {0.3: {'tp': [], 'fp': [], 'gt': 0},\\n 0.5: {'tp': [], 'fp': [], 'gt': 0},\\n 0.7: {'tp': [], 'fp': [], 'gt': 0}}\\n\\n if opt.show_sequence:\\n vis = o3d.visualization.Visualizer()\\n vis.create_window()\\n\\n vis.get_render_option().background_color = [0.05, 0.05, 0.05]\\n vis.get_render_option().point_size = 1.0\\n vis.get_render_option().show_coordinate_frame = True\\n\\n # used to visualize lidar points\\n vis_pcd = o3d.geometry.PointCloud()\\n # used to visualize object bounding box, maximum 50\\n vis_aabbs_gt = []\\n vis_aabbs_pred = []\\n for _ in range(50):\\n vis_aabbs_gt.append(o3d.geometry.LineSet())\\n vis_aabbs_pred.append(o3d.geometry.LineSet())\\n\\n for i, batch_data in enumerate(data_loader):\\n print(i)\\n with torch.no_grad():\\n torch.cuda.synchronize()\\n batch_data = train_utils.to_device(batch_data, device)\\n if opt.fusion_method == 'late':\\n pred_box_tensor, pred_score, gt_box_tensor = \\\\\\n infrence_utils.inference_late_fusion(batch_data,\\n model,\\n opencood_dataset)\\n elif opt.fusion_method == 'early':\\n pred_box_tensor, pred_score, gt_box_tensor = \\\\\\n infrence_utils.inference_early_fusion(batch_data,\\n model,\\n opencood_dataset)\\n elif opt.fusion_method == 'intermediate':\\n pred_box_tensor, pred_score, gt_box_tensor = \\\\\\n infrence_utils.inference_intermediate_fusion(batch_data,\\n model,\\n opencood_dataset)\\n else:\\n raise NotImplementedError('Only early, late and intermediate'\\n 'fusion is supported.')\\n eval_utils.caluclate_tp_fp(pred_box_tensor,\\n pred_score,\\n gt_box_tensor,\\n result_stat,\\n 0.3)\\n eval_utils.caluclate_tp_fp(pred_box_tensor,\\n pred_score,\\n gt_box_tensor,\\n result_stat,\\n 0.5)\\n eval_utils.caluclate_tp_fp(pred_box_tensor,\\n pred_score,\\n gt_box_tensor,\\n result_stat,\\n 0.7)\\n if opt.save_npy:\\n npy_save_path = os.path.join(opt.model_dir, 'npy')\\n if not os.path.exists(npy_save_path):\\n os.makedirs(npy_save_path)\\n infrence_utils.save_prediction_gt(pred_box_tensor,\\n gt_box_tensor,\\n batch_data['ego'][\\n 'origin_lidar'][0],\\n i,\\n npy_save_path)\\n\\n if opt.show_vis or opt.save_vis:\\n vis_save_path = ''\\n if opt.save_vis:\\n vis_save_path = os.path.join(opt.model_dir, 'vis')\\n if not os.path.exists(vis_save_path):\\n os.makedirs(vis_save_path)\\n vis_save_path = os.path.join(vis_save_path, '%05d.png' % i)\\n\\n opencood_dataset.visualize_result(pred_box_tensor,\\n gt_box_tensor,\\n batch_data['ego'][\\n 'origin_lidar'][0],\\n opt.show_vis,\\n vis_save_path,\\n dataset=opencood_dataset)\\n\\n if opt.show_sequence:\\n pcd, pred_o3d_box, gt_o3d_box = \\\\\\n vis_utils.visualize_inference_sample_dataloader(\\n pred_box_tensor,\\n gt_box_tensor,\\n batch_data['ego']['origin_lidar'][0],\\n vis_pcd,\\n mode='constant'\\n )\\n if i == 0:\\n vis.add_geometry(pcd)\\n vis_utils.linset_assign_list(vis,\\n vis_aabbs_pred,\\n pred_o3d_box,\\n update_mode='add')\\n\\n vis_utils.linset_assign_list(vis,\\n vis_aabbs_gt,\\n gt_o3d_box,\\n update_mode='add')\\n\\n vis_utils.linset_assign_list(vis,\\n vis_aabbs_pred,\\n pred_o3d_box)\\n vis_utils.linset_assign_list(vis,\\n vis_aabbs_gt,\\n gt_o3d_box)\\n vis.update_geometry(pcd)\\n vis.poll_events()\\n vis.update_renderer()\\n time.sleep(0.001)\\n\\n eval_utils.eval_final_results(result_stat,\\n opt.model_dir)\\n if opt.show_sequence:\\n vis.destroy_window()\\n\\n\\nif __name__ == '__main__':\\n main()\\n\\n\\nimport glob\\nimport importlib\\nimport yaml\\nimport os\\nimport re\\nfrom datetime import datetime\\n\\nimport torch\\nimport torch.optim as optim\\n\\n\\ndef load_saved_model(saved_path, model):\\n \\\"\\\"\\\"\\n Load saved model if exiseted\\n\\n Parameters\\n __________\\n saved_path : str\\n model saved path\\n model : opencood object\\n The model instance.\\n\\n Returns\\n -------\\n model : opencood object\\n The model instance loaded pretrained params.\\n \\\"\\\"\\\"\\n assert os.path.exists(saved_path), '{} not found'.format(saved_path)\\n\\n def findLastCheckpoint(save_dir):\\n file_list = glob.glob(os.path.join(save_dir, '*epoch*.pth'))\\n if file_list:\\n epochs_exist = []\\n for file_ in file_list:\\n result = re.findall(\\\".*epoch(.*).pth.*\\\", file_)\\n epochs_exist.append(int(result[0]))\\n initial_epoch_ = max(epochs_exist)\\n else:\\n initial_epoch_ = 0\\n return initial_epoch_\\n\\n initial_epoch = findLastCheckpoint(saved_path)\\n if initial_epoch > 0:\\n print('resuming by loading epoch %d' % initial_epoch)\\n model.load_state_dict(torch.load(\\n os.path.join(saved_path,\\n 'net_epoch%d.pth' % initial_epoch)), strict=False)\\n\\n return initial_epoch, model\\n\\n\\ndef setup_train(hypes):\\n \\\"\\\"\\\"\\n Create folder for saved model based on current timestep and model name\\n\\n Parameters\\n ----------\\n hypes: dict\\n Config yaml dictionary for training:\\n \\\"\\\"\\\"\\n model_name = hypes['name']\\n current_time = datetime.now()\\n\\n folder_name = current_time.strftime(\\\"_%Y_%m_%d_%H_%M_%S\\\")\\n folder_name = model_name + folder_name\\n\\n current_path = os.path.dirname(__file__)\\n current_path = os.path.join(current_path, '../logs')\\n\\n full_path = os.path.join(current_path, folder_name)\\n\\n if not os.path.exists(full_path):\\n os.makedirs(full_path)\\n # save the yaml file\\n save_name = os.path.join(full_path, 'config.yaml')\\n with open(save_name, 'w') as outfile:\\n yaml.dump(hypes, outfile)\\n\\n return full_path\\n\\n\\ndef create_model(hypes):\\n \\\"\\\"\\\"\\n Import the module \\\"models/[model_name].py\\n\\n Parameters\\n __________\\n hypes : dict\\n Dictionary containing parameters.\\n\\n Returns\\n -------\\n model : opencood,object\\n Model object.\\n \\\"\\\"\\\"\\n backbone_name = hypes['model']['core_method']\\n backbone_config = hypes['model']['args']\\n\\n model_filename = \\\"v2xvit.models.\\\" + backbone_name\\n model_lib = importlib.import_module(model_filename)\\n model = None\\n target_model_name = backbone_name.replace('_', '')\\n\\n for name, cls in model_lib.__dict__.items():\\n if name.lower() == target_model_name.lower():\\n model = cls\\n\\n if model is None:\\n print('backbone not found in models folder. Please make sure you '\\n 'have a python file named %s and has a class '\\n 'called %s ignoring upper/lower case' % (model_filename,\\n target_model_name))\\n exit(0)\\n instance = model(backbone_config)\\n return instance\\n\\n\\ndef create_loss(hypes):\\n \\\"\\\"\\\"\\n Create the loss function based on the given loss name.\\n\\n Parameters\\n ----------\\n hypes : dict\\n Configuration params for training.\\n Returns\\n -------\\n criterion : opencood.object\\n The loss function.\\n \\\"\\\"\\\"\\n loss_func_name = hypes['loss']['core_method']\\n loss_func_config = hypes['loss']['args']\\n\\n loss_filename = \\\"v2xvit.loss.\\\" + loss_func_name\\n loss_lib = importlib.import_module(loss_filename)\\n loss_func = None\\n target_loss_name = loss_func_name.replace('_', '')\\n\\n for name, lfunc in loss_lib.__dict__.items():\\n if name.lower() == target_loss_name.lower():\\n loss_func = lfunc\\n\\n if loss_func is None:\\n print('loss function not found in loss folder. Please make sure you '\\n 'have a python file named %s and has a class '\\n 'called %s ignoring upper/lower case' % (loss_filename,\\n target_loss_name))\\n exit(0)\\n\\n criterion = loss_func(loss_func_config)\\n return criterion\\n\\n\\ndef setup_optimizer(hypes, model):\\n \\\"\\\"\\\"\\n Create optimizer corresponding to the yaml file\\n\\n Parameters\\n ----------\\n hypes : dict\\n The training configurations.\\n model : opencood model\\n The pytorch model\\n \\\"\\\"\\\"\\n method_dict = hypes['optimizer']\\n optimizer_method = getattr(optim, method_dict['core_method'], None)\\n if not optimizer_method:\\n raise ValueError('{} is not supported'.format(method_dict['name']))\\n if 'args' in method_dict:\\n return optimizer_method(filter(lambda p: p.requires_grad,\\n model.parameters()),\\n lr=method_dict['lr'],\\n **method_dict['args'])\\n else:\\n return optimizer_method(filter(lambda p: p.requires_grad,\\n model.parameters()),\\n lr=method_dict['lr'])\\n\\n\\ndef setup_lr_schedular(hypes, optimizer):\\n \\\"\\\"\\\"\\n Set up the learning rate schedular.\\n\\n Parameters\\n ----------\\n hypes : dict\\n The training configurations.\\n\\n optimizer : torch.optimizer\\n \\\"\\\"\\\"\\n lr_schedule_config = hypes['lr_scheduler']\\n\\n if lr_schedule_config['core_method'] == 'step':\\n from torch.optim.lr_scheduler import StepLR\\n step_size = lr_schedule_config['step_size']\\n gamma = lr_schedule_config['gamma']\\n scheduler = StepLR(optimizer, step_size=step_size, gamma=gamma)\\n\\n elif lr_schedule_config['core_method'] == 'multistep':\\n from torch.optim.lr_scheduler import MultiStepLR\\n milestones = lr_schedule_config['step_size']\\n gamma = lr_schedule_config['gamma']\\n scheduler = MultiStepLR(optimizer,\\n milestones=milestones,\\n gamma=gamma)\\n\\n else:\\n from torch.optim.lr_scheduler import ExponentialLR\\n gamma = lr_schedule_config['gamma']\\n scheduler = ExponentialLR(optimizer, gamma)\\n\\n return scheduler\\n\\n\\ndef to_device(inputs, device):\\n if isinstance(inputs, list):\\n return [to_device(x, device) for x in inputs]\\n elif isinstance(inputs, dict):\\n return {k: to_device(v, device) for k, v in inputs.items()}\\n else:\\n if isinstance(inputs, int) or isinstance(inputs, float) \\\\\\n or isinstance(inputs, str):\\n return inputs\\n return inputs.to(device)\\n\\n\\nimport argparse\\n\\nimport torch\\nfrom torch.utils.data import DataLoader\\n\\nimport v2xvit.hypes_yaml.yaml_utils as yaml_utils\\nfrom v2xvit.tools import train_utils\\nfrom v2xvit.data_utils.datasets import build_dataset\\nfrom v2xvit.visualization import vis_utils\\n\\n\\ndef test_parser():\\n parser = argparse.ArgumentParser(description=\\\"synthetic data generation\\\")\\n parser.add_argument('--model_dir', type=str, required=True,\\n help='Continued training path')\\n parser.add_argument('--fusion_method', type=str, default='late',\\n help='late, early or intermediate')\\n opt = parser.parse_args()\\n return opt\\n\\n\\ndef test_bev_post_processing():\\n opt = test_parser()\\n assert opt.fusion_method in ['late', 'early', 'intermediate']\\n\\n hypes = yaml_utils.load_yaml(None, opt)\\n\\n print('Dataset Building')\\n opencood_dataset = build_dataset(hypes, visualize=True, train=False)\\n data_loader = DataLoader(opencood_dataset,\\n batch_size=1,\\n num_workers=0,\\n collate_fn=opencood_dataset.collate_batch_test,\\n shuffle=False,\\n pin_memory=False,\\n drop_last=False)\\n\\n print('Creating Model')\\n model = train_utils.create_model(hypes)\\n # we assume gpu is necessary\\n if torch.cuda.is_available():\\n model.cuda()\\n device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\\n\\n print('Loading Model from checkpoint')\\n saved_path = opt.model_dir\\n _, model = train_utils.load_saved_model(saved_path, model)\\n model.eval()\\n for i, batch_data in enumerate(data_loader):\\n batch_data = train_utils.to_device(batch_data, device)\\n label_map = batch_data[\\\"ego\\\"][\\\"label_dict\\\"][\\\"label_map\\\"]\\n output_dict = {\\n \\\"cls\\\": label_map[:, 0, :, :],\\n \\\"reg\\\": label_map[:, 1:, :, :]\\n }\\n gt_box_tensor, _ = opencood_dataset.post_processor.post_process_debug(\\n batch_data[\\\"ego\\\"], output_dict)\\n vis_utils.visualize_single_sample_output_bev(gt_box_tensor,\\n batch_data['ego'][\\n 'origin_lidar'].squeeze(\\n 0),\\n opencood_dataset)\\n\\n\\nif __name__ == '__main__':\\n test_bev_post_processing()\\n\\n\\nimport argparse\\nimport os\\nimport statistics\\n\\nimport torch\\nimport tqdm\\nfrom torch.utils.data import DataLoader\\nfrom tensorboardX import SummaryWriter\\n\\nimport v2xvit.hypes_yaml.yaml_utils as yaml_utils\\nfrom v2xvit.tools import train_utils\\nfrom v2xvit.data_utils.datasets import build_dataset\\n\\n\\ndef train_parser():\\n parser = argparse.ArgumentParser(description=\\\"synthetic data generation\\\")\\n parser.add_argument(\\\"--hypes_yaml\\\", type=str, required=True,\\n help='data generation yaml file needed ')\\n parser.add_argument('--model_dir', default='',\\n help='Continued training path')\\n parser.add_argument(\\\"--half\\\", action='store_true', help=\\\"whether train with half precision\\\")\\n opt = parser.parse_args()\\n return opt\\n\\n\\ndef main():\\n opt = train_parser()\\n hypes = yaml_utils.load_yaml(opt.hypes_yaml, opt)\\n\\n print('Dataset Building')\\n opencood_train_dataset = build_dataset(hypes, visualize=False, train=True)\\n opencood_validate_dataset = build_dataset(hypes,\\n visualize=False,\\n train=False)\\n\\n train_loader = DataLoader(opencood_train_dataset,\\n batch_size=hypes['train_params']['batch_size'],\\n num_workers=8,\\n collate_fn=opencood_train_dataset.collate_batch_train,\\n shuffle=True,\\n pin_memory=False,\\n drop_last=True)\\n val_loader = DataLoader(opencood_validate_dataset,\\n batch_size=hypes['train_params']['batch_size'],\\n num_workers=8,\\n collate_fn=opencood_train_dataset.collate_batch_train,\\n shuffle=False,\\n pin_memory=False,\\n drop_last=True)\\n\\n print('Creating Model')\\n model = train_utils.create_model(hypes)\\n device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\\n\\n # we assume gpu is necessary\\n if torch.cuda.is_available():\\n model.to(device)\\n\\n # define the loss\\n criterion = train_utils.create_loss(hypes)\\n\\n # optimizer setup\\n optimizer = train_utils.setup_optimizer(hypes, model)\\n # lr scheduler setup\\n scheduler = train_utils.setup_lr_schedular(hypes, optimizer)\\n\\n # if we want to train from last checkpoint.\\n if opt.model_dir:\\n saved_path = opt.model_dir\\n init_epoch, model = train_utils.load_saved_model(saved_path, model)\\n\\n else:\\n init_epoch = 0\\n # if we train the model from scratch, we need to create a folder\\n # to save the model,\\n saved_path = train_utils.setup_train(hypes)\\n\\n # record training\\n writer = SummaryWriter(saved_path)\\n\\n # half precision training\\n if opt.half:\\n scaler = torch.cuda.amp.GradScaler()\\n\\n print('Training start')\\n epoches = hypes['train_params']['epoches']\\n # used to help schedule learning rate\\n for epoch in range(init_epoch, max(epoches, init_epoch)):\\n scheduler.step(epoch)\\n for param_group in optimizer.param_groups:\\n print('learning rate %f' % param_group[\\\"lr\\\"])\\n pbar2 = tqdm.tqdm(total=len(train_loader), leave=True)\\n for i, batch_data in enumerate(train_loader):\\n # the model will be evaluation mode during validation\\n model.train()\\n model.zero_grad()\\n optimizer.zero_grad()\\n\\n batch_data = train_utils.to_device(batch_data, device)\\n\\n # case1 : late fusion train --> only ego needed\\n # case2 : early fusion train --> all data projected to ego\\n # case3 : intermediate fusion --> ['ego']['processed_lidar']\\n # becomes a list, which containing all data from other cavs\\n # as well\\n if not opt.half:\\n ouput_dict = model(batch_data['ego'])\\n # first argument is always your output dictionary,\\n # second argument is always your label dictionary.\\n final_loss = criterion(ouput_dict, batch_data['ego']['label_dict'])\\n else:\\n with torch.cuda.amp.autocast():\\n ouput_dict = model(batch_data['ego'])\\n final_loss = criterion(ouput_dict, batch_data['ego']['label_dict'])\\n\\n criterion.logging(epoch, i, len(train_loader), writer, pbar=pbar2)\\n pbar2.update(1)\\n # back-propagation\\n if not opt.half:\\n final_loss.backward()\\n optimizer.step()\\n else:\\n scaler.scale(final_loss).backward()\\n scaler.step(optimizer)\\n scaler.update()\\n if epoch % hypes['train_params']['eval_freq'] == 0:\\n valid_ave_loss = []\\n\\n with torch.no_grad():\\n for i, batch_data in enumerate(val_loader):\\n model.eval()\\n\\n batch_data = train_utils.to_device(batch_data, device)\\n ouput_dict = model(batch_data['ego'])\\n\\n final_loss = criterion(ouput_dict,\\n batch_data['ego']['label_dict'])\\n valid_ave_loss.append(final_loss.item())\\n valid_ave_loss = statistics.mean(valid_ave_loss)\\n print('At epoch %d, the validation loss is %f' % (epoch,\\n valid_ave_loss))\\n\\n writer.add_scalar('Validate_Loss', valid_ave_loss, epoch)\\n\\n if epoch % hypes['train_params']['save_freq'] == 0:\\n torch.save(model.state_dict(),\\n os.path.join(saved_path,\\n 'net_epoch%d.pth' % (epoch + 1)))\\n\\n print('Training Finished, checkpoints saved to %s' % saved_path)\\n\\n\\nif __name__ == '__main__':\\n main()\\n\\n\\n\\n\\nimport torch\\nimport torch.nn as nn\\nimport torch.nn.functional as F\\n\\n\\nclass VoxelNetLoss(nn.Module):\\n def __init__(self, args):\\n super(VoxelNetLoss, self).__init__()\\n self.smoothl1loss = nn.SmoothL1Loss(size_average=False)\\n self.alpha = args['alpha']\\n self.beta = args['beta']\\n self.reg_coe = args['reg']\\n self.loss_dict = {}\\n\\n def forward(self, output_dict, target_dict):\\n \\\"\\\"\\\"\\n Parameters\\n ----------\\n output_dict : dict\\n target_dict : dict\\n \\\"\\\"\\\"\\n rm = output_dict['rm']\\n psm = output_dict['psm']\\n\\n pos_equal_one = target_dict['pos_equal_one']\\n neg_equal_one = target_dict['neg_equal_one']\\n targets = target_dict['targets']\\n\\n p_pos = F.sigmoid(psm.permute(0, 2, 3, 1))\\n rm = rm.permute(0, 2, 3, 1).contiguous()\\n rm = rm.view(rm.size(0), rm.size(1), rm.size(2), -1, 7)\\n targets = targets.view(targets.size(0), targets.size(1),\\n targets.size(2), -1, 7)\\n pos_equal_one_for_reg = pos_equal_one.unsqueeze(\\n pos_equal_one.dim()).expand(-1, -1, -1, -1, 7)\\n\\n rm_pos = rm * pos_equal_one_for_reg\\n targets_pos = targets * pos_equal_one_for_reg\\n\\n cls_pos_loss = -pos_equal_one * torch.log(p_pos + 1e-6)\\n cls_pos_loss = cls_pos_loss.sum() / (pos_equal_one.sum() + 1e-6)\\n\\n cls_neg_loss = -neg_equal_one * torch.log(1 - p_pos + 1e-6)\\n cls_neg_loss = cls_neg_loss.sum() / (neg_equal_one.sum() + 1e-6)\\n\\n reg_loss = self.smoothl1loss(rm_pos, targets_pos)\\n reg_loss = reg_loss / (pos_equal_one.sum() + 1e-6)\\n conf_loss = self.alpha * cls_pos_loss + self.beta * cls_neg_loss\\n\\n total_loss = self.reg_coe * reg_loss + conf_loss\\n\\n self.loss_dict.update({'total_loss': total_loss,\\n 'reg_loss': reg_loss,\\n 'conf_loss': conf_loss})\\n\\n return total_loss\\n\\n def logging(self, epoch, batch_id, batch_len, writer):\\n \\\"\\\"\\\"\\n Print out the loss function for current iteration.\\n\\n Parameters\\n ----------\\n epoch : int\\n Current epoch for training.\\n batch_id : int\\n The current batch.\\n batch_len : int\\n Total batch length in one iteration of training,\\n writer : SummaryWriter\\n Used to visualize on tensorboard\\n \\\"\\\"\\\"\\n total_loss = self.loss_dict['total_loss']\\n reg_loss = self.loss_dict['reg_loss']\\n conf_loss = self.loss_dict['conf_loss']\\n\\n print(\\\"[epoch %d][%d/%d], || Loss: %.4f || Conf Loss: %.4f\\\"\\n \\\" || Loc Loss: %.4f\\\" % (\\n epoch, batch_id + 1, batch_len,\\n total_loss.item(), conf_loss.item(), reg_loss.item()))\\n\\n writer.add_scalar('Regression_loss', reg_loss.item(),\\n epoch*batch_len + batch_id)\\n writer.add_scalar('Confidence_loss', conf_loss.item(),\\n epoch*batch_len + batch_id)\\n\\n\\nfrom functools import reduce\\n\\nimport torch\\nimport torch.nn as nn\\nimport torch.nn.functional as F\\n\\n\\nclass PixorLoss(nn.Module):\\n def __init__(self, args):\\n super(PixorLoss, self).__init__()\\n self.alpha = args[\\\"alpha\\\"]\\n self.beta = args[\\\"beta\\\"]\\n self.loss_dict = {}\\n\\n def forward(self, output_dict, target_dict):\\n \\\"\\\"\\\"\\n Compute loss for pixor network\\n Parameters\\n ----------\\n output_dict : dict\\n The dictionary that contains the output.\\n\\n target_dict : dict\\n The dictionary that contains the target.\\n\\n Returns\\n -------\\n total_loss : torch.Tensor\\n Total loss.\\n\\n \\\"\\\"\\\"\\n targets = target_dict[\\\"label_map\\\"]\\n cls_preds, loc_preds = output_dict[\\\"cls\\\"], output_dict[\\\"reg\\\"]\\n\\n cls_targets, loc_targets = targets.split([1, 6], dim=1)\\n pos_count = cls_targets.sum()\\n neg_count = (cls_targets == 0).sum()\\n w1, w2 = neg_count / (pos_count + neg_count), pos_count / (\\n pos_count + neg_count)\\n weights = torch.ones_like(cls_preds.reshape(-1))\\n weights[cls_targets.reshape(-1) == 1] = w1\\n weights[cls_targets.reshape(-1) == 0] = w2\\n # cls_targets = cls_targets.float()\\n # cls_loss = F.binary_cross_entropy_with_logits(input=cls_preds.reshape(-1), target=cls_targets.reshape(-1), weight=weights,\\n # reduction='mean')\\n cls_loss = F.binary_cross_entropy_with_logits(\\n input=cls_preds, target=cls_targets,\\n reduction='mean')\\n pos_pixels = cls_targets.sum()\\n\\n loc_loss = F.smooth_l1_loss(cls_targets * loc_preds,\\n cls_targets * loc_targets,\\n reduction='sum')\\n loc_loss = loc_loss / pos_pixels if pos_pixels > 0 else loc_loss\\n\\n total_loss = self.alpha * cls_loss + self.beta * loc_loss\\n\\n self.loss_dict.update({'total_loss': total_loss,\\n 'reg_loss': loc_loss,\\n 'cls_loss': cls_loss})\\n\\n return total_loss\\n\\n def logging(self, epoch, batch_id, batch_len, writer):\\n \\\"\\\"\\\"\\n Print out the loss function for current iteration.\\n\\n Parameters\\n ----------\\n epoch : int\\n Current epoch for training.\\n batch_id : int\\n The current batch.\\n batch_len : int\\n Total batch length in one iteration of training,\\n writer : SummaryWriter\\n Used to visualize on tensorboard\\n \\\"\\\"\\\"\\n total_loss = self.loss_dict['total_loss']\\n reg_loss = self.loss_dict['reg_loss']\\n cls_loss = self.loss_dict['cls_loss']\\n\\n print(\\\"[epoch %d][%d/%d], || Loss: %.4f || cls Loss: %.4f\\\"\\n \\\" || reg Loss: %.4f\\\" % (\\n epoch, batch_id + 1, batch_len,\\n total_loss.item(), cls_loss.item(), reg_loss.item()))\\n\\n writer.add_scalar('Regression_loss', reg_loss.item(),\\n epoch * batch_len + batch_id)\\n writer.add_scalar('Confidence_loss', cls_loss.item(),\\n epoch * batch_len + batch_id)\\n\\n\\ndef test():\\n torch.manual_seed(0)\\n loss = PixorLoss(None)\\n pred = torch.sigmoid(torch.randn(1, 7, 2, 3))\\n label = torch.zeros(1, 7, 2, 3)\\n loss = loss(pred, label)\\n print(loss)\\n\\n\\nif __name__ == \\\"__main__\\\":\\n test()\\n\\n\\nimport torch\\nimport torch.nn as nn\\nimport torch.nn.functional as F\\nimport numpy as np\\n\\n\\nclass WeightedSmoothL1Loss(nn.Module):\\n \\\"\\\"\\\"\\n Code-wise Weighted Smooth L1 Loss modified based on fvcore.nn.smooth_l1_loss\\n https://github.com/facebookresearch/fvcore/blob/master/fvcore/nn/smooth_l1_loss.py\\n | 0.5 * x ** 2 / beta if abs(x) < beta\\n smoothl1(x) = |\\n | abs(x) - 0.5 * beta otherwise,\\n where x = input - target.\\n \\\"\\\"\\\"\\n def __init__(self, beta: float = 1.0 / 9.0, code_weights: list = None):\\n \\\"\\\"\\\"\\n Args:\\n beta: Scalar float.\\n L1 to L2 change point.\\n For beta values < 1e-5, L1 loss is computed.\\n code_weights: (#codes) float list if not None.\\n Code-wise weights.\\n \\\"\\\"\\\"\\n super(WeightedSmoothL1Loss, self).__init__()\\n self.beta = beta\\n if code_weights is not None:\\n self.code_weights = np.array(code_weights, dtype=np.float32)\\n self.code_weights = torch.from_numpy(self.code_weights).cuda()\\n\\n @staticmethod\\n def smooth_l1_loss(diff, beta):\\n if beta < 1e-5:\\n loss = torch.abs(diff)\\n else:\\n n = torch.abs(diff)\\n loss = torch.where(n < beta, 0.5 * n ** 2 / beta, n - 0.5 * beta)\\n\\n return loss\\n\\n def forward(self, input: torch.Tensor,\\n target: torch.Tensor, weights: torch.Tensor = None):\\n \\\"\\\"\\\"\\n Args:\\n input: (B, #anchors, #codes) float tensor.\\n Ecoded predicted locations of objects.\\n target: (B, #anchors, #codes) float tensor.\\n Regression targets.\\n weights: (B, #anchors) float tensor if not None.\\n\\n Returns:\\n loss: (B, #anchors) float tensor.\\n Weighted smooth l1 loss without reduction.\\n \\\"\\\"\\\"\\n target = torch.where(torch.isnan(target), input, target) # ignore nan targets\\n\\n diff = input - target\\n loss = self.smooth_l1_loss(diff, self.beta)\\n\\n # anchor-wise weighting\\n if weights is not None:\\n assert weights.shape[0] == loss.shape[0] and weights.shape[1] == loss.shape[1]\\n loss = loss * weights.unsqueeze(-1)\\n\\n return loss\\n\\n\\nclass PointPillarLoss(nn.Module):\\n def __init__(self, args):\\n super(PointPillarLoss, self).__init__()\\n self.reg_loss_func = WeightedSmoothL1Loss()\\n self.alpha = 0.25\\n self.gamma = 2.0\\n\\n self.cls_weight = args['cls_weight']\\n self.reg_coe = args['reg']\\n self.loss_dict = {}\\n\\n def forward(self, output_dict, target_dict):\\n \\\"\\\"\\\"\\n Parameters\\n ----------\\n output_dict : dict\\n target_dict : dict\\n \\\"\\\"\\\"\\n rm = output_dict['rm']\\n psm = output_dict['psm']\\n targets = target_dict['targets']\\n\\n cls_preds = psm.permute(0, 2, 3, 1).contiguous()\\n\\n box_cls_labels = target_dict['pos_equal_one']\\n box_cls_labels = box_cls_labels.view(psm.shape[0], -1).contiguous()\\n\\n positives = box_cls_labels > 0\\n negatives = box_cls_labels == 0\\n negative_cls_weights = negatives * 1.0\\n cls_weights = (negative_cls_weights + 1.0 * positives).float()\\n reg_weights = positives.float()\\n\\n pos_normalizer = positives.sum(1, keepdim=True).float()\\n reg_weights /= torch.clamp(pos_normalizer, min=1.0)\\n cls_weights /= torch.clamp(pos_normalizer, min=1.0)\\n cls_targets = box_cls_labels\\n cls_targets = cls_targets.unsqueeze(dim=-1)\\n\\n cls_targets = cls_targets.squeeze(dim=-1)\\n one_hot_targets = torch.zeros(\\n *list(cls_targets.shape), 2,\\n dtype=cls_preds.dtype, device=cls_targets.device\\n )\\n one_hot_targets.scatter_(-1, cls_targets.unsqueeze(dim=-1).long(), 1.0)\\n cls_preds = cls_preds.view(psm.shape[0], -1, 1)\\n one_hot_targets = one_hot_targets[..., 1:]\\n\\n cls_loss_src = self.cls_loss_func(cls_preds,\\n one_hot_targets,\\n weights=cls_weights) # [N, M]\\n cls_loss = cls_loss_src.sum() / psm.shape[0]\\n conf_loss = cls_loss * self.cls_weight\\n\\n # regression\\n rm = rm.permute(0, 2, 3, 1).contiguous()\\n rm = rm.view(rm.size(0), -1, 7)\\n targets = targets.view(targets.size(0), -1, 7)\\n box_preds_sin, reg_targets_sin = self.add_sin_difference(rm,\\n targets)\\n loc_loss_src =\\\\\\n self.reg_loss_func(box_preds_sin,\\n reg_targets_sin,\\n weights=reg_weights)\\n reg_loss = loc_loss_src.sum() / rm.shape[0]\\n reg_loss *= self.reg_coe\\n\\n total_loss = reg_loss + conf_loss\\n\\n self.loss_dict.update({'total_loss': total_loss,\\n 'reg_loss': reg_loss,\\n 'conf_loss': conf_loss})\\n\\n return total_loss\\n\\n def cls_loss_func(self, input: torch.Tensor,\\n target: torch.Tensor,\\n weights: torch.Tensor):\\n \\\"\\\"\\\"\\n Args:\\n input: (B, #anchors, #classes) float tensor.\\n Predicted logits for each class\\n target: (B, #anchors, #classes) float tensor.\\n One-hot encoded classification targets\\n weights: (B, #anchors) float tensor.\\n Anchor-wise weights.\\n\\n Returns:\\n weighted_loss: (B, #anchors, #classes) float tensor after weighting.\\n \\\"\\\"\\\"\\n pred_sigmoid = torch.sigmoid(input)\\n alpha_weight = target * self.alpha + (1 - target) * (1 - self.alpha)\\n pt = target * (1.0 - pred_sigmoid) + (1.0 - target) * pred_sigmoid\\n focal_weight = alpha_weight * torch.pow(pt, self.gamma)\\n\\n bce_loss = self.sigmoid_cross_entropy_with_logits(input, target)\\n\\n loss = focal_weight * bce_loss\\n\\n if weights.shape.__len__() == 2 or \\\\\\n (weights.shape.__len__() == 1 and target.shape.__len__() == 2):\\n weights = weights.unsqueeze(-1)\\n\\n assert weights.shape.__len__() == loss.shape.__len__()\\n\\n return loss * weights\\n\\n @staticmethod\\n def sigmoid_cross_entropy_with_logits(input: torch.Tensor, target: torch.Tensor):\\n \\\"\\\"\\\" PyTorch Implementation for tf.nn.sigmoid_cross_entropy_with_logits:\\n max(x, 0) - x * z + log(1 + exp(-abs(x))) in\\n https://www.tensorflow.org/api_docs/python/tf/nn/sigmoid_cross_entropy_with_logits\\n\\n Args:\\n input: (B, #anchors, #classes) float tensor.\\n Predicted logits for each class\\n target: (B, #anchors, #classes) float tensor.\\n One-hot encoded classification targets\\n\\n Returns:\\n loss: (B, #anchors, #classes) float tensor.\\n Sigmoid cross entropy loss without reduction\\n \\\"\\\"\\\"\\n loss = torch.clamp(input, min=0) - input * target + \\\\\\n torch.log1p(torch.exp(-torch.abs(input)))\\n return loss\\n\\n @staticmethod\\n def add_sin_difference(boxes1, boxes2, dim=6):\\n assert dim != -1\\n rad_pred_encoding = torch.sin(boxes1[..., dim:dim + 1]) * \\\\\\n torch.cos(boxes2[..., dim:dim + 1])\\n rad_tg_encoding = torch.cos(boxes1[..., dim:dim + 1]) * \\\\\\n torch.sin(boxes2[..., dim:dim + 1])\\n\\n boxes1 = torch.cat([boxes1[..., :dim], rad_pred_encoding,\\n boxes1[..., dim + 1:]], dim=-1)\\n boxes2 = torch.cat([boxes2[..., :dim], rad_tg_encoding,\\n boxes2[..., dim + 1:]], dim=-1)\\n return boxes1, boxes2\\n\\n\\n def logging(self, epoch, batch_id, batch_len, writer, pbar=None):\\n \\\"\\\"\\\"\\n Print out the loss function for current iteration.\\n\\n Parameters\\n ----------\\n epoch : int\\n Current epoch for training.\\n batch_id : int\\n The current batch.\\n batch_len : int\\n Total batch length in one iteration of training,\\n writer : SummaryWriter\\n Used to visualize on tensorboard\\n \\\"\\\"\\\"\\n total_loss = self.loss_dict['total_loss']\\n reg_loss = self.loss_dict['reg_loss']\\n conf_loss = self.loss_dict['conf_loss']\\n if pbar is None:\\n print(\\\"[epoch %d][%d/%d], || Loss: %.4f || Conf Loss: %.4f\\\"\\n \\\" || Loc Loss: %.4f\\\" % (\\n epoch, batch_id + 1, batch_len,\\n total_loss.item(), conf_loss.item(), reg_loss.item()))\\n else:\\n pbar.set_description(\\\"[epoch %d][%d/%d], || Loss: %.4f || Conf Loss: %.4f\\\"\\n \\\" || Loc Loss: %.4f\\\" % (\\n epoch, batch_id + 1, batch_len,\\n total_loss.item(), conf_loss.item(), reg_loss.item()))\\n\\n\\n writer.add_scalar('Regression_loss', reg_loss.item(),\\n epoch*batch_len + batch_id)\\n writer.add_scalar('Confidence_loss', conf_loss.item(),\\n epoch*batch_len + batch_id)\\n\\n\\n\\n\\n\\\"\\\"\\\"\\nCommon utilities\\n\\\"\\\"\\\"\\n\\nimport numpy as np\\nimport torch\\nfrom shapely.geometry import Polygon\\n\\n\\ndef check_numpy_to_torch(x):\\n if isinstance(x, np.ndarray):\\n return torch.from_numpy(x).float(), True\\n return x, False\\n\\n\\ndef check_contain_nan(x):\\n if isinstance(x, dict):\\n return any(check_contain_nan(v) for k, v in x.items())\\n if isinstance(x, list):\\n return any(check_contain_nan(itm) for itm in x)\\n if isinstance(x, int) or isinstance(x, float):\\n return False\\n if isinstance(x, np.ndarray):\\n return np.any(np.isnan(x))\\n return torch.any(x.isnan()).detach().cpu().item()\\n\\n\\ndef rotate_points_along_z(points, angle):\\n \\\"\\\"\\\"\\n Args:\\n points: (B, N, 3 + C)\\n angle: (B), radians, angle along z-axis, angle increases x ==> y\\n Returns:\\n\\n \\\"\\\"\\\"\\n points, is_numpy = check_numpy_to_torch(points)\\n angle, _ = check_numpy_to_torch(angle)\\n\\n cosa = torch.cos(angle)\\n sina = torch.sin(angle)\\n zeros = angle.new_zeros(points.shape[0])\\n ones = angle.new_ones(points.shape[0])\\n rot_matrix = torch.stack((\\n cosa, sina, zeros,\\n -sina, cosa, zeros,\\n zeros, zeros, ones\\n ), dim=1).view(-1, 3, 3).float()\\n points_rot = torch.matmul(points[:, :, 0:3].float(), rot_matrix)\\n points_rot = torch.cat((points_rot, points[:, :, 3:]), dim=-1)\\n return points_rot.numpy() if is_numpy else points_rot\\n\\n\\ndef rotate_points_along_z_2d(points, angle):\\n \\\"\\\"\\\"\\n Rorate the points along z-axis.\\n Parameters\\n ----------\\n points : torch.Tensor / np.ndarray\\n (N, 2).\\n angle : torch.Tensor / np.ndarray\\n (N,)\\n\\n Returns\\n -------\\n points_rot : torch.Tensor / np.ndarray\\n Rorated points with shape (N, 2)\\n\\n \\\"\\\"\\\"\\n points, is_numpy = check_numpy_to_torch(points)\\n angle, _ = check_numpy_to_torch(angle)\\n cosa = torch.cos(angle)\\n sina = torch.sin(angle)\\n # (N, 2, 2)\\n rot_matrix = torch.stack((cosa, sina, -sina, cosa), dim=1).view(-1, 2,\\n 2).float()\\n points_rot = torch.einsum(\\\"ik, ikj->ij\\\", points.float(), rot_matrix)\\n return points_rot.numpy() if is_numpy else points_rot\\n\\n\\ndef remove_ego_from_objects(objects, ego_id):\\n \\\"\\\"\\\"\\n Avoid adding ego vehicle to the object dictionary.\\n\\n Parameters\\n ----------\\n objects : dict\\n The dictionary contained all objects.\\n\\n ego_id : int\\n Ego id.\\n \\\"\\\"\\\"\\n if ego_id in objects:\\n del objects[ego_id]\\n\\n\\ndef retrieve_ego_id(base_data_dict):\\n \\\"\\\"\\\"\\n Retrieve the ego vehicle id from sample(origin format).\\n\\n Parameters\\n ----------\\n base_data_dict : dict\\n Data sample in origin format.\\n\\n Returns\\n -------\\n ego_id : str\\n The id of ego vehicle.\\n \\\"\\\"\\\"\\n ego_id = None\\n\\n for cav_id, cav_content in base_data_dict.items():\\n if cav_content['ego']:\\n ego_id = cav_id\\n break\\n return ego_id\\n\\n\\ndef compute_iou(box, boxes):\\n \\\"\\\"\\\"\\n Compute iou between box and boxes list\\n Parameters\\n ----------\\n box : shapely.geometry.Polygon\\n Bounding box Polygon.\\n\\n boxes : list\\n List of shapely.geometry.Polygon.\\n\\n Returns\\n -------\\n iou : np.ndarray\\n Array of iou between box and boxes.\\n\\n \\\"\\\"\\\"\\n # Calculate intersection areas\\n iou = [box.intersection(b).area / box.union(b).area for b in boxes]\\n\\n return np.array(iou, dtype=np.float32)\\n\\n\\ndef convert_format(boxes_array):\\n \\\"\\\"\\\"\\n Convert boxes array to shapely.geometry.Polygon format.\\n Parameters\\n ----------\\n boxes_array : np.ndarray\\n (N, 4, 2) or (N, 8, 3).\\n\\n Returns\\n -------\\n list of converted shapely.geometry.Polygon object.\\n\\n \\\"\\\"\\\"\\n polygons = [Polygon([(box[i, 0], box[i, 1]) for i in range(4)]) for box in\\n boxes_array]\\n return np.array(polygons)\\n\\n\\ndef torch_tensor_to_numpy(torch_tensor):\\n \\\"\\\"\\\"\\n Convert a torch tensor to numpy.\\n\\n Parameters\\n ----------\\n torch_tensor : torch.Tensor\\n\\n Returns\\n -------\\n A numpy array.\\n \\\"\\\"\\\"\\n return torch_tensor.numpy() if not torch_tensor.is_cuda else \\\\\\n torch_tensor.cpu().detach().numpy()\\n\\n\\n\\\"\\\"\\\"\\nTransformation utils\\n\\\"\\\"\\\"\\n\\nimport numpy as np\\n\\n\\ndef x_to_world(pose):\\n \\\"\\\"\\\"\\n The transformation matrix from x-coordinate system to carla world system\\n\\n Parameters\\n ----------\\n pose : list\\n [x, y, z, roll, yaw, pitch]\\n\\n Returns\\n -------\\n matrix : np.ndarray\\n The transformation matrix.\\n \\\"\\\"\\\"\\n x, y, z, roll, yaw, pitch = pose[:]\\n\\n # used for rotation matrix\\n c_y = np.cos(np.radians(yaw))\\n s_y = np.sin(np.radians(yaw))\\n c_r = np.cos(np.radians(roll))\\n s_r = np.sin(np.radians(roll))\\n c_p = np.cos(np.radians(pitch))\\n s_p = np.sin(np.radians(pitch))\\n\\n matrix = np.identity(4)\\n # translation matrix\\n matrix[0, 3] = x\\n matrix[1, 3] = y\\n matrix[2, 3] = z\\n\\n # rotation matrix\\n matrix[0, 0] = c_p * c_y\\n matrix[0, 1] = c_y * s_p * s_r - s_y * c_r\\n matrix[0, 2] = -c_y * s_p * c_r - s_y * s_r\\n matrix[1, 0] = s_y * c_p\\n matrix[1, 1] = s_y * s_p * s_r + c_y * c_r\\n matrix[1, 2] = -s_y * s_p * c_r + c_y * s_r\\n matrix[2, 0] = s_p\\n matrix[2, 1] = -c_p * s_r\\n matrix[2, 2] = c_p * c_r\\n\\n return matrix\\n\\n\\ndef x1_to_x2(x1, x2):\\n \\\"\\\"\\\"\\n Transformation matrix from x1 to x2.\\n\\n Parameters\\n ----------\\n x1 : list\\n The pose of x1 under world coordinates.\\n x2 : list\\n The pose of x2 under world coordinates.\\n\\n Returns\\n -------\\n transformation_matrix : np.ndarray\\n The transformation matrix.\\n\\n \\\"\\\"\\\"\\n x1_to_world = x_to_world(x1)\\n x2_to_world = x_to_world(x2)\\n world_to_x2 = np.linalg.inv(x2_to_world)\\n\\n transformation_matrix = np.dot(world_to_x2, x1_to_world)\\n return transformation_matrix\\n\\n\\ndef dist_to_continuous(p_dist, displacement_dist, res, downsample_rate):\\n \\\"\\\"\\\"\\n Convert points discretized format to continuous space for BEV representation.\\n Parameters\\n ----------\\n p_dist : numpy.array\\n Points in discretized coorindates.\\n\\n displacement_dist : numpy.array\\n Discretized coordinates of bottom left origin.\\n\\n res : float\\n Discretization resolution.\\n\\n downsample_rate : int\\n Dowmsamping rate.\\n\\n Returns\\n -------\\n p_continuous : numpy.array\\n Points in continuous coorindates.\\n\\n \\\"\\\"\\\"\\n p_dist = np.copy(p_dist)\\n p_dist = p_dist + displacement_dist\\n p_continuous = p_dist * res * downsample_rate\\n return p_continuous\\n\\n\\n\\\"\\\"\\\"\\nBounding box related utility functions\\n\\\"\\\"\\\"\\nimport sys\\n\\nimport numpy as np\\n\\nimport torch\\nimport torch.nn.functional as F\\nimport v2xvit.utils.common_utils as common_utils\\nfrom v2xvit.utils.transformation_utils import x1_to_x2\\n\\n\\ndef corner_to_center(corner3d, order='lwh'):\\n \\\"\\\"\\\"\\n Convert 8 corners to x, y, z, dx, dy, dz, yaw.\\n\\n Parameters\\n ----------\\n corner3d : np.ndarray\\n (N, 8, 3)\\n\\n order : str\\n 'lwh' or 'hwl'\\n\\n Returns\\n -------\\n box3d : np.ndarray\\n (N, 7)\\n \\\"\\\"\\\"\\n assert corner3d.ndim == 3\\n batch_size = corner3d.shape[0]\\n\\n xyz = np.mean(corner3d[:, [0, 3, 5, 6], :], axis=1)\\n h = abs(np.mean(corner3d[:, 4:, 2] - corner3d[:, :4, 2], axis=1,\\n keepdims=True))\\n l = (np.sqrt(np.sum((corner3d[:, 0, [0, 1]] - corner3d[:, 3, [0, 1]]) ** 2,\\n axis=1, keepdims=True)) +\\n np.sqrt(np.sum((corner3d[:, 2, [0, 1]] - corner3d[:, 1, [0, 1]]) ** 2,\\n axis=1, keepdims=True)) +\\n np.sqrt(np.sum((corner3d[:, 4, [0, 1]] - corner3d[:, 7, [0, 1]]) ** 2,\\n axis=1, keepdims=True)) +\\n np.sqrt(np.sum((corner3d[:, 5, [0, 1]] - corner3d[:, 6, [0, 1]]) ** 2,\\n axis=1, keepdims=True))) / 4\\n\\n w = (np.sqrt(\\n np.sum((corner3d[:, 0, [0, 1]] - corner3d[:, 1, [0, 1]]) ** 2, axis=1,\\n keepdims=True)) +\\n np.sqrt(np.sum((corner3d[:, 2, [0, 1]] - corner3d[:, 3, [0, 1]]) ** 2,\\n axis=1, keepdims=True)) +\\n np.sqrt(np.sum((corner3d[:, 4, [0, 1]] - corner3d[:, 5, [0, 1]]) ** 2,\\n axis=1, keepdims=True)) +\\n np.sqrt(np.sum((corner3d[:, 6, [0, 1]] - corner3d[:, 7, [0, 1]]) ** 2,\\n axis=1, keepdims=True))) / 4\\n\\n theta = (np.arctan2(corner3d[:, 1, 1] - corner3d[:, 2, 1],\\n corner3d[:, 1, 0] - corner3d[:, 2, 0]) +\\n np.arctan2(corner3d[:, 0, 1] - corner3d[:, 3, 1],\\n corner3d[:, 0, 0] - corner3d[:, 3, 0]) +\\n np.arctan2(corner3d[:, 5, 1] - corner3d[:, 6, 1],\\n corner3d[:, 5, 0] - corner3d[:, 6, 0]) +\\n np.arctan2(corner3d[:, 4, 1] - corner3d[:, 7, 1],\\n corner3d[:, 4, 0] - corner3d[:, 7, 0]))[:,\\n np.newaxis] / 4\\n\\n if order == 'lwh':\\n return np.concatenate([xyz, l, w, h, theta], axis=1).reshape(\\n batch_size, 7)\\n elif order == 'hwl':\\n return np.concatenate([xyz, h, w, l, theta], axis=1).reshape(\\n batch_size, 7)\\n else:\\n sys.exit('Unknown order')\\n\\n\\ndef boxes_to_corners2d(boxes3d, order):\\n \\\"\\\"\\\"\\n 0 -------- 1\\n | |\\n | |\\n | |\\n 3 -------- 2\\n Parameters\\n __________\\n boxes3d: np.ndarray or torch.Tensor\\n (N, 7) [x, y, z, dx, dy, dz, heading], (x, y, z) is the box center.\\n\\n order : str\\n 'lwh' or 'hwl'\\n\\n Returns:\\n corners2d: np.ndarray or torch.Tensor\\n (N, 4, 3), the 4 corners of the bounding box.\\n\\n \\\"\\\"\\\"\\n corners3d = boxes_to_corners_3d(boxes3d, order)\\n corners2d = corners3d[:, :4, :]\\n return corners2d\\n\\n\\ndef boxes2d_to_corners2d(boxes2d, order=\\\"lwh\\\"):\\n \\\"\\\"\\\"\\n 0 -------- 1\\n | |\\n | |\\n | |\\n 3 -------- 2\\n Parameters\\n __________\\n boxes2d: np.ndarray or torch.Tensor\\n (..., 5) [x, y, dx, dy, heading], (x, y) is the box center.\\n\\n order : str\\n 'lwh' or 'hwl'\\n\\n Returns:\\n corners2d: np.ndarray or torch.Tensor\\n (..., 4, 2), the 4 corners of the bounding box.\\n\\n \\\"\\\"\\\"\\n assert order == \\\"lwh\\\", \\\"boxes2d_to_corners_2d only supports lwh order for now.\\\"\\n boxes2d, is_numpy = common_utils.check_numpy_to_torch(boxes2d)\\n template = boxes2d.new_tensor((\\n [1, -1], [1, 1], [-1, 1], [-1, -1]\\n )) / 2\\n input_shape = boxes2d.shape\\n boxes2d = boxes2d.view(-1, 5)\\n corners2d = boxes2d[:, None, 2:4].repeat(1, 4, 1) * template[None, :, :]\\n corners2d = common_utils.rotate_points_along_z_2d(corners2d.view(-1, 2),\\n boxes2d[:,\\n 4].repeat_interleave(\\n 4)).view(-1, 4,\\n 2)\\n corners2d += boxes2d[:, None, 0:2]\\n corners2d = corners2d.view(*(input_shape[:-1]), 4, 2)\\n return corners2d\\n\\n\\ndef boxes_to_corners_3d(boxes3d, order):\\n \\\"\\\"\\\"\\n 4 -------- 5\\n /| /|\\n 7 -------- 6 .\\n | | | |\\n . 0 -------- 1\\n |/ |/\\n 3 -------- 2\\n Parameters\\n __________\\n boxes3d: np.ndarray or torch.Tensor\\n (N, 7) [x, y, z, dx, dy, dz, heading], (x, y, z) is the box center.\\n\\n order : str\\n 'lwh' or 'hwl'\\n\\n Returns:\\n corners3d: np.ndarray or torch.Tensor\\n (N, 8, 3), the 8 corners of the bounding box.\\n\\n \\\"\\\"\\\"\\n # ^ z\\n # |\\n # |\\n # | . x\\n # |/\\n # +-------> y\\n\\n boxes3d, is_numpy = common_utils.check_numpy_to_torch(boxes3d)\\n\\n if order == 'hwl':\\n boxes3d[:, 3:6] = boxes3d[:, [5, 4, 3]]\\n\\n template = boxes3d.new_tensor((\\n [1, -1, -1], [1, 1, -1], [-1, 1, -1], [-1, -1, -1],\\n [1, -1, 1], [1, 1, 1], [-1, 1, 1], [-1, -1, 1],\\n )) / 2\\n\\n corners3d = boxes3d[:, None, 3:6].repeat(1, 8, 1) * template[None, :, :]\\n corners3d = common_utils.rotate_points_along_z(corners3d.view(-1, 8, 3),\\n boxes3d[:, 6]).view(-1, 8,\\n 3)\\n corners3d += boxes3d[:, None, 0:3]\\n\\n return corners3d.numpy() if is_numpy else corners3d\\n\\n\\ndef box3d_to_2d(box3d):\\n \\\"\\\"\\\"\\n Convert 3D bounding box to 2D.\\n\\n Parameters\\n ----------\\n box3d : np.ndarray\\n (n, 8, 3)\\n\\n Returns\\n -------\\n box2d : np.ndarray\\n (n, 4, 2), project 3d to 2d.\\n \\\"\\\"\\\"\\n box2d = box3d[:, :4, :2]\\n return box2d\\n\\n\\ndef corner2d_to_standup_box(box2d):\\n \\\"\\\"\\\"\\n Find the minmaxx, minmaxy for each 2d box. (N, 4, 2) -> (N, 4)\\n x1, y1, x2, y2\\n\\n Parameters\\n ----------\\n box2d : np.ndarray\\n (n, 4, 2), four corners of the 2d bounding box.\\n\\n Returns\\n -------\\n standup_box2d : np.ndarray\\n (n, 4)\\n \\\"\\\"\\\"\\n N = box2d.shape[0]\\n standup_boxes2d = np.zeros((N, 4))\\n\\n standup_boxes2d[:, 0] = np.min(box2d[:, :, 0], axis=1)\\n standup_boxes2d[:, 1] = np.min(box2d[:, :, 1], axis=1)\\n standup_boxes2d[:, 2] = np.max(box2d[:, :, 0], axis=1)\\n standup_boxes2d[:, 3] = np.max(box2d[:, :, 1], axis=1)\\n\\n return standup_boxes2d\\n\\n\\ndef corner_to_standup_box_torch(box_corner):\\n \\\"\\\"\\\"\\n Find the minmax x and y for each bounding box.\\n\\n Parameters\\n ----------\\n box_corner : torch.Tensor\\n Shape: (N, 8, 3) or (N, 4)\\n\\n Returns\\n -------\\n standup_box2d : torch.Tensor\\n (n, 4)\\n \\\"\\\"\\\"\\n N = box_corner.shape[0]\\n standup_boxes2d = torch.zeros((N, 4))\\n\\n standup_boxes2d = standup_boxes2d.to(box_corner.device)\\n\\n standup_boxes2d[:, 0] = torch.min(box_corner[:, :, 0], dim=1).values\\n standup_boxes2d[:, 1] = torch.min(box_corner[:, :, 1], dim=1).values\\n standup_boxes2d[:, 2] = torch.max(box_corner[:, :, 0], dim=1).values\\n standup_boxes2d[:, 3] = torch.max(box_corner[:, :, 1], dim=1).values\\n\\n return standup_boxes2d\\n\\n\\ndef project_box3d(box3d, transformation_matrix):\\n \\\"\\\"\\\"\\n Project the 3d bounding box to another coordinate system based on the\\n transfomration matrix.\\n\\n Parameters\\n ----------\\n box3d : torch.Tensor or np.ndarray\\n 3D bounding box, (N, 8, 3)\\n\\n transformation_matrix : torch.Tensor or np.ndarray\\n Transformation matrix, (4, 4)\\n\\n Returns\\n -------\\n projected_box3d : torch.Tensor\\n The projected bounding box, (N, 8, 3)\\n \\\"\\\"\\\"\\n assert transformation_matrix.shape == (4, 4)\\n box3d, is_numpy = \\\\\\n common_utils.check_numpy_to_torch(box3d)\\n transformation_matrix, _ = \\\\\\n common_utils.check_numpy_to_torch(transformation_matrix)\\n\\n # (N, 3, 8)\\n box3d_corner = box3d.transpose(1, 2)\\n # (N, 1, 8)\\n torch_ones = torch.ones((box3d_corner.shape[0], 1, 8))\\n torch_ones = torch_ones.to(box3d_corner.device)\\n # (N, 4, 8)\\n box3d_corner = torch.cat((box3d_corner, torch_ones),\\n dim=1)\\n # (N, 4, 8)\\n projected_box3d = torch.matmul(transformation_matrix,\\n box3d_corner)\\n # (N, 8, 3)\\n projected_box3d = projected_box3d[:, :3, :].transpose(1, 2)\\n\\n return projected_box3d if not is_numpy else projected_box3d.numpy()\\n\\n\\ndef project_points_by_matrix_torch(points, transformation_matrix):\\n \\\"\\\"\\\"\\n Project the points to another coordinate system based on the\\n transfomration matrix.\\n\\n Parameters\\n ----------\\n points : torch.Tensor\\n 3D points, (N, 3)\\n\\n transformation_matrix : torch.Tensor\\n Transformation matrix, (4, 4)\\n\\n Returns\\n -------\\n projected_points : torch.Tensor\\n The projected points, (N, 3)\\n \\\"\\\"\\\"\\n # convert to homogeneous coordinates via padding 1 at the last dimension.\\n # (N, 4)\\n points_homogeneous = F.pad(points, (0, 1), mode=\\\"constant\\\", value=1)\\n # (N, 4)\\n projected_points = torch.einsum(\\\"ik, jk->ij\\\", points_homogeneous,\\n transformation_matrix)\\n return projected_points[:, :3]\\n\\n\\ndef get_mask_for_boxes_within_range_torch(boxes):\\n \\\"\\\"\\\"\\n Generate mask to remove the bounding boxes\\n outside the range.\\n\\n Parameters\\n ----------\\n boxes : torch.Tensor\\n Groundtruth bbx, shape: N,8,3 or N,4,2\\n Returns\\n -------\\n mask: torch.Tensor\\n The mask for bounding box -- True means the\\n bbx is within the range and False means the\\n bbx is outside the range.\\n \\\"\\\"\\\"\\n from v2xvit.data_utils.datasets import GT_RANGE\\n\\n # mask out the gt bounding box out fixed range (-140, -40, -3, 140, 40 1)\\n device = boxes.device\\n boundary_lower_range = \\\\\\n torch.Tensor(GT_RANGE[:2]).reshape(1, 1, -1).to(device)\\n boundary_higher_range = \\\\\\n torch.Tensor(GT_RANGE[3:5]).reshape(1, 1, -1).to(device)\\n\\n mask = torch.all(\\n torch.all(boxes[:, :, :2] >= boundary_lower_range,\\n dim=-1) & \\\\\\n torch.all(boxes[:, :, :2] <= boundary_higher_range,\\n dim=-1), dim=-1)\\n\\n return mask\\n\\n\\ndef mask_boxes_outside_range_numpy(boxes, limit_range, order,\\n min_num_corners=8):\\n \\\"\\\"\\\"\\n Parameters\\n ----------\\n boxes: np.ndarray\\n (N, 7) [x, y, z, dx, dy, dz, heading], (x, y, z) is the box center\\n\\n limit_range: list\\n [minx, miny, minz, maxx, maxy, maxz]\\n\\n min_num_corners: int\\n The required minimum number of corners to be considered as in range.\\n\\n order : str\\n 'lwh' or 'hwl'\\n\\n Returns\\n -------\\n boxes: np.ndarray\\n The filtered boxes.\\n \\\"\\\"\\\"\\n assert boxes.shape[1] == 8 or boxes.shape[1] == 7\\n\\n new_boxes = boxes.copy()\\n if boxes.shape[1] == 7:\\n new_boxes = boxes_to_corners_3d(new_boxes, order)\\n\\n mask = ((new_boxes >= limit_range[0:3]) &\\n (new_boxes <= limit_range[3:6])).all(axis=2)\\n mask = mask.sum(axis=1) >= min_num_corners # (N)\\n\\n return boxes[mask]\\n\\n\\ndef create_bbx(extent):\\n \\\"\\\"\\\"\\n Create bounding box with 8 corners under obstacle vehicle reference.\\n\\n Parameters\\n ----------\\n extent : list\\n Width, height, length of the bbx.\\n\\n Returns\\n -------\\n bbx : np.array\\n The bounding box with 8 corners, shape: (8, 3)\\n \\\"\\\"\\\"\\n\\n bbx = np.array([[extent[0], -extent[1], -extent[2]],\\n [extent[0], extent[1], -extent[2]],\\n [-extent[0], extent[1], -extent[2]],\\n [-extent[0], -extent[1], -extent[2]],\\n [extent[0], -extent[1], extent[2]],\\n [extent[0], extent[1], extent[2]],\\n [-extent[0], extent[1], extent[2]],\\n [-extent[0], -extent[1], extent[2]]])\\n\\n return bbx\\n\\n\\ndef project_world_objects(object_dict,\\n output_dict,\\n lidar_pose,\\n lidar_range,\\n order):\\n \\\"\\\"\\\"\\n Project the objects under world coordinates into another coordinate\\n based on the provided extrinsic.\\n\\n Parameters\\n ----------\\n object_dict : dict\\n The dictionary contains all objects surrounding a certain cav.\\n\\n output_dict : dict\\n key: object id, value: object bbx (xyzlwhyaw).\\n\\n lidar_pose : list\\n (6, ), lidar pose under world coordinate, [x, y, z, roll, yaw, pitch].\\n\\n lidar_range : list\\n [minx, miny, minz, maxx, maxy, maxz]\\n\\n order : str\\n 'lwh' or 'hwl'\\n \\\"\\\"\\\"\\n for object_id, object_content in object_dict.items():\\n location = object_content['location']\\n rotation = object_content['angle']\\n center = object_content['center']\\n extent = object_content['extent']\\n\\n object_pose = [location[0] + center[0],\\n location[1] + center[1],\\n location[2] + center[2],\\n rotation[0], rotation[1], rotation[2]]\\n object2lidar = x1_to_x2(object_pose, lidar_pose)\\n\\n # shape (3, 8)\\n bbx = create_bbx(extent).T\\n # bounding box under ego coordinate shape (4, 8)\\n bbx = np.r_[bbx, [np.ones(bbx.shape[1])]]\\n\\n # project the 8 corners to world coordinate\\n bbx_lidar = np.dot(object2lidar, bbx).T\\n bbx_lidar = np.expand_dims(bbx_lidar[:, :3], 0)\\n bbx_lidar = corner_to_center(bbx_lidar, order=order)\\n bbx_lidar = mask_boxes_outside_range_numpy(bbx_lidar,\\n lidar_range,\\n order)\\n\\n if bbx_lidar.shape[0] > 0:\\n output_dict.update({object_id: bbx_lidar})\\n\\n\\ndef get_points_in_rotated_box(p, box_corner):\\n \\\"\\\"\\\"\\n Get points within a rotated bounding box (2D version).\\n\\n Parameters\\n ----------\\n p : numpy.array\\n Points to be tested with shape (N, 2).\\n box_corner : numpy.array\\n Corners of bounding box with shape (4, 2).\\n\\n Returns\\n -------\\n p_in_box : numpy.array\\n Points within the box.\\n\\n \\\"\\\"\\\"\\n edge1 = box_corner[1, :] - box_corner[0, :]\\n edge2 = box_corner[3, :] - box_corner[0, :]\\n p_rel = p - box_corner[0, :].reshape(1, -1)\\n\\n l1 = get_projection_length_for_vector_projection(p_rel, edge1)\\n l2 = get_projection_length_for_vector_projection(p_rel, edge2)\\n # A point is within the box, if and only after projecting the\\n # point onto the two edges s.t. p_rel = [edge1, edge2] @ [l1, l2]^T,\\n # we have 0<=l1<=1 and 0<=l2<=1.\\n mask = np.logical_and(l1 >= 0, l1 <= 1)\\n mask = np.logical_and(mask, l2 >= 0)\\n mask = np.logical_and(mask, l2 <= 1)\\n p_in_box = p[mask, :]\\n return p_in_box\\n\\n\\ndef get_points_in_rotated_box_3d(p, box_corner):\\n \\\"\\\"\\\"\\n Get points within a rotated bounding box (3D version).\\n\\n Parameters\\n ----------\\n p : numpy.array\\n Points to be tested with shape (N, 3).\\n box_corner : numpy.array\\n Corners of bounding box with shape (8, 3).\\n\\n Returns\\n -------\\n p_in_box : numpy.array\\n Points within the box.\\n\\n \\\"\\\"\\\"\\n edge1 = box_corner[1, :] - box_corner[0, :]\\n edge2 = box_corner[3, :] - box_corner[0, :]\\n edge3 = box_corner[4, :] - box_corner[0, :]\\n\\n p_rel = p - box_corner[0, :].reshape(1, -1)\\n\\n l1 = get_projection_length_for_vector_projection(p_rel, edge1)\\n l2 = get_projection_length_for_vector_projection(p_rel, edge2)\\n l3 = get_projection_length_for_vector_projection(p_rel, edge3)\\n # A point is within the box, if and only after projecting the\\n # point onto the two edges s.t. p_rel = [edge1, edge2] @ [l1, l2]^T,\\n # we have 0<=l1<=1 and 0<=l2<=1.\\n mask1 = np.logical_and(l1 >= 0, l1 <= 1)\\n mask2 = np.logical_and(l2 >= 0, l2 <= 1)\\n mask3 = np.logical_and(l3 >= 0, l3 <= 1)\\n\\n mask = np.logical_and(mask1, mask2)\\n mask = np.logical_and(mask, mask3)\\n p_in_box = p[mask, :]\\n\\n return p_in_box\\n\\n\\ndef get_projection_length_for_vector_projection(a, b):\\n \\\"\\\"\\\"\\n Get projection length for the Vector projection of a onto b s.t.\\n a_projected = length * b. (2D version) See\\n https://en.wikipedia.org/wiki/Vector_projection#Vector_projection_2\\n for more details.\\n\\n Parameters\\n ----------\\n a : numpy.array\\n The vectors to be projected with shape (N, 2).\\n\\n b : numpy.array\\n The vector that is projected onto with shape (2).\\n\\n Returns\\n -------\\n length : numpy.array\\n The length of projected a with respect to b.\\n \\\"\\\"\\\"\\n assert np.sum(b ** 2, axis=-1) > 1e-6\\n length = a.dot(b) / np.sum(b ** 2, axis=-1)\\n return length\\n\\n\\ndef nms_rotated(boxes, scores, threshold):\\n \\\"\\\"\\\"Performs rorated non-maximum suppression and returns indices of kept\\n boxes.\\n\\n Parameters\\n ----------\\n boxes : torch.tensor\\n The location preds with shape (N, 4, 2).\\n\\n scores : torch.tensor\\n The predicted confidence score with shape (N,)\\n\\n threshold: float\\n IoU threshold to use for filtering.\\n\\n Returns\\n -------\\n An array of index\\n \\\"\\\"\\\"\\n if boxes.shape[0] == 0:\\n return np.array([], dtype=np.int32)\\n boxes = boxes.cpu().detach().numpy()\\n scores = scores.cpu().detach().numpy()\\n\\n polygons = common_utils.convert_format(boxes)\\n\\n top = 1000\\n # Get indicies of boxes sorted by scores (highest first)\\n ixs = scores.argsort()[::-1][:top]\\n\\n pick = []\\n while len(ixs) > 0:\\n # Pick top box and add its index to the list\\n i = ixs[0]\\n pick.append(i)\\n # Compute IoU of the picked box with the rest\\n iou = common_utils.compute_iou(polygons[i], polygons[ixs[1:]])\\n # Identify boxes with IoU over the threshold. This\\n # returns indices into ixs[1:], so add 1 to get\\n # indices into ixs.\\n remove_ixs = np.where(iou > threshold)[0] + 1\\n # Remove indices of the picked and overlapped boxes.\\n ixs = np.delete(ixs, remove_ixs)\\n ixs = np.delete(ixs, 0)\\n\\n return np.array(pick, dtype=np.int32)\\n\\n\\ndef nms_pytorch(boxes: torch.tensor, thresh_iou: float):\\n \\\"\\\"\\\"\\n Apply non-maximum suppression to avoid detecting too many\\n overlapping bounding boxes for a given object.\\n\\n Parameters\\n ----------\\n boxes : torch.tensor\\n The location preds along with the class predscores,\\n Shape: [num_boxes,5].\\n thresh_iou : float\\n (float) The overlap thresh for suppressing unnecessary boxes.\\n Returns\\n -------\\n A list of index\\n \\\"\\\"\\\"\\n\\n # we extract coordinates for every\\n # prediction box present in P\\n x1 = boxes[:, 0]\\n y1 = boxes[:, 1]\\n x2 = boxes[:, 2]\\n y2 = boxes[:, 3]\\n\\n # we extract the confidence scores as well\\n scores = boxes[:, 4]\\n\\n # calculate area of every block in P\\n areas = (x2 - x1) * (y2 - y1)\\n\\n # sort the prediction boxes in P\\n # according to their confidence scores\\n order = scores.argsort()\\n\\n # initialise an empty list for\\n # filtered prediction boxes\\n keep = []\\n\\n while len(order) > 0:\\n\\n # extract the index of the\\n # prediction with highest score\\n # we call this prediction S\\n idx = order[-1]\\n\\n # push S in filtered predictions list\\n keep.append(idx.numpy().item()\\n if not idx.is_cuda else idx.cpu().detach().numpy().item())\\n\\n # remove S from P\\n order = order[:-1]\\n\\n # sanity check\\n if len(order) == 0:\\n break\\n\\n # select coordinates of BBoxes according to\\n # the indices in order\\n xx1 = torch.index_select(x1, dim=0, index=order)\\n xx2 = torch.index_select(x2, dim=0, index=order)\\n yy1 = torch.index_select(y1, dim=0, index=order)\\n yy2 = torch.index_select(y2, dim=0, index=order)\\n\\n # find the coordinates of the intersection boxes\\n xx1 = torch.max(xx1, x1[idx])\\n yy1 = torch.max(yy1, y1[idx])\\n xx2 = torch.min(xx2, x2[idx])\\n yy2 = torch.min(yy2, y2[idx])\\n\\n # find height and width of the intersection boxes\\n w = xx2 - xx1\\n h = yy2 - yy1\\n\\n # take max with 0.0 to avoid negative w and h\\n # due to non-overlapping boxes\\n w = torch.clamp(w, min=0.0)\\n h = torch.clamp(h, min=0.0)\\n\\n # find the intersection area\\n inter = w * h\\n\\n # find the areas of BBoxes according the indices in order\\n rem_areas = torch.index_select(areas, dim=0, index=order)\\n\\n # find the union of every prediction T in P\\n # with the prediction S\\n # Note that areas[idx] represents area of S\\n union = (rem_areas - inter) + areas[idx]\\n\\n # find the IoU of every prediction in P with S\\n IoU = inter / union\\n\\n # keep the boxes with IoU less than thresh_iou\\n mask = IoU < thresh_iou\\n order = order[mask]\\n\\n return keep\\n\\n\\ndef remove_large_pred_bbx(bbx_3d):\\n \\\"\\\"\\\"\\n Remove large bounding box.\\n\\n Parameters\\n ----------\\n bbx_3d : torch.Tensor\\n Predcited 3d bounding box, shape:(N,8,3)\\n\\n Returns\\n -------\\n index : torch.Tensor\\n The keep index.\\n \\\"\\\"\\\"\\n bbx_x_max = torch.max(bbx_3d[:, :, 0], dim=1)[0]\\n bbx_x_min = torch.min(bbx_3d[:, :, 0], dim=1)[0]\\n x_len = bbx_x_max - bbx_x_min\\n\\n bbx_y_max = torch.max(bbx_3d[:, :, 1], dim=1)[0]\\n bbx_y_min = torch.min(bbx_3d[:, :, 1], dim=1)[0]\\n y_len = bbx_y_max - bbx_y_min\\n\\n bbx_z_max = torch.max(bbx_3d[:, :, 1], dim=1)[0]\\n bbx_z_min = torch.min(bbx_3d[:, :, 1], dim=1)[0]\\n z_len = bbx_z_max - bbx_z_min\\n\\n index = torch.logical_and(x_len <= 6, y_len <= 6)\\n index = torch.logical_and(index, z_len)\\n\\n return index\\n\\n\\ndef remove_bbx_abnormal_z(bbx_3d):\\n \\\"\\\"\\\"\\n Remove bounding box that has negative z axis.\\n\\n Parameters\\n ----------\\n bbx_3d : torch.Tensor\\n Predcited 3d bounding box, shape:(N,8,3)\\n\\n Returns\\n -------\\n index : torch.Tensor\\n The keep index.\\n \\\"\\\"\\\"\\n bbx_z_min = torch.min(bbx_3d[:, :, 2], dim=1)[0]\\n bbx_z_max = torch.max(bbx_3d[:, :, 2], dim=1)[0]\\n index = torch.logical_and(bbx_z_min >= -3, bbx_z_max <= 1)\\n\\n return index\\n\\n\\ndef project_points_by_matrix_torch(points, transformation_matrix):\\n \\\"\\\"\\\"\\n Project the points to another coordinate system based on the\\n transformation matrix.\\n\\n Parameters\\n ----------\\n points : torch.Tensor\\n 3D points, (N, 3)\\n transformation_matrix : torch.Tensor\\n Transformation matrix, (4, 4)\\n Returns\\n -------\\n projected_points : torch.Tensor\\n The projected points, (N, 3)\\n \\\"\\\"\\\"\\n points, is_numpy = \\\\\\n common_utils.check_numpy_to_torch(points)\\n transformation_matrix, _ = \\\\\\n common_utils.check_numpy_to_torch(transformation_matrix)\\n\\n # convert to homogeneous coordinates via padding 1 at the last dimension.\\n # (N, 4)\\n points_homogeneous = F.pad(points, (0, 1), mode=\\\"constant\\\", value=1)\\n # (N, 4)\\n projected_points = torch.einsum(\\\"ik, jk->ij\\\", points_homogeneous,\\n transformation_matrix)\\n\\n return projected_points[:, :3] if not is_numpy \\\\\\n else projected_points[:, :3].numpy()\\n\\n\\nif __name__ == \\\"__main__\\\":\\n x = np.arange(-5, 5, 0.1)\\n y = np.arange(-5, 5, 0.1)\\n xx, yy = np.meshgrid(x, y)\\n points = np.concatenate([xx.reshape(-1, 1), yy.reshape(-1, 1)], axis=-1)\\n box_corners = np.array([\\n [2, -2], [2, 2], [-2, 2], [-2, -2]\\n ])\\n temp = get_points_in_rotated_box(points, box_corners)\\n assert np.all(np.logical_and(temp[:, 0] >= -2, temp[:, 0] <= 2))\\n assert np.all(np.logical_and(temp[:, 1] >= -2, temp[:, 1] <= 2))\\n\\n\\nfrom distutils.core import setup\\nfrom Cython.Build import cythonize\\nimport numpy\\nsetup(\\n name='box overlaps',\\n ext_modules=cythonize('v2xvit/utils/box_overlaps.pyx'),\\n include_dirs=[numpy.get_include()]\\n)\\n\\n\\\"\\\"\\\"\\nUtility functions related to point cloud\\n\\\"\\\"\\\"\\n\\nimport open3d as o3d\\nimport numpy as np\\n\\n\\ndef pcd_to_np(pcd_file):\\n \\\"\\\"\\\"\\n Read pcd and return numpy array.\\n\\n Parameters\\n ----------\\n pcd_file : str\\n The pcd file that contains the point cloud.\\n\\n Returns\\n -------\\n pcd : o3d.PointCloud\\n PointCloud object, used for visualization\\n pcd_np : np.ndarray\\n The lidar data in numpy format, shape:(n, 4)\\n\\n \\\"\\\"\\\"\\n pcd = o3d.io.read_point_cloud(pcd_file)\\n\\n xyz = np.asarray(pcd.points)\\n # we save the intensity in the first channel\\n intensity = np.expand_dims(np.asarray(pcd.colors)[:, 0], -1)\\n pcd_np = np.hstack((xyz, intensity))\\n\\n return np.asarray(pcd_np, dtype=np.float32)\\n\\n\\ndef mask_points_by_range(points, limit_range):\\n \\\"\\\"\\\"\\n Remove the lidar points out of the boundary.\\n\\n Parameters\\n ----------\\n points : np.ndarray\\n Lidar points under lidar sensor coordinate system.\\n\\n limit_range : list\\n [x_min, y_min, z_min, x_max, y_max, z_max]\\n\\n Returns\\n -------\\n points : np.ndarray\\n Filtered lidar points.\\n \\\"\\\"\\\"\\n\\n mask = (points[:, 0] > limit_range[0]) & (points[:, 0] < limit_range[3])\\\\\\n & (points[:, 1] > limit_range[1]) & (\\n points[:, 1] < limit_range[4]) \\\\\\n & (points[:, 2] > limit_range[2]) & (\\n points[:, 2] < limit_range[5])\\n\\n points = points[mask]\\n\\n return points\\n\\n\\ndef mask_ego_points(points):\\n \\\"\\\"\\\"\\n Remove the lidar points of the ego vehicle itself.\\n\\n Parameters\\n ----------\\n points : np.ndarray\\n Lidar points under lidar sensor coordinate system.\\n\\n Returns\\n -------\\n points : np.ndarray\\n Filtered lidar points.\\n \\\"\\\"\\\"\\n mask = (points[:, 0] >= -1.95) & (points[:, 0] <= 2.95) \\\\\\n & (points[:, 1] >= -1.1) & (points[:, 1] <= 1.1)\\n points = points[np.logical_not(mask)]\\n\\n return points\\n\\n\\ndef shuffle_points(points):\\n shuffle_idx = np.random.permutation(points.shape[0])\\n points = points[shuffle_idx]\\n\\n return points\\n\\n\\ndef lidar_project(lidar_data, extrinsic):\\n \\\"\\\"\\\"\\n Given the extrinsic matrix, project lidar data to another space.\\n\\n Parameters\\n ----------\\n lidar_data : np.ndarray\\n Lidar data, shape: (n, 4)\\n\\n extrinsic : np.ndarray\\n Extrinsic matrix, shape: (4, 4)\\n\\n Returns\\n -------\\n projected_lidar : np.ndarray\\n Projected lida data, shape: (n, 4)\\n \\\"\\\"\\\"\\n\\n lidar_xyz = lidar_data[:, :3].T\\n # (3, n) -> (4, n), homogeneous transformation\\n lidar_xyz = np.r_[lidar_xyz, [np.ones(lidar_xyz.shape[1])]]\\n lidar_int = lidar_data[:, 3]\\n\\n # transform to ego vehicle space, (3, n)\\n project_lidar_xyz = np.dot(extrinsic, lidar_xyz)[:3, :]\\n # (n, 3)\\n project_lidar_xyz = project_lidar_xyz.T\\n # concatenate the intensity with xyz, (n, 4)\\n projected_lidar = np.hstack((project_lidar_xyz,\\n np.expand_dims(lidar_int, -1)))\\n\\n return projected_lidar\\n\\n\\ndef projected_lidar_stack(projected_lidar_list):\\n \\\"\\\"\\\"\\n Stack all projected lidar together.\\n\\n Parameters\\n ----------\\n projected_lidar_list : list\\n The list containing all projected lidar.\\n\\n Returns\\n -------\\n stack_lidar : np.ndarray\\n Stack all projected lidar data together.\\n \\\"\\\"\\\"\\n stack_lidar = []\\n for lidar_data in projected_lidar_list:\\n stack_lidar.append(lidar_data)\\n\\n return np.vstack(stack_lidar)\\n\\n\\ndef downsample_lidar(pcd_np, num):\\n \\\"\\\"\\\"\\n Downsample the lidar points to a certain number.\\n\\n Parameters\\n ----------\\n pcd_np : np.ndarray\\n The lidar points, (n, 4).\\n\\n num : int\\n The downsample target number.\\n\\n Returns\\n -------\\n pcd_np : np.ndarray\\n The downsampled lidar points.\\n \\\"\\\"\\\"\\n assert pcd_np.shape[0] >= num\\n\\n selected_index = np.random.choice((pcd_np.shape[0]),\\n num,\\n replace=False)\\n pcd_np = pcd_np[selected_index]\\n\\n return pcd_np\\n\\n\\ndef downsample_lidar_minimum(pcd_np_list):\\n \\\"\\\"\\\"\\n Given a list of pcd, find the minimum number and downsample all\\n point clouds to the minimum number.\\n\\n Parameters\\n ----------\\n pcd_np_list : list\\n A list of pcd numpy array(n, 4).\\n Returns\\n -------\\n pcd_np_list : list\\n Downsampled point clouds.\\n \\\"\\\"\\\"\\n minimum = np.Inf\\n\\n for i in range(len(pcd_np_list)):\\n num = pcd_np_list[i].shape[0]\\n minimum = num if minimum > num else minimum\\n\\n for (i, pcd_np) in enumerate(pcd_np_list):\\n pcd_np_list[i] = downsample_lidar(pcd_np, minimum)\\n\\n return pcd_np_list\\n\\n\\n\\n\\nimport os\\n\\nimport numpy as np\\nimport torch\\n\\nfrom v2xvit.utils import common_utils\\nfrom v2xvit.hypes_yaml import yaml_utils\\n\\n\\ndef voc_ap(rec, prec):\\n \\\"\\\"\\\"\\n VOC 2010 Average Precision.\\n \\\"\\\"\\\"\\n rec.insert(0, 0.0)\\n rec.append(1.0)\\n mrec = rec[:]\\n\\n prec.insert(0, 0.0)\\n prec.append(0.0)\\n mpre = prec[:]\\n\\n for i in range(len(mpre) - 2, -1, -1):\\n mpre[i] = max(mpre[i], mpre[i + 1])\\n\\n i_list = []\\n for i in range(1, len(mrec)):\\n if mrec[i] != mrec[i - 1]:\\n i_list.append(i)\\n\\n ap = 0.0\\n for i in i_list:\\n ap += ((mrec[i] - mrec[i - 1]) * mpre[i])\\n return ap, mrec, mpre\\n\\n\\ndef caluclate_tp_fp(det_boxes, det_score, gt_boxes, result_stat, iou_thresh):\\n \\\"\\\"\\\"\\n Calculate the true positive and false positive numbers of the current\\n frames.\\n\\n Parameters\\n ----------\\n det_boxes : torch.Tensor\\n The detection bounding box, shape (N, 8, 3) or (N, 4, 2).\\n det_score :torch.Tensor\\n The confidence score for each preditect bounding box.\\n gt_boxes : torch.Tensor\\n The groundtruth bounding box.\\n result_stat: dict\\n A dictionary contains fp, tp and gt number.\\n iou_thresh : float\\n The iou thresh.\\n \\\"\\\"\\\"\\n # fp, tp and gt in the current frame\\n fp = []\\n tp = []\\n gt = gt_boxes.shape[0]\\n if det_boxes is not None:\\n # convert bounding boxes to numpy array\\n det_boxes = common_utils.torch_tensor_to_numpy(det_boxes)\\n det_score = common_utils.torch_tensor_to_numpy(det_score)\\n gt_boxes = common_utils.torch_tensor_to_numpy(gt_boxes)\\n\\n # sort the prediction bounding box by score\\n score_order_descend = np.argsort(-det_score)\\n det_polygon_list = list(common_utils.convert_format(det_boxes))\\n gt_polygon_list = list(common_utils.convert_format(gt_boxes))\\n\\n # match prediction and gt bounding box\\n for i in range(score_order_descend.shape[0]):\\n det_polygon = det_polygon_list[score_order_descend[i]]\\n ious = common_utils.compute_iou(det_polygon, gt_polygon_list)\\n\\n if len(gt_polygon_list) == 0 or np.max(ious) < iou_thresh:\\n fp.append(1)\\n tp.append(0)\\n continue\\n\\n fp.append(0)\\n tp.append(1)\\n\\n gt_index = np.argmax(ious)\\n gt_polygon_list.pop(gt_index)\\n\\n result_stat[iou_thresh]['fp'] += fp\\n result_stat[iou_thresh]['tp'] += tp\\n result_stat[iou_thresh]['gt'] += gt\\n\\n\\ndef calculate_ap(result_stat, iou):\\n \\\"\\\"\\\"\\n Calculate the average precision and recall, and save them into a txt.\\n\\n Parameters\\n ----------\\n result_stat : dict\\n A dictionary contains fp, tp and gt number.\\n iou : float\\n \\\"\\\"\\\"\\n iou_5 = result_stat[iou]\\n\\n fp = iou_5['fp']\\n tp = iou_5['tp']\\n assert len(fp) == len(tp)\\n\\n gt_total = iou_5['gt']\\n\\n cumsum = 0\\n for idx, val in enumerate(fp):\\n fp[idx] += cumsum\\n cumsum += val\\n\\n cumsum = 0\\n for idx, val in enumerate(tp):\\n tp[idx] += cumsum\\n cumsum += val\\n\\n rec = tp[:]\\n for idx, val in enumerate(tp):\\n rec[idx] = float(tp[idx]) / gt_total\\n\\n prec = tp[:]\\n for idx, val in enumerate(tp):\\n prec[idx] = float(tp[idx]) / (fp[idx] + tp[idx])\\n\\n ap, mrec, mprec = voc_ap(rec[:], prec[:])\\n\\n return ap, mrec, mprec\\n\\n\\ndef eval_final_results(result_stat, save_path):\\n dump_dict = {}\\n\\n ap_30, mrec_30, mpre_30 = calculate_ap(result_stat, 0.30)\\n ap_50, mrec_50, mpre_50 = calculate_ap(result_stat, 0.50)\\n ap_70, mrec_70, mpre_70 = calculate_ap(result_stat, 0.70)\\n\\n dump_dict.update({'ap30': ap_30,\\n 'ap_50': ap_50,\\n 'ap_70': ap_70,\\n 'mpre_50': mpre_50,\\n 'mrec_50': mrec_50,\\n 'mpre_70': mpre_70,\\n 'mrec_70': mrec_70,\\n })\\n yaml_utils.save_yaml(dump_dict, os.path.join(save_path, 'eval.yaml'))\\n\\n print('The Average Precision at IOU 0.3 is %.2f, '\\n 'The Average Precision at IOU 0.5 is %.2f, '\\n 'The Average Precision at IOU 0.7 is %.2f' % (ap_30, ap_50, ap_70))\\n\\n\\n{\\n\\t\\\"class_name\\\" : \\\"PinholeCameraParameters\\\",\\n\\t\\\"extrinsic\\\" : \\n\\t[\\n\\t\\t1.0,\\n\\t\\t-0.0,\\n\\t\\t-0.0,\\n\\t\\t0.0,\\n\\t\\t0.0,\\n\\t\\t-1.0,\\n\\t\\t-0.0,\\n\\t\\t0.0,\\n\\t\\t0.0,\\n\\t\\t-0.0,\\n\\t\\t-1.0,\\n\\t\\t0.0,\\n\\t\\t14.870189666748047,\\n\\t\\t0.0001621246337890625,\\n\\t\\t141.0903074604017,\\n\\t\\t1.0\\n\\t],\\n\\t\\\"intrinsic\\\" : \\n\\t{\\n\\t\\t\\\"height\\\" : 1025,\\n\\t\\t\\\"intrinsic_matrix\\\" : \\n\\t\\t[\\n\\t\\t\\t887.67603887904966,\\n\\t\\t\\t0.0,\\n\\t\\t\\t0.0,\\n\\t\\t\\t0.0,\\n\\t\\t\\t887.67603887904966,\\n\\t\\t\\t0.0,\\n\\t\\t\\t926.0,\\n\\t\\t\\t512.0,\\n\\t\\t\\t1.0\\n\\t\\t],\\n\\t\\t\\\"width\\\" : 1853\\n\\t},\\n\\t\\\"version_major\\\" : 1,\\n\\t\\\"version_minor\\\" : 0\\n}\\n\\nimport time\\n\\nimport cv2\\nimport numpy as np\\nimport open3d as o3d\\nimport matplotlib\\nimport matplotlib.pyplot as plt\\n\\nfrom matplotlib import cm\\n\\nfrom v2xvit.utils import box_utils\\nfrom v2xvit.utils import common_utils\\n\\nVIRIDIS = np.array(cm.get_cmap('plasma').colors)\\nVID_RANGE = np.linspace(0.0, 1.0, VIRIDIS.shape[0])\\n\\n\\ndef bbx2linset(bbx_corner, order='hwl', color=(0, 1, 0)):\\n \\\"\\\"\\\"\\n Convert the torch tensor bounding box to o3d lineset for visualization.\\n\\n Parameters\\n ----------\\n bbx_corner : torch.Tensor\\n shape: (n, 8, 3).\\n\\n order : str\\n The order of the bounding box if shape is (n, 7)\\n\\n color : tuple\\n The bounding box color.\\n\\n Returns\\n -------\\n line_set : list\\n The list containing linsets.\\n \\\"\\\"\\\"\\n if not isinstance(bbx_corner, np.ndarray):\\n bbx_corner = common_utils.torch_tensor_to_numpy(bbx_corner)\\n\\n if len(bbx_corner.shape) == 2:\\n bbx_corner = box_utils.boxes_to_corners_3d(bbx_corner,\\n order)\\n\\n # Our lines span from points 0 to 1, 1 to 2, 2 to 3, etc...\\n lines = [[0, 1], [1, 2], [2, 3], [0, 3],\\n [4, 5], [5, 6], [6, 7], [4, 7],\\n [0, 4], [1, 5], [2, 6], [3, 7]]\\n\\n # Use the same color for all lines\\n colors = [list(color) for _ in range(len(lines))]\\n bbx_linset = []\\n\\n for i in range(bbx_corner.shape[0]):\\n bbx = bbx_corner[i]\\n # o3d use right-hand coordinate\\n bbx[:, :1] = - bbx[:, :1]\\n\\n line_set = o3d.geometry.LineSet()\\n line_set.points = o3d.utility.Vector3dVector(bbx)\\n line_set.lines = o3d.utility.Vector2iVector(lines)\\n line_set.colors = o3d.utility.Vector3dVector(colors)\\n bbx_linset.append(line_set)\\n\\n return bbx_linset\\n\\n\\ndef bbx2oabb(bbx_corner, order='hwl', color=(0, 0, 1)):\\n \\\"\\\"\\\"\\n Convert the torch tensor bounding box to o3d oabb for visualization.\\n\\n Parameters\\n ----------\\n bbx_corner : torch.Tensor\\n shape: (n, 8, 3).\\n\\n order : str\\n The order of the bounding box if shape is (n, 7)\\n\\n color : tuple\\n The bounding box color.\\n\\n Returns\\n -------\\n oabbs : list\\n The list containing all oriented bounding boxes.\\n \\\"\\\"\\\"\\n if not isinstance(bbx_corner, np.ndarray):\\n bbx_corner = common_utils.torch_tensor_to_numpy(bbx_corner)\\n\\n if len(bbx_corner.shape) == 2:\\n bbx_corner = box_utils.boxes_to_corners_3d(bbx_corner,\\n order)\\n oabbs = []\\n\\n for i in range(bbx_corner.shape[0]):\\n bbx = bbx_corner[i]\\n # o3d use right-hand coordinate\\n bbx[:, :1] = - bbx[:, :1]\\n\\n tmp_pcd = o3d.geometry.PointCloud()\\n tmp_pcd.points = o3d.utility.Vector3dVector(bbx)\\n\\n oabb = tmp_pcd.get_oriented_bounding_box()\\n oabb.color = color\\n oabbs.append(oabb)\\n\\n return oabbs\\n\\n\\ndef bbx2aabb(bbx_center, order):\\n \\\"\\\"\\\"\\n Convert the torch tensor bounding box to o3d aabb for visualization.\\n\\n Parameters\\n ----------\\n bbx_center : torch.Tensor\\n shape: (n, 7).\\n\\n order: str\\n hwl or lwh.\\n\\n Returns\\n -------\\n aabbs : list\\n The list containing all o3d.aabb\\n \\\"\\\"\\\"\\n if not isinstance(bbx_center, np.ndarray):\\n bbx_center = common_utils.torch_tensor_to_numpy(bbx_center)\\n bbx_corner = box_utils.boxes_to_corners_3d(bbx_center, order)\\n\\n aabbs = []\\n\\n for i in range(bbx_corner.shape[0]):\\n bbx = bbx_corner[i]\\n # o3d use right-hand coordinate\\n bbx[:, :1] = - bbx[:, :1]\\n\\n tmp_pcd = o3d.geometry.PointCloud()\\n tmp_pcd.points = o3d.utility.Vector3dVector(bbx)\\n\\n aabb = tmp_pcd.get_axis_aligned_bounding_box()\\n aabb.color = (0, 0, 1)\\n aabbs.append(aabb)\\n\\n return aabbs\\n\\ndef linset_assign_list(vis,\\n lineset_list1,\\n lineset_list2,\\n update_mode='update'):\\n \\\"\\\"\\\"\\n Associate two lists of lineset.\\n\\n Parameters\\n ----------\\n vis : open3d.Visualizer\\n lineset_list1 : list\\n lineset_list2 : list\\n update_mode : str\\n Add or update the geometry.\\n \\\"\\\"\\\"\\n for j in range(len(lineset_list1)):\\n index = j if j < len(lineset_list2) else -1\\n lineset_list1[j] = \\\\\\n lineset_assign(lineset_list1[j],\\n lineset_list2[index])\\n if update_mode == 'add':\\n vis.add_geometry(lineset_list1[j])\\n else:\\n vis.update_geometry(lineset_list1[j])\\n\\n\\ndef lineset_assign(lineset1, lineset2):\\n \\\"\\\"\\\"\\n Assign the attributes of lineset2 to lineset1.\\n\\n Parameters\\n ----------\\n lineset1 : open3d.LineSet\\n lineset2 : open3d.LineSet\\n\\n Returns\\n -------\\n The lineset1 object with 2's attributes.\\n \\\"\\\"\\\"\\n\\n lineset1.points = lineset2.points\\n lineset1.lines = lineset2.lines\\n lineset1.colors = lineset2.colors\\n\\n return lineset1\\n\\n\\ndef color_encoding(intensity, mode='intensity'):\\n \\\"\\\"\\\"\\n Encode the single-channel intensity to 3 channels rgb color.\\n\\n Parameters\\n ----------\\n intensity : np.ndarray\\n Lidar intensity, shape (n,)\\n\\n mode : str\\n The color rendering mode. intensity, z-value and constant are\\n supported.\\n\\n Returns\\n -------\\n color : np.ndarray\\n Encoded Lidar color, shape (n, 3)\\n \\\"\\\"\\\"\\n assert mode in ['intensity', 'z-value', 'constant']\\n\\n if mode == 'intensity':\\n intensity_col = 1.0 - np.log(intensity) / np.log(np.exp(-0.004 * 100))\\n int_color = np.c_[\\n np.interp(intensity_col, VID_RANGE, VIRIDIS[:, 0]),\\n np.interp(intensity_col, VID_RANGE, VIRIDIS[:, 1]),\\n np.interp(intensity_col, VID_RANGE, VIRIDIS[:, 2])]\\n\\n elif mode == 'z-value':\\n min_value = -1.5\\n max_value = 0.5\\n norm = matplotlib.colors.Normalize(vmin=min_value, vmax=max_value)\\n cmap = cm.jet\\n m = cm.ScalarMappable(norm=norm, cmap=cmap)\\n\\n colors = m.to_rgba(intensity)\\n colors[:, [2, 1, 0, 3]] = colors[:, [0, 1, 2, 3]]\\n colors[:, 3] = 0.5\\n int_color = colors[:, :3]\\n\\n elif mode == 'constant':\\n # regard all point cloud the same color\\n int_color = np.ones((intensity.shape[0], 3))\\n int_color[:, 0] *= 247 / 255\\n int_color[:, 1] *= 244 / 255\\n int_color[:, 2] *= 237 / 255\\n\\n return int_color\\n\\n\\ndef visualize_single_sample_output_gt(pred_tensor,\\n gt_tensor,\\n pcd,\\n show_vis=True,\\n save_path='',\\n mode='constant'):\\n \\\"\\\"\\\"\\n Visualize the prediction, groundtruth with point cloud together.\\n\\n Parameters\\n ----------\\n pred_tensor : torch.Tensor\\n (N, 8, 3) prediction.\\n\\n gt_tensor : torch.Tensor\\n (N, 8, 3) groundtruth bbx\\n\\n pcd : torch.Tensor\\n PointCloud, (N, 4).\\n\\n show_vis : bool\\n Whether to show visualization.\\n\\n save_path : str\\n Save the visualization results to given path.\\n\\n mode : str\\n Color rendering mode.\\n \\\"\\\"\\\"\\n\\n def custom_draw_geometry(pcd, pred, gt):\\n vis = o3d.visualization.Visualizer()\\n vis.create_window()\\n\\n opt = vis.get_render_option()\\n opt.background_color = np.asarray([0, 0, 0])\\n opt.point_size = 1.0\\n\\n vis.add_geometry(pcd)\\n for ele in pred:\\n vis.add_geometry(ele)\\n for ele in gt:\\n vis.add_geometry(ele)\\n\\n vis.run()\\n vis.destroy_window()\\n\\n origin_lidar = pcd\\n if not isinstance(pcd, np.ndarray):\\n origin_lidar = common_utils.torch_tensor_to_numpy(pcd)\\n\\n origin_lidar_intcolor = \\\\\\n color_encoding(origin_lidar[:, -1] if mode == 'intensity'\\n else origin_lidar[:, 2], mode=mode)\\n # left -> right hand\\n origin_lidar[:, :1] = -origin_lidar[:, :1]\\n\\n o3d_pcd = o3d.geometry.PointCloud()\\n o3d_pcd.points = o3d.utility.Vector3dVector(origin_lidar[:, :3])\\n o3d_pcd.colors = o3d.utility.Vector3dVector(origin_lidar_intcolor)\\n\\n oabbs_pred = bbx2oabb(pred_tensor, color=(1, 0, 0))\\n oabbs_gt = bbx2oabb(gt_tensor, color=(0, 1, 0))\\n\\n visualize_elements = [o3d_pcd] + oabbs_pred + oabbs_gt\\n if show_vis:\\n custom_draw_geometry(o3d_pcd, oabbs_pred, oabbs_gt)\\n if save_path:\\n save_o3d_visualization(visualize_elements, save_path)\\n\\n\\ndef visualize_sequence_sample_output(pred_tensor_list,\\n gt_tensor_list,\\n pcd_list):\\n vis = o3d.visualization.Visualizer()\\n vis.create_window()\\n\\n vis.get_render_option().background_color = [0.05, 0.05, 0.05]\\n vis.get_render_option().point_size = 1.0\\n vis.get_render_option().show_coordinate_frame = True\\n\\n # used to visualize lidar points\\n vis_pcd = o3d.geometry.PointCloud()\\n\\n while True:\\n for i, (pred_tensor, gt_tensor, pcd) in \\\\\\n enumerate(zip(pred_tensor_list, gt_tensor_list, pcd_list)):\\n pred_tensor = pred_tensor.copy()\\n gt_tensor = gt_tensor.copy()\\n pcd = pcd.copy()\\n\\n pcd_intcolor = color_encoding(pcd[:, -1])\\n pcd[:, :1] = -pcd[:, :1]\\n vis_pcd.points = o3d.utility.Vector3dVector(pcd[:, :3])\\n vis_pcd.colors = o3d.utility.Vector3dVector(pcd_intcolor)\\n\\n oabbs_pred = bbx2oabb(pred_tensor, 'hwl')\\n oabbs_gt = bbx2oabb(gt_tensor, 'hwl', color=(0, 1, 0))\\n oabbs = oabbs_pred + oabbs_gt\\n\\n if i == 0:\\n vis.add_geometry(vis_pcd)\\n\\n for oabb in oabbs:\\n vis.add_geometry(oabb)\\n\\n vis.update_geometry(vis_pcd)\\n\\n ctr = vis.get_view_control()\\n param = o3d.io.read_pinhole_camera_parameters('pinhole_param.json')\\n ctr.convert_from_pinhole_camera_parameters(param)\\n\\n vis.poll_events()\\n vis.update_renderer()\\n\\n for oabb in oabbs:\\n vis.remove_geometry(oabb)\\n time.sleep(0.01)\\n vis.destroy_window()\\n\\n\\ndef visualize_single_sample_output_bev(pred_box, gt_box, pcd, dataset,\\n show_vis=True,\\n save_path=''):\\n \\\"\\\"\\\"\\n Visualize the prediction, groundtruth with point cloud together in\\n a bev format.\\n\\n Parameters\\n ----------\\n pred_box : torch.Tensor\\n (N, 4, 2) prediction.\\n\\n gt_box : torch.Tensor\\n (N, 4, 2) groundtruth bbx\\n\\n pcd : torch.Tensor\\n PointCloud, (N, 4).\\n\\n show_vis : bool\\n Whether to show visualization.\\n\\n save_path : str\\n Save the visualization results to given path.\\n \\\"\\\"\\\"\\n\\n if not isinstance(pcd, np.ndarray):\\n pcd = common_utils.torch_tensor_to_numpy(pcd)\\n if pred_box is not None and not isinstance(pred_box, np.ndarray):\\n pred_box = common_utils.torch_tensor_to_numpy(pred_box)\\n if gt_box is not None and not isinstance(gt_box, np.ndarray):\\n gt_box = common_utils.torch_tensor_to_numpy(gt_box)\\n\\n ratio = dataset.params[\\\"preprocess\\\"][\\\"args\\\"][\\\"res\\\"]\\n L1, W1, H1, L2, W2, H2 = dataset.params[\\\"preprocess\\\"][\\\"cav_lidar_range\\\"]\\n bev_origin = np.array([L1, W1]).reshape(1, -1)\\n # (img_row, img_col)\\n bev_map = dataset.project_points_to_bev_map(pcd, ratio)\\n # (img_row, img_col, 3)\\n bev_map = \\\\\\n np.repeat(bev_map[:, :, np.newaxis], 3, axis=-1).astype(np.float32)\\n bev_map = bev_map * 255\\n\\n if pred_box is not None:\\n num_bbx = pred_box.shape[0]\\n for i in range(num_bbx):\\n bbx = pred_box[i]\\n\\n bbx = ((bbx - bev_origin) / ratio).astype(int)\\n bbx = bbx[:, ::-1]\\n cv2.polylines(bev_map, [bbx], True, (0, 0, 255), 1)\\n\\n if gt_box is not None and len(gt_box):\\n for i in range(gt_box.shape[0]):\\n bbx = gt_box[i][:4, :2]\\n bbx = (((bbx - bev_origin)) / ratio).astype(int)\\n bbx = bbx[:, ::-1]\\n cv2.polylines(bev_map, [bbx], True, (255, 0, 0), 1)\\n\\n if show_vis:\\n plt.axis(\\\"off\\\")\\n plt.imshow(bev_map)\\n plt.show()\\n if save_path:\\n plt.axis(\\\"off\\\")\\n plt.imshow(bev_map)\\n plt.savefig(save_path)\\n\\n\\ndef visualize_single_sample_dataloader(batch_data,\\n o3d_pcd,\\n order,\\n key='origin_lidar',\\n visualize=False,\\n save_path='',\\n oabb=False,\\n mode='constant'):\\n \\\"\\\"\\\"\\n Visualize a single frame of a single CAV for validation of data pipeline.\\n\\n Parameters\\n ----------\\n o3d_pcd : o3d.PointCloud\\n Open3d PointCloud.\\n\\n order : str\\n The bounding box order.\\n\\n key : str\\n origin_lidar for late fusion and stacked_lidar for early fusion.\\n todo: consider intermediate fusion in the future.\\n\\n visualize : bool\\n Whether to visualize the sample.\\n\\n batch_data : dict\\n The dictionary that contains current timestamp's data.\\n\\n save_path : str\\n If set, save the visualization image to the path.\\n\\n oabb : bool\\n If oriented bounding box is used.\\n \\\"\\\"\\\"\\n\\n origin_lidar = batch_data[key]\\n if not isinstance(origin_lidar, np.ndarray):\\n origin_lidar = common_utils.torch_tensor_to_numpy(origin_lidar)\\n # we only visualize the first cav for single sample\\n if len(origin_lidar.shape) > 2:\\n origin_lidar = origin_lidar[0]\\n origin_lidar_intcolor = \\\\\\n color_encoding(origin_lidar[:, -1] if mode == 'intensity'\\n else origin_lidar[:, 2], mode=mode)\\n\\n # left -> right hand\\n origin_lidar[:, :1] = -origin_lidar[:, :1]\\n\\n o3d_pcd.points = o3d.utility.Vector3dVector(origin_lidar[:, :3])\\n o3d_pcd.colors = o3d.utility.Vector3dVector(origin_lidar_intcolor)\\n\\n object_bbx_center = batch_data['object_bbx_center']\\n object_bbx_mask = batch_data['object_bbx_mask']\\n object_bbx_center = object_bbx_center[object_bbx_mask == 1]\\n\\n aabbs = bbx2linset(object_bbx_center, order) if not oabb else \\\\\\n bbx2oabb(object_bbx_center, order)\\n visualize_elements = [o3d_pcd] + aabbs\\n if visualize:\\n o3d.visualization.draw_geometries(visualize_elements)\\n\\n if save_path:\\n save_o3d_visualization(visualize_elements, save_path)\\n\\n return o3d_pcd, aabbs\\n\\n\\ndef visualize_inference_sample_dataloader(pred_box_tensor,\\n gt_box_tensor,\\n origin_lidar,\\n o3d_pcd,\\n mode='constant'):\\n \\\"\\\"\\\"\\n Visualize a frame during inference for video stream.\\n\\n Parameters\\n ----------\\n pred_box_tensor : torch.Tensor\\n (N, 8, 3) prediction.\\n\\n gt_box_tensor : torch.Tensor\\n (N, 8, 3) groundtruth bbx\\n\\n origin_lidar : torch.Tensor\\n PointCloud, (N, 4).\\n\\n o3d_pcd : open3d.PointCloud\\n Used to visualize the pcd.\\n\\n mode : str\\n lidar point rendering mode.\\n \\\"\\\"\\\"\\n\\n if not isinstance(origin_lidar, np.ndarray):\\n origin_lidar = common_utils.torch_tensor_to_numpy(origin_lidar)\\n # we only visualize the first cav for single sample\\n if len(origin_lidar.shape) > 2:\\n origin_lidar = origin_lidar[0]\\n origin_lidar_intcolor = \\\\\\n color_encoding(origin_lidar[:, -1] if mode == 'intensity'\\n else origin_lidar[:, 2], mode=mode)\\n\\n if not isinstance(pred_box_tensor, np.ndarray):\\n pred_box_tensor = common_utils.torch_tensor_to_numpy(pred_box_tensor)\\n if not isinstance(gt_box_tensor, np.ndarray):\\n gt_box_tensor = common_utils.torch_tensor_to_numpy(gt_box_tensor)\\n\\n # left -> right hand\\n origin_lidar[:, :1] = -origin_lidar[:, :1]\\n\\n o3d_pcd.points = o3d.utility.Vector3dVector(origin_lidar[:, :3])\\n o3d_pcd.colors = o3d.utility.Vector3dVector(origin_lidar_intcolor)\\n\\n gt_o3d_box = bbx2linset(gt_box_tensor, order='hwl', color=(0, 1, 0))\\n pred_o3d_box = bbx2linset(pred_box_tensor, color=(1, 0, 0))\\n\\n return o3d_pcd, pred_o3d_box, gt_o3d_box\\n\\n\\ndef visualize_sequence_dataloader(dataloader, order, color_mode='constant'):\\n \\\"\\\"\\\"\\n Visualize the batch data in animation.\\n\\n Parameters\\n ----------\\n dataloader : torch.Dataloader\\n Pytorch dataloader\\n\\n order : str\\n Bounding box order(N, 7).\\n\\n color_mode : str\\n Color rendering mode.\\n \\\"\\\"\\\"\\n vis = o3d.visualization.Visualizer()\\n vis.create_window()\\n\\n vis.get_render_option().background_color = [0.05, 0.05, 0.05]\\n vis.get_render_option().point_size = 1.0\\n vis.get_render_option().show_coordinate_frame = True\\n\\n # used to visualize lidar points\\n vis_pcd = o3d.geometry.PointCloud()\\n # used to visualize object bounding box, maximum 50\\n vis_aabbs = []\\n for _ in range(50):\\n vis_aabbs.append(o3d.geometry.LineSet())\\n\\n while True:\\n for i_batch, sample_batched in enumerate(dataloader):\\n print(i_batch)\\n pcd, aabbs = \\\\\\n visualize_single_sample_dataloader(sample_batched['ego'],\\n vis_pcd,\\n order,\\n mode=color_mode)\\n if i_batch == 0:\\n vis.add_geometry(pcd)\\n for i in range(len(vis_aabbs)):\\n index = i if i < len(aabbs) else -1\\n vis_aabbs[i] = lineset_assign(vis_aabbs[i], aabbs[index])\\n vis.add_geometry(vis_aabbs[i])\\n\\n for i in range(len(vis_aabbs)):\\n index = i if i < len(aabbs) else -1\\n vis_aabbs[i] = lineset_assign(vis_aabbs[i], aabbs[index])\\n vis.update_geometry(vis_aabbs[i])\\n\\n vis.update_geometry(pcd)\\n vis.poll_events()\\n vis.update_renderer()\\n time.sleep(0.001)\\n\\n vis.destroy_window()\\n\\n\\ndef save_o3d_visualization(element, save_path):\\n \\\"\\\"\\\"\\n Save the open3d drawing to folder.\\n\\n Parameters\\n ----------\\n element : list\\n List of o3d.geometry objects.\\n\\n save_path : str\\n The save path.\\n \\\"\\\"\\\"\\n vis = o3d.visualization.Visualizer()\\n vis.create_window()\\n for i in range(len(element)):\\n vis.add_geometry(element[i])\\n vis.update_geometry(element[i])\\n\\n vis.poll_events()\\n vis.update_renderer()\\n\\n vis.capture_screen_image(save_path)\\n vis.destroy_window()\\n\\n\\ndef visualize_bev(batch_data):\\n bev_input = batch_data[\\\"processed_lidar\\\"][\\\"bev_input\\\"]\\n label_map = batch_data[\\\"label_dict\\\"][\\\"label_map\\\"]\\n if not isinstance(bev_input, np.ndarray):\\n bev_input = common_utils.torch_tensor_to_numpy(bev_input)\\n\\n if not isinstance(label_map, np.ndarray):\\n label_map = label_map[0].numpy() if not label_map[0].is_cuda else \\\\\\n label_map[0].cpu().detach().numpy()\\n\\n if len(bev_input.shape) > 3:\\n bev_input = bev_input[0, ...]\\n\\n plt.matshow(np.sum(bev_input, axis=0))\\n plt.axis(\\\"off\\\")\\n plt.matshow(label_map[0, :, :])\\n plt.axis(\\\"off\\\")\\n plt.show()\\n\\n\\nimport os\\nimport argparse\\nfrom torch.utils.data import DataLoader\\n\\nfrom v2xvit.hypes_yaml.yaml_utils import load_yaml\\nfrom v2xvit.visualization import vis_utils\\nfrom v2xvit.data_utils.datasets.early_fusion_vis_dataset import \\\\\\n EarlyFusionVisDataset\\n\\n\\ndef vis_parser():\\n parser = argparse.ArgumentParser(description=\\\"data visualization\\\")\\n parser.add_argument('--color_mode', type=str, default=\\\"intensity\\\",\\n help='lidar color rendering mode, e.g. intensity,'\\n 'z-value or constant.')\\n opt = parser.parse_args()\\n return opt\\n\\n\\nif __name__ == '__main__':\\n current_path = os.path.dirname(os.path.realpath(__file__))\\n params = load_yaml(os.path.join(current_path,\\n '../hypes_yaml/visualization.yaml'))\\n\\n opencda_dataset = EarlyFusionVisDataset(params, visualize=True,\\n train=False)\\n data_loader = DataLoader(opencda_dataset, batch_size=1, num_workers=8,\\n collate_fn=opencda_dataset.collate_batch_train,\\n shuffle=False,\\n pin_memory=False)\\n\\n opt = vis_parser()\\n vis_utils.visualize_sequence_dataloader(data_loader,\\n params['postprocess']['order'],\\n color_mode=opt.color_mode)\\n\\n\\n\\n\\n\\n\\n\\\"\\\"\\\"\\nConvert lidar to bev\\n\\\"\\\"\\\"\\n\\nimport numpy as np\\nimport torch\\nfrom v2xvit.data_utils.pre_processor.base_preprocessor import \\\\\\n BasePreprocessor\\n\\nclass BevPreprocessor(BasePreprocessor):\\n def __init__(self, preprocess_params, train):\\n super(BevPreprocessor, self).__init__(preprocess_params, train)\\n self.lidar_range = self.params['cav_lidar_range']\\n self.geometry_param = preprocess_params[\\\"geometry_param\\\"]\\n\\n def preprocess(self, pcd_raw):\\n \\\"\\\"\\\"\\n Preprocess the lidar points to BEV representations.\\n\\n Parameters\\n ----------\\n pcd_raw : np.ndarray\\n The raw lidar.\\n\\n Returns\\n -------\\n data_dict : the structured output dictionary.\\n \\\"\\\"\\\"\\n bev = np.zeros(self.geometry_param['input_shape'], dtype=np.float32)\\n intensity_map_count = np.zeros((bev.shape[0], bev.shape[1]), dtype=np.int)\\n bev_origin = np.array(\\n [self.geometry_param[\\\"L1\\\"], self.geometry_param[\\\"W1\\\"],\\n self.geometry_param[\\\"H1\\\"]]).reshape(1, -1)\\n\\n indices = ((pcd_raw[:, :3] - bev_origin) / self.geometry_param[\\n \\\"res\\\"]).astype(int)\\n ## bev[indices[:, 0], indices[:, 1], indices[:, 2]] = 1\\n # np.add.at(bev, (indices[:, 0], indices[:, 1], indices[:, 2]), 1)\\n # bev[indices[:, 0], indices[:, 1], -1] += pcd_raw[:, 3]\\n # intensity_map_count[indices[:, 0], indices[:, 1]] += 1\\n\\n for i in range(indices.shape[0]):\\n bev[indices[i, 0], indices[i, 1], indices[i, 2]] = 1\\n bev[indices[i, 0], indices[i, 1], -1] += pcd_raw[i, 3]\\n intensity_map_count[indices[i, 0], indices[i, 1]] += 1\\n divide_mask = intensity_map_count!=0\\n bev[divide_mask, -1] = np.divide(bev[divide_mask, -1], intensity_map_count[divide_mask])\\n\\n data_dict = {\\n \\\"bev_input\\\": np.transpose(bev, (2, 0, 1))\\n }\\n return data_dict\\n\\n @staticmethod\\n def collate_batch_list(batch):\\n \\\"\\\"\\\"\\n Customized pytorch data loader collate function.\\n\\n Parameters\\n ----------\\n batch : list\\n List of dictionary. Each dictionary represent a single frame.\\n\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n bev_input_list = [\\n x[\\\"bev_input\\\"][np.newaxis, ...] for x in batch\\n ]\\n processed_batch = {\\n \\\"bev_input\\\": torch.from_numpy(\\n np.concatenate(bev_input_list, axis=0))\\n }\\n return processed_batch\\n @staticmethod\\n def collate_batch_dict(batch):\\n \\\"\\\"\\\"\\n Customized pytorch data loader collate function.\\n\\n Parameters\\n ----------\\n batch : dict\\n Dict of list. Each element represents a CAV.\\n\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n bev_input_list = [\\n x[np.newaxis, ...] for x in batch[\\\"bev_input\\\"]\\n ]\\n processed_batch = {\\n \\\"bev_input\\\": torch.from_numpy(\\n np.concatenate(bev_input_list, axis=0))\\n }\\n return processed_batch\\n\\n def collate_batch(self, batch):\\n \\\"\\\"\\\"\\n Customized pytorch data loader collate function.\\n\\n Parameters\\n ----------\\n batch : list / dict\\n Batched data.\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n if isinstance(batch, list):\\n return self.collate_batch_list(batch)\\n elif isinstance(batch, dict):\\n return self.collate_batch_dict(batch)\\n else:\\n raise NotImplemented\\n\\n\\n\\nimport numpy as np\\n\\nfrom v2xvit.utils import pcd_utils\\n\\n\\nclass BasePreprocessor(object):\\n \\\"\\\"\\\"\\n Basic Lidar pre-processor.\\n\\n Parameters\\n ----------\\n preprocess_params : dict\\n The dictionary containing all parameters of the preprocessing.\\n\\n train : bool\\n Train or test mode.\\n \\\"\\\"\\\"\\n\\n def __init__(self, preprocess_params, train):\\n self.params = preprocess_params\\n self.train = train\\n\\n def preprocess(self, pcd_np):\\n \\\"\\\"\\\"\\n Preprocess the lidar points by simple sampling.\\n\\n Parameters\\n ----------\\n pcd_np : np.ndarray\\n The raw lidar.\\n\\n Returns\\n -------\\n data_dict : the output dictionary.\\n \\\"\\\"\\\"\\n data_dict = {}\\n sample_num = self.params['args']['sample_num']\\n\\n pcd_np = pcd_utils.downsample_lidar(pcd_np, sample_num)\\n data_dict['downsample_lidar'] = pcd_np\\n\\n return data_dict\\n\\n def project_points_to_bev_map(self, points, ratio=0.1):\\n \\\"\\\"\\\"\\n Project points to BEV occupancy map with default ratio=0.1.\\n\\n Parameters\\n ----------\\n points : np.ndarray\\n (N, 3) / (N, 4)\\n\\n ratio : float\\n Discretization parameters. Default is 0.1.\\n\\n Returns\\n -------\\n bev_map : np.ndarray\\n BEV occupancy map including projected points with shape\\n (img_row, img_col).\\n\\n \\\"\\\"\\\"\\n L1, W1, H1, L2, W2, H2 = self.params[\\\"cav_lidar_range\\\"]\\n img_row = int((L2 - L1) / ratio)\\n img_col = int((W2 - W1) / ratio)\\n bev_map = np.zeros((img_row, img_col))\\n bev_origin = np.array([L1, W1, H1]).reshape(1, -1)\\n # (N, 3)\\n indices = ((points[:, :3] - bev_origin) / ratio).astype(int)\\n mask = np.logical_and(indices[:, 0] > 0, indices[:, 0] < img_row)\\n mask = np.logical_and(mask, np.logical_and(indices[:, 1] > 0,\\n indices[:, 1] < img_col))\\n indices = indices[mask, :]\\n bev_map[indices[:, 0], indices[:, 1]] = 1\\n return bev_map\\n\\n\\n\\\"\\\"\\\"\\nTransform points to voxels using sparse conv library\\n\\\"\\\"\\\"\\nimport sys\\n\\nimport numpy as np\\nimport torch\\nfrom cumm import tensorview as tv\\nfrom spconv.utils import Point2VoxelCPU3d\\n\\nfrom v2xvit.data_utils.pre_processor.base_preprocessor import \\\\\\n BasePreprocessor\\n\\n\\nclass SpVoxelPreprocessor(BasePreprocessor):\\n def __init__(self, preprocess_params, train):\\n super(SpVoxelPreprocessor, self).__init__(preprocess_params,\\n train)\\n\\n self.lidar_range = self.params['cav_lidar_range']\\n self.voxel_size = self.params['args']['voxel_size']\\n self.max_points_per_voxel = self.params['args']['max_points_per_voxel']\\n\\n if train:\\n self.max_voxels = self.params['args']['max_voxel_train']\\n else:\\n self.max_voxels = self.params['args']['max_voxel_test']\\n\\n grid_size = (np.array(self.lidar_range[3:6]) -\\n np.array(self.lidar_range[0:3])) / np.array(self.voxel_size)\\n self.grid_size = np.round(grid_size).astype(np.int64)\\n\\n # use sparse conv library to generate voxel\\n self.voxel_generator = Point2VoxelCPU3d(\\n vsize_xyz=self.voxel_size,\\n coors_range_xyz=self.lidar_range,\\n max_num_points_per_voxel=self.max_points_per_voxel,\\n num_point_features=4,\\n max_num_voxels=self.max_voxels\\n )\\n\\n def preprocess(self, pcd_np):\\n data_dict = {}\\n pcd_tv = tv.from_numpy(pcd_np)\\n voxel_output = self.voxel_generator.point_to_voxel(pcd_tv)\\n if isinstance(voxel_output, dict):\\n voxels, coordinates, num_points = \\\\\\n voxel_output['voxels'], voxel_output['coordinates'], \\\\\\n voxel_output['num_points_per_voxel']\\n else:\\n voxels, coordinates, num_points = voxel_output\\n\\n data_dict['voxel_features'] = voxels.numpy()\\n data_dict['voxel_coords'] = coordinates.numpy()\\n data_dict['voxel_num_points'] = num_points.numpy()\\n\\n return data_dict\\n\\n def collate_batch(self, batch):\\n \\\"\\\"\\\"\\n Customized pytorch data loader collate function.\\n\\n Parameters\\n ----------\\n batch : list or dict\\n List or dictionary.\\n\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n\\n if isinstance(batch, list):\\n return self.collate_batch_list(batch)\\n elif isinstance(batch, dict):\\n return self.collate_batch_dict(batch)\\n else:\\n sys.exit('Batch has too be a list or a dictionarn')\\n\\n @staticmethod\\n def collate_batch_list(batch):\\n \\\"\\\"\\\"\\n Customized pytorch data loader collate function.\\n\\n Parameters\\n ----------\\n batch : list\\n List of dictionary. Each dictionary represent a single frame.\\n\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n voxel_features = []\\n voxel_num_points = []\\n voxel_coords = []\\n\\n for i in range(len(batch)):\\n voxel_features.append(batch[i]['voxel_features'])\\n voxel_num_points.append(batch[i]['voxel_num_points'])\\n coords = batch[i]['voxel_coords']\\n voxel_coords.append(\\n np.pad(coords, ((0, 0), (1, 0)),\\n mode='constant', constant_values=i))\\n\\n voxel_num_points = torch.from_numpy(np.concatenate(voxel_num_points))\\n voxel_features = torch.from_numpy(np.concatenate(voxel_features))\\n voxel_coords = torch.from_numpy(np.concatenate(voxel_coords))\\n\\n return {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords,\\n 'voxel_num_points': voxel_num_points}\\n\\n @staticmethod\\n def collate_batch_dict(batch: dict):\\n \\\"\\\"\\\"\\n Collate batch if the batch is a dictionary,\\n eg: {'voxel_features': [feature1, feature2...., feature n]}\\n\\n Parameters\\n ----------\\n batch : dict\\n\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n voxel_features = \\\\\\n torch.from_numpy(np.concatenate(batch['voxel_features']))\\n voxel_num_points = \\\\\\n torch.from_numpy(np.concatenate(batch['voxel_num_points']))\\n coords = batch['voxel_coords']\\n voxel_coords = []\\n\\n for i in range(len(coords)):\\n voxel_coords.append(\\n np.pad(coords[i], ((0, 0), (1, 0)),\\n mode='constant', constant_values=i))\\n voxel_coords = torch.from_numpy(np.concatenate(voxel_coords))\\n\\n return {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords,\\n 'voxel_num_points': voxel_num_points}\\n\\n\\n\\\"\\\"\\\"\\nConvert lidar to voxel\\n\\\"\\\"\\\"\\nimport sys\\n\\nimport numpy as np\\nimport torch\\n\\nfrom v2xvit.data_utils.pre_processor.base_preprocessor import \\\\\\n BasePreprocessor\\n\\n\\nclass VoxelPreprocessor(BasePreprocessor):\\n def __init__(self, preprocess_params, train):\\n super(VoxelPreprocessor, self).__init__(preprocess_params, train)\\n self.lidar_range = self.params['cav_lidar_range']\\n\\n self.vw = self.params['args']['vw']\\n self.vh = self.params['args']['vh']\\n self.vd = self.params['args']['vd']\\n self.T = self.params['args']['T']\\n\\n def preprocess(self, pcd_np):\\n \\\"\\\"\\\"\\n Preprocess the lidar points by voxelization.\\n\\n Parameters\\n ----------\\n pcd_np : np.ndarray\\n The raw lidar.\\n\\n Returns\\n -------\\n data_dict : the structured output dictionary.\\n \\\"\\\"\\\"\\n data_dict = {}\\n\\n # calculate the voxel coordinates\\n voxel_coords = ((pcd_np[:, :3] -\\n np.floor(np.array([self.lidar_range[0],\\n self.lidar_range[1],\\n self.lidar_range[2]])) / (\\n self.vw, self.vh, self.vd))).astype(np.int32)\\n\\n # convert to (D, H, W) as the paper\\n voxel_coords = voxel_coords[:, [2, 1, 0]]\\n voxel_coords, inv_ind, voxel_counts = np.unique(voxel_coords, axis=0,\\n return_inverse=True,\\n return_counts=True)\\n\\n voxel_features = []\\n\\n for i in range(len(voxel_coords)):\\n voxel = np.zeros((self.T, 7), dtype=np.float32)\\n pts = pcd_np[inv_ind == i]\\n if voxel_counts[i] > self.T:\\n pts = pts[:self.T, :]\\n voxel_counts[i] = self.T\\n\\n # augment the points\\n voxel[:pts.shape[0], :] = np.concatenate((pts, pts[:, :3] -\\n np.mean(pts[:, :3], 0)),\\n axis=1)\\n voxel_features.append(voxel)\\n\\n data_dict['voxel_features'] = np.array(voxel_features)\\n data_dict['voxel_coords'] = voxel_coords\\n\\n return data_dict\\n\\n def collate_batch(self, batch):\\n \\\"\\\"\\\"\\n Customized pytorch data loader collate function.\\n\\n Parameters\\n ----------\\n batch : list or dict\\n List or dictionary.\\n\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n\\n if isinstance(batch, list):\\n return self.collate_batch_list(batch)\\n elif isinstance(batch, dict):\\n return self.collate_batch_dict(batch)\\n else:\\n sys.exit('Batch has too be a list or a dictionarn')\\n\\n @staticmethod\\n def collate_batch_list(batch):\\n \\\"\\\"\\\"\\n Customized pytorch data loader collate function.\\n\\n Parameters\\n ----------\\n batch : list\\n List of dictionary. Each dictionary represent a single frame.\\n\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n voxel_features = []\\n voxel_coords = []\\n\\n for i in range(len(batch)):\\n voxel_features.append(batch[i]['voxel_features'])\\n coords = batch[i]['voxel_coords']\\n voxel_coords.append(\\n np.pad(coords, ((0, 0), (1, 0)),\\n mode='constant', constant_values=i))\\n\\n voxel_features = torch.from_numpy(np.concatenate(voxel_features))\\n voxel_coords = torch.from_numpy(np.concatenate(voxel_coords))\\n\\n return {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords}\\n\\n @staticmethod\\n def collate_batch_dict(batch: dict):\\n \\\"\\\"\\\"\\n Collate batch if the batch is a dictionary,\\n eg: {'voxel_features': [feature1, feature2...., feature n]}\\n\\n Parameters\\n ----------\\n batch : dict\\n\\n Returns\\n -------\\n processed_batch : dict\\n Updated lidar batch.\\n \\\"\\\"\\\"\\n voxel_features = \\\\\\n torch.from_numpy(np.concatenate(batch['voxel_features']))\\n coords = batch['voxel_coords']\\n voxel_coords = []\\n\\n for i in range(len(coords)):\\n voxel_coords.append(\\n np.pad(coords[i], ((0, 0), (1, 0)),\\n mode='constant', constant_values=i))\\n voxel_coords = torch.from_numpy(np.concatenate(voxel_coords))\\n\\n return {'voxel_features': voxel_features,\\n 'voxel_coords': voxel_coords}\\n\\n\\nfrom v2xvit.data_utils.pre_processor.base_preprocessor import BasePreprocessor\\nfrom v2xvit.data_utils.pre_processor.voxel_preprocessor import VoxelPreprocessor\\nfrom v2xvit.data_utils.pre_processor.bev_preprocessor import BevPreprocessor\\nfrom v2xvit.data_utils.pre_processor.sp_voxel_preprocessor import SpVoxelPreprocessor\\n\\n__all__ = {\\n 'BasePreprocessor': BasePreprocessor,\\n 'VoxelPreprocessor': VoxelPreprocessor,\\n 'BevPreprocessor': BevPreprocessor,\\n 'SpVoxelPreprocessor': SpVoxelPreprocessor\\n}\\n\\n\\ndef build_preprocessor(preprocess_cfg, train):\\n process_method_name = preprocess_cfg['core_method']\\n error_message = f\\\"{process_method_name} is not found. \\\" \\\\\\n f\\\"Please add your processor file's name in opencood/\\\" \\\\\\n f\\\"data_utils/processor/init.py\\\"\\n assert process_method_name in ['BasePreprocessor', 'VoxelPreprocessor',\\n 'BevPreprocessor', 'SpVoxelPreprocessor'], \\\\\\n error_message\\n\\n processor = __all__[process_method_name](\\n preprocess_params=preprocess_cfg,\\n train=train\\n )\\n\\n return processor\\n\\n\\n\\\"\\\"\\\"\\nTemplate for AnchorGenerator\\n\\\"\\\"\\\"\\n\\nimport numpy as np\\nimport torch\\n\\nfrom v2xvit.utils import box_utils\\n\\n\\nclass BasePostprocessor(object):\\n \\\"\\\"\\\"\\n Template for Anchor generator.\\n\\n Parameters\\n ----------\\n anchor_params : dict\\n The dictionary containing all anchor-related parameters.\\n train : bool\\n Indicate train or test mode.\\n\\n Attributes\\n ----------\\n bbx_dict : dictionary\\n Contain all objects information across the cav, key: id, value: bbx\\n coordinates (1, 7)\\n \\\"\\\"\\\"\\n\\n def __init__(self, anchor_params, train=True):\\n self.params = anchor_params\\n self.bbx_dict = {}\\n self.train = train\\n\\n def generate_anchor_box(self):\\n # needs to be overloaded\\n return None\\n\\n def generate_label(self, *argv):\\n return None\\n\\n def generate_gt_bbx(self, data_dict):\\n \\\"\\\"\\\"\\n The base postprocessor will generate 3d groundtruth bounding box.\\n\\n Parameters\\n ----------\\n data_dict : dict\\n The dictionary containing the origin input data of model.\\n\\n Returns\\n -------\\n gt_box3d_tensor : torch.Tensor\\n The groundtruth bounding box tensor, shape (N, 8, 3).\\n \\\"\\\"\\\"\\n gt_box3d_list = []\\n # used to avoid repetitive bounding box\\n object_id_list = []\\n\\n for cav_id, cav_content in data_dict.items():\\n # used to project gt bounding box to ego space.\\n # the transformation matrix for gt should always be based on\\n # current timestamp (object transformation matrix is for\\n # late fusion only since other fusion method already did\\n # the transformation in the preprocess)\\n transformation_matrix = cav_content['transformation_matrix'] \\\\\\n if 'gt_transformation_matrix' not in cav_content \\\\\\n else cav_content['gt_transformation_matrix']\\n\\n object_bbx_center = cav_content['object_bbx_center']\\n object_bbx_mask = cav_content['object_bbx_mask']\\n object_ids = cav_content['object_ids']\\n object_bbx_center = object_bbx_center[object_bbx_mask == 1]\\n\\n # convert center to corner\\n object_bbx_corner = \\\\\\n box_utils.boxes_to_corners_3d(object_bbx_center,\\n self.params['order'])\\n projected_object_bbx_corner = \\\\\\n box_utils.project_box3d(object_bbx_corner.float(),\\n transformation_matrix)\\n gt_box3d_list.append(projected_object_bbx_corner)\\n\\n # append the corresponding ids\\n object_id_list += object_ids\\n\\n # gt bbx 3d\\n gt_box3d_list = torch.vstack(gt_box3d_list)\\n # some of the bbx may be repetitive, use the id list to filter\\n gt_box3d_selected_indices = \\\\\\n [object_id_list.index(x) for x in set(object_id_list)]\\n gt_box3d_tensor = gt_box3d_list[gt_box3d_selected_indices]\\n\\n # filter the gt_box to make sure all bbx are in the range\\n mask = \\\\\\n box_utils.get_mask_for_boxes_within_range_torch(gt_box3d_tensor)\\n gt_box3d_tensor = gt_box3d_tensor[mask, :, :]\\n\\n return gt_box3d_tensor\\n\\n def generate_object_center(self,\\n cav_contents,\\n reference_lidar_pose):\\n \\\"\\\"\\\"\\n Retrieve all objects in a format of (n, 7), where 7 represents\\n x, y, z, l, w, h, yaw or x, y, z, h, w, l, yaw.\\n\\n Parameters\\n ----------\\n cav_contents : list\\n List of dictionary, save all cavs' information.\\n\\n reference_lidar_pose : list\\n The final target lidar pose with length 6.\\n\\n Returns\\n -------\\n object_np : np.ndarray\\n Shape is (max_num, 7).\\n mask : np.ndarray\\n Shape is (max_num,).\\n object_ids : list\\n Length is number of bbx in current sample.\\n \\\"\\\"\\\"\\n from v2xvit.data_utils.datasets import GT_RANGE\\n\\n tmp_object_dict = {}\\n for cav_content in cav_contents:\\n tmp_object_dict.update(cav_content['params']['vehicles'])\\n\\n output_dict = {}\\n filter_range = self.params['anchor_args']['cav_lidar_range'] \\\\\\n if self.train else GT_RANGE\\n\\n box_utils.project_world_objects(tmp_object_dict,\\n output_dict,\\n reference_lidar_pose,\\n filter_range,\\n self.params['order'])\\n\\n object_np = np.zeros((self.params['max_num'], 7))\\n mask = np.zeros(self.params['max_num'])\\n object_ids = []\\n\\n for i, (object_id, object_bbx) in enumerate(output_dict.items()):\\n object_np[i] = object_bbx[0, :]\\n mask[i] = 1\\n object_ids.append(object_id)\\n\\n return object_np, mask, object_ids\\n\\n\\n\\\"\\\"\\\"\\n3D Anchor Generator for Voxel\\n\\\"\\\"\\\"\\nimport math\\nimport sys\\n\\nimport numpy as np\\nimport torch\\nimport torch.nn.functional as F\\n\\nfrom v2xvit.data_utils.post_processor.base_postprocessor \\\\\\n import BasePostprocessor\\nfrom v2xvit.utils import box_utils\\nfrom v2xvit.utils.box_overlaps import bbox_overlaps\\nfrom v2xvit.visualization import vis_utils\\n\\n\\nclass VoxelPostprocessor(BasePostprocessor):\\n def __init__(self, anchor_params, train):\\n super(VoxelPostprocessor, self).__init__(anchor_params, train)\\n self.anchor_num = self.params['anchor_args']['num']\\n\\n def generate_anchor_box(self):\\n W = self.params['anchor_args']['W']\\n H = self.params['anchor_args']['H']\\n\\n l = self.params['anchor_args']['l']\\n w = self.params['anchor_args']['w']\\n h = self.params['anchor_args']['h']\\n r = self.params['anchor_args']['r']\\n\\n assert self.anchor_num == len(r)\\n r = [math.radians(ele) for ele in r]\\n\\n vh = self.params['anchor_args']['vh']\\n vw = self.params['anchor_args']['vw']\\n\\n xrange = [self.params['anchor_args']['cav_lidar_range'][0],\\n self.params['anchor_args']['cav_lidar_range'][3]]\\n yrange = [self.params['anchor_args']['cav_lidar_range'][1],\\n self.params['anchor_args']['cav_lidar_range'][4]]\\n\\n if 'feature_stride' in self.params['anchor_args']:\\n feature_stride = self.params['anchor_args']['feature_stride']\\n else:\\n feature_stride = 2\\n\\n x = np.linspace(xrange[0] + vw, xrange[1] - vw, W // feature_stride)\\n y = np.linspace(yrange[0] + vh, yrange[1] - vh, H // feature_stride)\\n\\n cx, cy = np.meshgrid(x, y)\\n cx = np.tile(cx[..., np.newaxis], self.anchor_num)\\n cy = np.tile(cy[..., np.newaxis], self.anchor_num)\\n cz = np.ones_like(cx) * -1.0\\n\\n w = np.ones_like(cx) * w\\n l = np.ones_like(cx) * l\\n h = np.ones_like(cx) * h\\n\\n r_ = np.ones_like(cx)\\n for i in range(self.anchor_num):\\n r_[..., i] = r[i]\\n\\n if self.params['order'] == 'hwl':\\n anchors = np.stack([cx, cy, cz, h, w, l, r_], axis=-1)\\n elif self.params['order'] == 'lhw':\\n anchors = np.stack([cx, cy, cz, l, h, w, r_], axis=-1)\\n else:\\n sys.exit('Unknown bbx order.')\\n\\n return anchors\\n\\n def generate_label(self, **kwargs):\\n \\\"\\\"\\\"\\n Generate targets for training.\\n\\n Parameters\\n ----------\\n argv : list\\n gt_box_center:(max_num, 7), anchor:(H, W, anchor_num, 7)\\n\\n Returns\\n -------\\n label_dict : dict\\n Dictionary that contains all target related info.\\n \\\"\\\"\\\"\\n assert self.params['order'] == 'hwl', 'Currently Voxel only support' \\\\\\n 'hwl bbx order.'\\n # (max_num, 7)\\n gt_box_center = kwargs['gt_box_center']\\n # (H, W, anchor_num, 7)\\n anchors = kwargs['anchors']\\n # (max_num)\\n masks = kwargs['mask']\\n\\n # (H, W)\\n feature_map_shape = anchors.shape[:2]\\n\\n # (H*W*anchor_num, 7)\\n anchors = anchors.reshape(-1, 7)\\n # normalization factor, (H * W * anchor_num)\\n anchors_d = np.sqrt(anchors[:, 4] ** 2 + anchors[:, 5] ** 2)\\n\\n # (H, W, 2)\\n pos_equal_one = np.zeros((*feature_map_shape, self.anchor_num))\\n neg_equal_one = np.zeros((*feature_map_shape, self.anchor_num))\\n # (H, W, self.anchor_num * 7)\\n targets = np.zeros((*feature_map_shape, self.anchor_num * 7))\\n\\n # (n, 7)\\n gt_box_center_valid = gt_box_center[masks == 1]\\n # (n, 8, 3)\\n gt_box_corner_valid = \\\\\\n box_utils.boxes_to_corners_3d(gt_box_center_valid,\\n self.params['order'])\\n # (H*W*anchor_num, 8, 3)\\n anchors_corner = \\\\\\n box_utils.boxes_to_corners_3d(anchors,\\n order=self.params['order'])\\n # (H*W*anchor_num, 4)\\n anchors_standup_2d = \\\\\\n box_utils.corner2d_to_standup_box(anchors_corner)\\n # (n, 4)\\n gt_standup_2d = \\\\\\n box_utils.corner2d_to_standup_box(gt_box_corner_valid)\\n\\n # (H*W*anchor_n)\\n iou = bbox_overlaps(\\n np.ascontiguousarray(anchors_standup_2d).astype(np.float32),\\n np.ascontiguousarray(gt_standup_2d).astype(np.float32),\\n )\\n\\n # the anchor boxes has the largest iou across\\n # shape: (n)\\n id_highest = np.argmax(iou.T, axis=1)\\n # [0, 1, 2, ..., n-1]\\n id_highest_gt = np.arange(iou.T.shape[0])\\n # make sure all highest iou is larger than 0\\n mask = iou.T[id_highest_gt, id_highest] > 0\\n id_highest, id_highest_gt = id_highest[mask], id_highest_gt[mask]\\n\\n # find anchors iou > params['pos_iou']\\n id_pos, id_pos_gt = \\\\\\n np.where(iou >\\n self.params['target_args']['pos_threshold'])\\n # find anchors iou params['neg_iou']\\n id_neg = np.where(np.sum(iou <\\n self.params['target_args']['neg_threshold'],\\n axis=1) == iou.shape[1])[0]\\n id_pos = np.concatenate([id_pos, id_highest])\\n id_pos_gt = np.concatenate([id_pos_gt, id_highest_gt])\\n id_pos, index = np.unique(id_pos, return_index=True)\\n id_pos_gt = id_pos_gt[index]\\n id_neg.sort()\\n\\n # cal the target and set the equal one\\n index_x, index_y, index_z = np.unravel_index(\\n id_pos, (*feature_map_shape, self.anchor_num))\\n pos_equal_one[index_x, index_y, index_z] = 1\\n\\n # calculate the targets\\n targets[index_x, index_y, np.array(index_z) * 7] = \\\\\\n (gt_box_center[id_pos_gt, 0] - anchors[id_pos, 0]) / anchors_d[\\n id_pos]\\n targets[index_x, index_y, np.array(index_z) * 7 + 1] = \\\\\\n (gt_box_center[id_pos_gt, 1] - anchors[id_pos, 1]) / anchors_d[\\n id_pos]\\n targets[index_x, index_y, np.array(index_z) * 7 + 2] = \\\\\\n (gt_box_center[id_pos_gt, 2] - anchors[id_pos, 2]) / anchors[\\n id_pos, 3]\\n targets[index_x, index_y, np.array(index_z) * 7 + 3] = np.log(\\n gt_box_center[id_pos_gt, 3] / anchors[id_pos, 3])\\n targets[index_x, index_y, np.array(index_z) * 7 + 4] = np.log(\\n gt_box_center[id_pos_gt, 4] / anchors[id_pos, 4])\\n targets[index_x, index_y, np.array(index_z) * 7 + 5] = np.log(\\n gt_box_center[id_pos_gt, 5] / anchors[id_pos, 5])\\n targets[index_x, index_y, np.array(index_z) * 7 + 6] = (\\n gt_box_center[id_pos_gt, 6] - anchors[id_pos, 6])\\n\\n index_x, index_y, index_z = np.unravel_index(\\n id_neg, (*feature_map_shape, self.anchor_num))\\n neg_equal_one[index_x, index_y, index_z] = 1\\n\\n # to avoid a box be pos/neg in the same time\\n index_x, index_y, index_z = np.unravel_index(\\n id_highest, (*feature_map_shape, self.anchor_num))\\n neg_equal_one[index_x, index_y, index_z] = 0\\n\\n label_dict = {'pos_equal_one': pos_equal_one,\\n 'neg_equal_one': neg_equal_one,\\n 'targets': targets}\\n\\n return label_dict\\n\\n @staticmethod\\n def collate_batch(label_batch_list):\\n \\\"\\\"\\\"\\n Customized collate function for target label generation.\\n\\n Parameters\\n ----------\\n label_batch_list : list\\n The list of dictionary that contains all labels for several\\n frames.\\n\\n Returns\\n -------\\n target_batch : dict\\n Reformatted labels in torch tensor.\\n \\\"\\\"\\\"\\n pos_equal_one = []\\n neg_equal_one = []\\n targets = []\\n\\n for i in range(len(label_batch_list)):\\n pos_equal_one.append(label_batch_list[i]['pos_equal_one'])\\n neg_equal_one.append(label_batch_list[i]['neg_equal_one'])\\n targets.append(label_batch_list[i]['targets'])\\n\\n pos_equal_one = \\\\\\n torch.from_numpy(np.array(pos_equal_one))\\n neg_equal_one = \\\\\\n torch.from_numpy(np.array(neg_equal_one))\\n targets = \\\\\\n torch.from_numpy(np.array(targets))\\n\\n return {'targets': targets,\\n 'pos_equal_one': pos_equal_one,\\n 'neg_equal_one': neg_equal_one}\\n\\n def post_process(self, data_dict, output_dict):\\n \\\"\\\"\\\"\\n Process the outputs of the model to 2D/3D bounding box.\\n Step1: convert each cav's output to bounding box format\\n Step2: project the bounding boxes to ego space.\\n Step:3 NMS\\n\\n Parameters\\n ----------\\n data_dict : dict\\n The dictionary containing the origin input data of model.\\n\\n output_dict :dict\\n The dictionary containing the output of the model.\\n\\n Returns\\n -------\\n pred_box3d_tensor : torch.Tensor\\n The prediction bounding box tensor after NMS.\\n gt_box3d_tensor : torch.Tensor\\n The groundtruth bounding box tensor.\\n \\\"\\\"\\\"\\n # the final bounding box list\\n pred_box3d_list = []\\n pred_box2d_list = []\\n\\n for cav_id, cav_content in data_dict.items():\\n assert cav_id in output_dict\\n # the transformation matrix to ego space\\n transformation_matrix = cav_content['transformation_matrix']\\n\\n # (H, W, anchor_num, 7)\\n anchor_box = cav_content['anchor_box']\\n\\n # classification probability\\n prob = output_dict[cav_id]['psm']\\n prob = F.sigmoid(prob.permute(0, 2, 3, 1))\\n prob = prob.reshape(1, -1)\\n\\n # regression map\\n reg = output_dict[cav_id]['rm']\\n\\n # convert regression map back to bounding box\\n # (N, W*L*anchor_num, 7)\\n batch_box3d = self.delta_to_boxes3d(reg, anchor_box)\\n mask = \\\\\\n torch.gt(prob, self.params['target_args']['score_threshold'])\\n mask = mask.view(1, -1)\\n mask_reg = mask.unsqueeze(2).repeat(1, 1, 7)\\n\\n # during validation/testing, the batch size should be 1\\n assert batch_box3d.shape[0] == 1\\n boxes3d = torch.masked_select(batch_box3d[0],\\n mask_reg[0]).view(-1, 7)\\n scores = torch.masked_select(prob[0], mask[0])\\n\\n # convert output to bounding box\\n if len(boxes3d) != 0:\\n # (N, 8, 3)\\n boxes3d_corner = \\\\\\n box_utils.boxes_to_corners_3d(boxes3d,\\n order=self.params['order'])\\n # (N, 8, 3)\\n projected_boxes3d = \\\\\\n box_utils.project_box3d(boxes3d_corner,\\n transformation_matrix)\\n # convert 3d bbx to 2d, (N,4)\\n projected_boxes2d = \\\\\\n box_utils.corner_to_standup_box_torch(projected_boxes3d)\\n # (N, 5)\\n boxes2d_score = \\\\\\n torch.cat((projected_boxes2d, scores.unsqueeze(1)), dim=1)\\n\\n pred_box2d_list.append(boxes2d_score)\\n pred_box3d_list.append(projected_boxes3d)\\n\\n if len(pred_box2d_list) ==0 or len(pred_box3d_list) == 0:\\n return None, None\\n # shape: (N, 5)\\n pred_box2d_list = torch.vstack(pred_box2d_list)\\n # scores\\n scores = pred_box2d_list[:, -1]\\n # predicted 3d bbx\\n pred_box3d_tensor = torch.vstack(pred_box3d_list)\\n # remove large bbx\\n keep_index_1 = box_utils.remove_large_pred_bbx(pred_box3d_tensor)\\n keep_index_2 = box_utils.remove_bbx_abnormal_z(pred_box3d_tensor)\\n keep_index = torch.logical_and(keep_index_1, keep_index_2)\\n\\n pred_box3d_tensor = pred_box3d_tensor[keep_index]\\n scores = scores[keep_index]\\n\\n # nms\\n keep_index = box_utils.nms_rotated(pred_box3d_tensor,\\n scores,\\n self.params['nms_thresh']\\n )\\n\\n pred_box3d_tensor = pred_box3d_tensor[keep_index]\\n\\n # select cooresponding score\\n scores = scores[keep_index]\\n\\n # filter out the prediction out of the range.\\n mask = \\\\\\n box_utils.get_mask_for_boxes_within_range_torch(pred_box3d_tensor)\\n pred_box3d_tensor = pred_box3d_tensor[mask, :, :]\\n scores = scores[mask]\\n\\n assert scores.shape[0] == pred_box3d_tensor.shape[0]\\n\\n return pred_box3d_tensor, scores\\n\\n @staticmethod\\n def delta_to_boxes3d(deltas, anchors):\\n \\\"\\\"\\\"\\n Convert the output delta to 3d bbx.\\n\\n Parameters\\n ----------\\n deltas : torch.Tensor\\n (N, W, L, 14)\\n anchors : torch.Tensor\\n (W, L, 2, 7) -> xyzhwlr\\n\\n Returns\\n -------\\n box3d : torch.Tensor\\n (N, W*L*2, 7)\\n \\\"\\\"\\\"\\n # batch size\\n N = deltas.shape[0]\\n deltas = deltas.permute(0, 2, 3, 1).contiguous().view(N, -1, 7)\\n boxes3d = torch.zeros_like(deltas)\\n\\n if deltas.is_cuda:\\n anchors = anchors.cuda()\\n boxes3d = boxes3d.cuda()\\n\\n # (W*L*2, 7)\\n anchors_reshaped = anchors.view(-1, 7).float()\\n # the diagonal of the anchor 2d box, (W*L*2)\\n anchors_d = torch.sqrt(\\n anchors_reshaped[:, 4] ** 2 + anchors_reshaped[:, 5] ** 2)\\n anchors_d = anchors_d.repeat(N, 2, 1).transpose(1, 2)\\n anchors_reshaped = anchors_reshaped.repeat(N, 1, 1)\\n\\n # Inv-normalize to get xyz\\n boxes3d[..., [0, 1]] = torch.mul(deltas[..., [0, 1]], anchors_d) + \\\\\\n anchors_reshaped[..., [0, 1]]\\n boxes3d[..., [2]] = torch.mul(deltas[..., [2]],\\n anchors_reshaped[..., [3]]) + \\\\\\n anchors_reshaped[..., [2]]\\n # hwl\\n boxes3d[..., [3, 4, 5]] = torch.exp(\\n deltas[..., [3, 4, 5]]) * anchors_reshaped[..., [3, 4, 5]]\\n # yaw angle\\n boxes3d[..., 6] = deltas[..., 6] + anchors_reshaped[..., 6]\\n\\n return boxes3d\\n\\n @staticmethod\\n def visualize(pred_box_tensor, gt_tensor, pcd, show_vis, save_path, dataset=None):\\n \\\"\\\"\\\"\\n Visualize the prediction, ground truth with point cloud together.\\n\\n Parameters\\n ----------\\n pred_box_tensor : torch.Tensor\\n (N, 8, 3) prediction.\\n\\n gt_tensor : torch.Tensor\\n (N, 8, 3) groundtruth bbx\\n\\n pcd : torch.Tensor\\n PointCloud, (N, 4).\\n\\n show_vis : bool\\n Whether to show visualization.\\n\\n save_path : str\\n Save the visualization results to given path.\\n\\n dataset : BaseDataset\\n opencood dataset object.\\n\\n \\\"\\\"\\\"\\n vis_utils.visualize_single_sample_output_gt(pred_box_tensor,\\n gt_tensor,\\n pcd,\\n show_vis,\\n save_path)\\n\\n\\n\\\"\\\"\\\"\\nAnchor-free 2d Generator\\n\\\"\\\"\\\"\\n\\nimport numpy as np\\nimport torch\\nimport torch.nn.functional as F\\n\\nfrom v2xvit.utils.transformation_utils import dist_to_continuous\\nfrom v2xvit.data_utils.post_processor.base_postprocessor \\\\\\n import BasePostprocessor\\nfrom v2xvit.utils import box_utils\\nfrom v2xvit.visualization import vis_utils\\n\\n\\nclass BevPostprocessor(BasePostprocessor):\\n def __init__(self, anchor_params, train):\\n super(BevPostprocessor, self).__init__(anchor_params, train)\\n # self.geometry_param = anchor_params[\\\"geometry\\\"]\\n self.geometry_param = anchor_params[\\\"geometry_param\\\"]\\n\\n # TODO\\n # Hard coded for now. Need to calculate for our own training dataset\\n self.target_mean = np.array([0.008, 0.001, 0.202, 0.2, 0.43, 1.368])\\n self.target_std_dev = np.array([0.866, 0.5, 0.954, 0.668, 0.09, 0.111])\\n\\n def generate_anchor_box(self):\\n return None\\n\\n def generate_label(self, **kwargs):\\n \\\"\\\"\\\"\\n Generate targets for training.\\n\\n Parameters\\n ----------\\n kwargs : list\\n gt_box_center:(max_num, 7)\\n\\n Returns\\n -------\\n label_dict : dict\\n Dictionary that contains all target related info.\\n \\\"\\\"\\\"\\n assert self.params['order'] == 'lwh', \\\\\\n 'Currently BEV only support lwh bbx order.'\\n # (max_num, 7)\\n gt_box_center = kwargs['gt_box_center']\\n\\n # (max_num)\\n masks = kwargs['mask']\\n\\n # (n, 7)\\n gt_box_center_valid = gt_box_center[masks == 1]\\n # (n, 4, 3)\\n bev_corners = box_utils.boxes_to_corners2d(gt_box_center_valid,\\n self.params['order'])\\n\\n n = gt_box_center_valid.shape[0]\\n # (n, 4, 2)\\n bev_corners = bev_corners[:, :, :2]\\n yaw = gt_box_center_valid[:, -1]\\n x, y = gt_box_center_valid[:, 0], gt_box_center_valid[:, 1]\\n dx, dy = gt_box_center_valid[:, 3], gt_box_center_valid[:, 4]\\n # (n, 6)\\n reg_targets = np.column_stack([np.cos(yaw), np.sin(yaw), x, y, dx, dy])\\n\\n # target label map including classification and regression targets\\n # shape -- (label_shape[0], label_shape[1], 7)\\n # (binary, cos(yaw), sin(yaw), displacement_x, displacement_y, log(dx), log(dy))\\n label_map = np.zeros(self.geometry_param[\\\"label_shape\\\"])\\n self.update_label_map(label_map, bev_corners, reg_targets)\\n label_map = self.normalize_targets(label_map)\\n label_dict = {\\n # (7, label_shape[0], label_shape[1])\\n \\\"label_map\\\": np.transpose(label_map, (2, 0, 1)).astype(np.float32),\\n \\\"bev_corners\\\": bev_corners\\n }\\n return label_dict\\n\\n def update_label_map(self, label_map, bev_corners, reg_targets):\\n \\\"\\\"\\\"\\n Update label_map based on bbx and regression targets.\\n\\n Parameters\\n ----------\\n label_map : numpy.array\\n Targets array for classification and regression tasks with\\n the shape of label_shape.\\n\\n bev_corners : numpy.array\\n The bbx corners in lidar frame with shape (n, 4, 2)\\n\\n reg_targets : numpy.array\\n Array containing the regression targets information. It need to be\\n further processed.\\n\\n \\\"\\\"\\\"\\n res = self.geometry_param[\\\"res\\\"]\\n downsample_rate = self.geometry_param[\\\"downsample_rate\\\"]\\n\\n bev_origin = np.array([self.geometry_param[\\\"L1\\\"],\\n self.geometry_param[\\\"W1\\\"]]).reshape(1, -1)\\n\\n # discretized bbx corner representations -- (n, 4, 2)\\n bev_corners_dist = (bev_corners - bev_origin) / res / downsample_rate\\n # generate the coordinates of m\\n x = np.arange(self.geometry_param[\\\"label_shape\\\"][0])\\n y = np.arange(self.geometry_param[\\\"label_shape\\\"][1])\\n xx, yy = np.meshgrid(x, y)\\n\\n # (label_shape[0]*label_shape[1], 2)\\n points = np.concatenate([xx.reshape(-1, 1), yy.reshape(-1, 1)], axis=-1)\\n bev_origin_dist = bev_origin / res / downsample_rate\\n\\n # loop over each bbx, find the points within the bbx.\\n for i in range(bev_corners.shape[0]):\\n reg_target = reg_targets[i, :]\\n\\n # find discredited points in bbx\\n points_in_box = \\\\\\n box_utils.get_points_in_rotated_box(points,\\n bev_corners_dist[i, ...])\\n # convert points to continuous space\\n points_continuous = dist_to_continuous(points_in_box,\\n bev_origin_dist,\\n res,\\n downsample_rate)\\n actual_reg_target = np.repeat(reg_target.reshape(1, -1),\\n points_continuous.shape[0],\\n axis=0)\\n # build learning targets\\n actual_reg_target[:, 2:4] = \\\\\\n actual_reg_target[:, 2:4] - points_continuous\\n actual_reg_target[:, 4:] = np.log(actual_reg_target[:, 4:])\\n\\n # update label map\\n label_map[points_in_box[:, 0], points_in_box[:, 1], 0] = 1.0\\n label_map[points_in_box[:, 0], points_in_box[:, 1], 1:] = \\\\\\n actual_reg_target\\n\\n def normalize_targets(self, label_map):\\n \\\"\\\"\\\"\\n Normalize label_map\\n\\n Parameters\\n ----------\\n label_map : numpy.array\\n Targets array for classification and regression tasks with the\\n shape of label_shape.\\n\\n Returns\\n -------\\n label_map: numpy.array\\n Nromalized label_map.\\n\\n \\\"\\\"\\\"\\n label_map[..., 1:] = \\\\\\n (label_map[..., 1:] - self.target_mean) / self.target_std_dev\\n return label_map\\n\\n def denormalize_reg_map(self, reg_map):\\n \\\"\\\"\\\"\\n Denormalize the regression map\\n\\n Parameters\\n ----------\\n reg_map : np.ndarray / torch.Tensor\\n Regression output mapwith the shape of (label_shape[0],\\n label_shape[1], 6).\\n\\n Returns\\n -------\\n reg_map : np.ndarray / torch.Tensor\\n Denormalized regression map.\\n\\n \\\"\\\"\\\"\\n if isinstance(reg_map, np.ndarray):\\n target_mean = self.target_mean\\n target_std_dev = self.target_std_dev\\n\\n else:\\n target_mean = \\\\\\n torch.from_numpy(self.target_mean).to(reg_map.device)\\n target_std_dev =\\\\\\n torch.from_numpy(self.target_std_dev).to(reg_map.device)\\n reg_map = reg_map * target_std_dev + target_mean\\n return reg_map\\n\\n @staticmethod\\n def collate_batch(label_batch_list):\\n \\\"\\\"\\\"\\n Customized collate function for target label generation.\\n\\n Parameters\\n ----------\\n label_batch_list : list\\n The list of dictionary that contains all labels for several\\n frames.\\n\\n Returns\\n -------\\n processed_batch : dict\\n Reformatted labels in torch tensor.\\n \\\"\\\"\\\"\\n label_map_list = [x[\\\"label_map\\\"][np.newaxis, ...] for x in\\n label_batch_list]\\n processed_batch = {\\n # (batch_size, 7, label_shape[0], label_shape[1])\\n \\\"label_map\\\": torch.from_numpy(np.concatenate(label_map_list,\\n axis=0)),\\n \\\"bev_corners\\\": [torch.from_numpy(x[\\\"bev_corners\\\"]) for x in\\n label_batch_list]\\n }\\n return processed_batch\\n\\n def post_process(self, data_dict, output_dict):\\n \\\"\\\"\\\"\\n Process the outputs of the model to 2D bounding box.\\n Step1: convert each cav's output to bounding box format\\n Step2: project the bounding boxes to ego space.\\n Step:3 NMS\\n\\n Parameters\\n ----------\\n data_dict : dict\\n The dictionary containing the origin input data of model.\\n\\n output_dict :dict\\n The dictionary containing the output of the model.\\n\\n Returns\\n -------\\n pred_box2d_tensor : torch.Tensor\\n The prediction bounding box tensor after NMS.\\n\\n gt_box2d_tensor : torch.Tensor\\n The groundtruth bounding box tensor.\\n \\\"\\\"\\\"\\n\\n # the final bounding box list\\n pred_box2d_list = []\\n pred_score_list = []\\n\\n for cav_id, cav_content in data_dict.items():\\n assert cav_id in output_dict\\n # the transformation matrix to ego space\\n transformation_matrix = cav_content['transformation_matrix']\\n\\n # classification probability -- (label_shape[0], label_shape[1])\\n prob = output_dict[cav_id]['cls'].squeeze(0).squeeze(0)\\n prob = torch.sigmoid(prob)\\n # regression map -- (label_shape[0], label_shape[1], 6)\\n reg_map = output_dict[cav_id]['reg'].squeeze(0).permute(1, 2, 0)\\n reg_map = self.denormalize_reg_map(reg_map)\\n threshold = self.params['target_args']['score_threshold']\\n mask = torch.gt(prob, threshold)\\n if mask.sum() > 0:\\n # (number of high confidence bbx, 4, 2)\\n corners2d = self.reg_map_to_bbx_corners(reg_map, mask)\\n # assume the z-diviation in transformation_matrix is small,\\n # thus we can pad zeros to simulate the 3d transformation.\\n # (number of high confidence bbx, 4, 3)\\n box3d = F.pad(corners2d, (0, 1))\\n # (number of high confidence bbx, 4, 2)\\n projected_boxes2d = \\\\\\n box_utils.project_points_by_matrix_torch(box3d.view(-1, 3),\\n transformation_matrix)[:, :2]\\n\\n projected_boxes2d = projected_boxes2d.view(-1, 4, 2)\\n scores = prob[mask]\\n pred_box2d_list.append(projected_boxes2d)\\n pred_score_list.append(scores)\\n\\n if len(pred_box2d_list):\\n pred_box2ds = torch.cat(pred_box2d_list, dim=0)\\n pred_scores = torch.cat(pred_score_list, dim=0)\\n else:\\n return None, None\\n\\n\\n keep_index = box_utils.nms_rotated(pred_box2ds, pred_scores,\\n self.params['nms_thresh'])\\n if len(keep_index):\\n pred_box2ds = pred_box2ds[keep_index]\\n pred_scores = pred_scores[keep_index]\\n\\n # filter out the prediction out of the range.\\n mask = box_utils.get_mask_for_boxes_within_range_torch(pred_box2ds)\\n pred_box2ds = pred_box2ds[mask, :, :]\\n pred_scores = pred_scores[mask]\\n assert pred_scores.shape[0] == pred_box2ds.shape[0]\\n return pred_box2ds, pred_scores\\n\\n def reg_map_to_bbx_corners(self, reg_map, mask):\\n \\\"\\\"\\\"\\n Construct bbx from the regression output of the model.\\n\\n Parameters\\n ----------\\n reg_map : torch.Tensor\\n Regression output of neural networks.\\n\\n mask : torch.Tensor\\n Masks used to filter bbx.\\n\\n Returns\\n -------\\n corners : torch.Tensor\\n Bbx output with shape (N, 4, 2).\\n\\n \\\"\\\"\\\"\\n\\n assert len(reg_map.shape) == 3,\\\\\\n \\\"only support shape of label_shape i.e. (*, *, 6)\\\"\\n device = reg_map.device\\n\\n cos_t, sin_t, x, y, log_dx, log_dy = \\\\\\n [tt.squeeze(-1) for tt in torch.chunk(reg_map, 6, dim=-1)]\\n yaw = torch.atan2(sin_t, cos_t)\\n dx, dy = log_dx.exp(), log_dy.exp()\\n\\n grid_size = self.geometry_param[\\\"res\\\"] * \\\\\\n self.geometry_param[\\\"downsample_rate\\\"]\\n grid_x = torch.arange(self.geometry_param[\\\"L1\\\"],\\n self.geometry_param[\\\"L2\\\"],\\n grid_size, dtype=torch.float32, device=device)\\n grid_y = torch.arange(self.geometry_param[\\\"W1\\\"],\\n self.geometry_param[\\\"W2\\\"],\\n grid_size,\\n dtype=torch.float32,\\n device=device)\\n\\n xx, yy = torch.meshgrid([grid_x, grid_y])\\n center_x = xx + x\\n center_y = yy + y\\n\\n bbx2d = torch.stack([center_x, center_y, dx, dy, yaw], dim=-1)\\n bbx2d = bbx2d[mask, :]\\n corners = box_utils.boxes2d_to_corners2d(bbx2d)\\n\\n return corners\\n\\n def post_process_debug(self, data_dict, output_dict):\\n \\\"\\\"\\\"\\n Process the outputs of the model to 2D bounding box for debug purpose.\\n Step1: convert each cav's output to bounding box format\\n Step2: project the bounding boxes to ego space.\\n Step:3 NMS\\n\\n Parameters\\n ----------\\n data_dict : dict\\n The dictionary containing the origin input data of model.\\n\\n output_dict :dict\\n The dictionary containing the output of the model.\\n\\n Returns\\n -------\\n pred_box2d_tensor : torch.Tensor\\n The prediction bounding box tensor after NMS.\\n gt_box2d_tensor : torch.Tensor\\n The groundtruth bounding box tensor.\\n \\\"\\\"\\\"\\n # the final bounding box list\\n pred_box2d_list = []\\n pred_score_list = []\\n\\n # the transformation matrix to ego space\\n transformation_matrix = data_dict['transformation_matrix']\\n\\n # classification probability -- (label_shape[0], label_shape[1])\\n prob = output_dict['cls'].squeeze(0).squeeze(0)\\n prob = torch.sigmoid(prob)\\n\\n # regression map -- (label_shape[0], label_shape[1], 6)\\n reg_map = output_dict['reg'].squeeze(0).permute(1, 2, 0)\\n reg_map = self.denormalize_reg_map(reg_map)\\n # threshold = self.params['target_args']['score_threshold']\\n threshold = 0.5\\n mask = torch.gt(prob, threshold)\\n if mask.sum() > 0:\\n # (number of high confidence bbx, 4, 2)\\n corners2d = self.reg_map_to_bbx_corners(reg_map, mask)\\n # assume the z-diviation in transformation_matrix is small,\\n # thus we can pad zeros to simulate the 3d transformation.\\n # (number of high confidence bbx, 4, 3)\\n box3d = F.pad(corners2d, (0, 1))\\n\\n # (number of high confidence bbx, 4, 2)\\n projected_boxes2d = \\\\\\n box_utils.project_points_by_matrix_torch(box3d.view(-1, 3),\\n transformation_matrix)[:, :2]\\n projected_boxes2d = projected_boxes2d.view(-1, 4, 2)\\n scores = prob[mask]\\n pred_box2d_list.append(projected_boxes2d)\\n pred_score_list.append(scores)\\n\\n pred_box2ds = torch.cat(pred_box2d_list, dim=0)\\n pred_scores = torch.cat(pred_score_list, dim=0)\\n\\n keep_index = box_utils.nms_rotated(pred_box2ds,\\n pred_scores,\\n self.params['nms_thresh'])\\n pred_box2ds = pred_box2ds[keep_index]\\n\\n # filter out the prediction out of the range.\\n mask = box_utils.get_mask_for_boxes_within_range_torch(pred_box2ds)\\n pred_box2ds = pred_box2ds[mask, :, :]\\n return pred_box2ds\\n\\n @staticmethod\\n def visualize(pred_box_tensor, gt_tensor, pcd, show_vis, save_path, dataset = None):\\n \\\"\\\"\\\"\\n Visualize the BEV 2D prediction, ground truth with point cloud together.\\n\\n Parameters\\n ----------\\n pred_box_tensor : torch.Tensor\\n (N, 8, 3) prediction.\\n\\n gt_tensor : torch.Tensor\\n (N, 8, 3) groundtruth bbx\\n\\n pcd : torch.Tensor\\n PointCloud, (N, 4).\\n\\n show_vis : bool\\n Whether to show visualization.\\n\\n save_path : str\\n Save the visualization results to given path.\\n\\n dataset : BaseDataset\\n opencood dataset object.\\n \\\"\\\"\\\"\\n assert dataset is not None, \\\"dataset argument can't be None\\\"\\n vis_utils.visualize_single_sample_output_bev(pred_box_tensor,\\n gt_tensor,\\n pcd,\\n dataset,\\n show_vis,\\n save_path)\\n\\nfrom v2xvit.data_utils.post_processor.voxel_postprocessor import VoxelPostprocessor\\nfrom v2xvit.data_utils.post_processor.bev_postprocessor import BevPostprocessor\\n\\n__all__ = {\\n 'VoxelPostprocessor': VoxelPostprocessor,\\n 'BevPostprocessor': BevPostprocessor,\\n}\\n\\n\\ndef build_postprocessor(anchor_cfg, train):\\n process_method_name = anchor_cfg['core_method']\\n assert process_method_name in ['VoxelPostprocessor', 'BevPostprocessor']\\n anchor_generator = __all__[process_method_name](\\n anchor_params=anchor_cfg,\\n train=train\\n )\\n\\n return anchor_generator\\n\\n\\n\\\"\\\"\\\"\\nDataset class for late fusion\\n\\\"\\\"\\\"\\nimport random\\nimport math\\nfrom collections import OrderedDict\\n\\nimport numpy as np\\nimport torch\\nfrom torch.utils.data import DataLoader\\n\\nimport v2xvit\\nfrom v2xvit.data_utils.post_processor import build_postprocessor\\nfrom v2xvit.data_utils.datasets import basedataset\\nfrom v2xvit.data_utils.pre_processor import build_preprocessor\\nfrom v2xvit.hypes_yaml.yaml_utils import load_yaml\\nfrom v2xvit.utils import box_utils\\nfrom v2xvit.utils.pcd_utils import \\\\\\n mask_points_by_range, mask_ego_points, shuffle_points, \\\\\\n downsample_lidar_minimum\\n\\n\\nclass LateFusionDataset(basedataset.BaseDataset):\\n def __init__(self, params, visualize, train=True):\\n super(LateFusionDataset, self).__init__(params, visualize, train)\\n self.pre_processor = build_preprocessor(params['preprocess'],\\n train)\\n self.post_processor = build_postprocessor(params['postprocess'], train)\\n\\n def __getitem__(self, idx):\\n base_data_dict = self.retrieve_base_data(idx, cur_ego_pose_flag=True)\\n if self.train:\\n reformat_data_dict = self.get_item_train(base_data_dict)\\n else:\\n reformat_data_dict = self.get_item_test(base_data_dict)\\n\\n return reformat_data_dict\\n\\n def get_item_single_car(self, selected_cav_base):\\n \\\"\\\"\\\"\\n Process a single CAV's information for the train/test pipeline.\\n\\n Parameters\\n ----------\\n selected_cav_base : dict\\n The dictionary contains a single CAV's raw information.\\n\\n Returns\\n -------\\n selected_cav_processed : dict\\n The dictionary contains the cav's processed information.\\n \\\"\\\"\\\"\\n selected_cav_processed = {}\\n\\n # filter lidar\\n lidar_np = selected_cav_base['lidar_np']\\n lidar_np = shuffle_points(lidar_np)\\n lidar_np = mask_points_by_range(lidar_np,\\n self.params['preprocess'][\\n 'cav_lidar_range'])\\n # remove points that hit ego vehicle\\n lidar_np = mask_ego_points(lidar_np)\\n\\n # generate the bounding box(n, 7) under the cav's space\\n object_bbx_center, object_bbx_mask, object_ids = \\\\\\n self.post_processor.generate_object_center([selected_cav_base],\\n selected_cav_base[\\n 'params'][\\n 'lidar_pose'])\\n # data augmentation\\n lidar_np, object_bbx_center, object_bbx_mask = \\\\\\n self.augment(lidar_np, object_bbx_center, object_bbx_mask)\\n\\n if self.visualize:\\n selected_cav_processed.update({'origin_lidar': lidar_np})\\n\\n # pre-process the lidar to voxel/bev/downsampled lidar\\n lidar_dict = self.pre_processor.preprocess(lidar_np)\\n selected_cav_processed.update({'processed_lidar': lidar_dict})\\n\\n # generate the anchor boxes\\n anchor_box = self.post_processor.generate_anchor_box()\\n selected_cav_processed.update({'anchor_box': anchor_box})\\n\\n selected_cav_processed.update({'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': object_bbx_mask,\\n 'object_ids': object_ids})\\n\\n # generate targets label\\n label_dict = \\\\\\n self.post_processor.generate_label(\\n gt_box_center=object_bbx_center,\\n anchors=anchor_box,\\n mask=object_bbx_mask)\\n selected_cav_processed.update({'label_dict': label_dict})\\n\\n return selected_cav_processed\\n\\n def get_item_train(self, base_data_dict):\\n processed_data_dict = OrderedDict()\\n\\n # during training, we return a random cav's data\\n if not self.visualize:\\n selected_cav_id, selected_cav_base = \\\\\\n random.choice(list(base_data_dict.items()))\\n else:\\n selected_cav_id, selected_cav_base = \\\\\\n list(base_data_dict.items())[0]\\n\\n selected_cav_processed = self.get_item_single_car(selected_cav_base)\\n processed_data_dict.update({'ego': selected_cav_processed})\\n\\n return processed_data_dict\\n\\n def get_item_test(self, base_data_dict):\\n processed_data_dict = OrderedDict()\\n ego_id = -1\\n ego_lidar_pose = []\\n\\n # first find the ego vehicle's lidar pose\\n for cav_id, cav_content in base_data_dict.items():\\n if cav_content['ego']:\\n ego_id = cav_id\\n ego_lidar_pose = cav_content['params']['lidar_pose']\\n break\\n\\n assert ego_id != -1\\n assert len(ego_lidar_pose) > 0\\n\\n # loop over all CAVs to process information\\n for cav_id, selected_cav_base in base_data_dict.items():\\n distance = \\\\\\n math.sqrt((selected_cav_base['params']['lidar_pose'][0] -\\n ego_lidar_pose[0]) ** 2 + (\\n selected_cav_base['params'][\\n 'lidar_pose'][1] - ego_lidar_pose[\\n 1]) ** 2)\\n if distance > v2xvit.data_utils.datasets.COM_RANGE:\\n continue\\n\\n # find the transformation matrix from current cav to ego.\\n # this is used to project prediction to the right space\\n transformation_matrix = \\\\\\n selected_cav_base['params']['transformation_matrix']\\n # this is used to project gt objects to ego space\\n gt_transformation_matrix = \\\\\\n selected_cav_base['params']['gt_transformation_matrix']\\n\\n selected_cav_processed = \\\\\\n self.get_item_single_car(selected_cav_base)\\n selected_cav_processed.update({'transformation_matrix':\\n transformation_matrix})\\n selected_cav_processed.update({'gt_transformation_matrix':\\n gt_transformation_matrix})\\n\\n update_cav = \\\"ego\\\" if cav_id == ego_id else cav_id\\n processed_data_dict.update({update_cav: selected_cav_processed})\\n\\n return processed_data_dict\\n\\n def collate_batch_test(self, batch):\\n \\\"\\\"\\\"\\n Customized collate function for pytorch dataloader during testing\\n for late fusion dataset.\\n\\n Parameters\\n ----------\\n batch : dict\\n\\n Returns\\n -------\\n batch : dict\\n Reformatted batch.\\n \\\"\\\"\\\"\\n # currently, we only support batch size of 1 during testing\\n assert len(batch) <= 1, \\\"Batch size 1 is required during testing!\\\"\\n batch = batch[0]\\n\\n output_dict = {}\\n\\n # for late fusion, we also need to stack the lidar for better\\n # visualization\\n if self.visualize:\\n projected_lidar_list = []\\n origin_lidar = []\\n\\n for cav_id, cav_content in batch.items():\\n output_dict.update({cav_id: {}})\\n # shape: (1, max_num, 7)\\n object_bbx_center = \\\\\\n torch.from_numpy(np.array([cav_content['object_bbx_center']]))\\n object_bbx_mask = \\\\\\n torch.from_numpy(np.array([cav_content['object_bbx_mask']]))\\n object_ids = cav_content['object_ids']\\n\\n # the anchor box is the same for all bounding boxes usually, thus\\n # we don't need the batch dimension.\\n if cav_content['anchor_box'] is not None:\\n output_dict[cav_id].update({'anchor_box':\\n torch.from_numpy(np.array(\\n cav_content[\\n 'anchor_box']))})\\n if self.visualize:\\n transformation_matrix = cav_content['transformation_matrix']\\n origin_lidar = [cav_content['origin_lidar']]\\n\\n projected_lidar = cav_content['origin_lidar']\\n projected_lidar[:, :3] = \\\\\\n box_utils.project_points_by_matrix_torch(\\n projected_lidar[:, :3],\\n transformation_matrix)\\n projected_lidar_list.append(projected_lidar)\\n\\n # processed lidar dictionary\\n processed_lidar_torch_dict = \\\\\\n self.pre_processor.collate_batch(\\n [cav_content['processed_lidar']])\\n # label dictionary\\n label_torch_dict = \\\\\\n self.post_processor.collate_batch([cav_content['label_dict']])\\n\\n # save the transformation matrix (4, 4) to ego vehicle\\n transformation_matrix_torch = \\\\\\n torch.from_numpy(\\n np.array(cav_content['transformation_matrix'])).float()\\n gt_transformation_matrix_torch = \\\\\\n torch.from_numpy(\\n np.array(cav_content['gt_transformation_matrix'])).float()\\n\\n output_dict[cav_id].update({'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': object_bbx_mask,\\n 'processed_lidar': processed_lidar_torch_dict,\\n 'label_dict': label_torch_dict,\\n 'object_ids': object_ids,\\n 'transformation_matrix': transformation_matrix_torch,\\n 'gt_transformation_matrix': gt_transformation_matrix_torch})\\n\\n if self.visualize:\\n origin_lidar = \\\\\\n np.array(\\n downsample_lidar_minimum(pcd_np_list=origin_lidar))\\n origin_lidar = torch.from_numpy(origin_lidar)\\n output_dict[cav_id].update({'origin_lidar': origin_lidar})\\n\\n if self.visualize:\\n projected_lidar_stack = [torch.from_numpy(\\n np.vstack(projected_lidar_list))]\\n output_dict['ego'].update({'origin_lidar': projected_lidar_stack})\\n\\n return output_dict\\n\\n def post_process(self, data_dict, output_dict):\\n \\\"\\\"\\\"\\n Process the outputs of the model to 2D/3D bounding box.\\n\\n Parameters\\n ----------\\n data_dict : dict\\n The dictionary containing the origin input data of model.\\n\\n output_dict :dict\\n The dictionary containing the output of the model.\\n\\n Returns\\n -------\\n pred_box_tensor : torch.Tensor\\n The tensor of prediction bounding box after NMS.\\n gt_box_tensor : torch.Tensor\\n The tensor of gt bounding box.\\n \\\"\\\"\\\"\\n pred_box_tensor, pred_score = \\\\\\n self.post_processor.post_process(data_dict, output_dict)\\n gt_box_tensor = self.post_processor.generate_gt_bbx(data_dict)\\n\\n return pred_box_tensor, pred_score, gt_box_tensor\\n\\n\\n\\\"\\\"\\\"\\nDataset class for early fusion\\n\\\"\\\"\\\"\\nimport math\\nfrom collections import OrderedDict\\n\\nimport numpy as np\\nimport torch\\n\\nimport v2xvit\\nimport v2xvit.data_utils.post_processor as post_processor\\nfrom v2xvit.utils import box_utils\\nfrom v2xvit.data_utils.datasets import basedataset\\nfrom v2xvit.data_utils.pre_processor import build_preprocessor\\nfrom v2xvit.utils.pcd_utils import \\\\\\n mask_points_by_range, mask_ego_points, shuffle_points, \\\\\\n downsample_lidar_minimum\\n\\n\\nclass IntermediateFusionDataset(basedataset.BaseDataset):\\n def __init__(self, params, visualize, train=True):\\n super(IntermediateFusionDataset, self). \\\\\\n __init__(params, visualize, train)\\n self.cur_ego_pose_flag = params['fusion']['args']['cur_ego_pose_flag']\\n self.pre_processor = build_preprocessor(params['preprocess'],\\n train)\\n self.post_processor = post_processor.build_postprocessor(\\n params['postprocess'],\\n train)\\n\\n def __getitem__(self, idx):\\n # when the cur_ego_pose_flag is set to True, there is no time gap\\n # between the time when the LiDAR data is captured by connected\\n # agents and when the extracted features are received by\\n # the ego vehicle. This is equal to implement STCM.\\n base_data_dict = \\\\\\n self.retrieve_base_data(idx,\\n cur_ego_pose_flag=self.cur_ego_pose_flag)\\n\\n processed_data_dict = OrderedDict()\\n processed_data_dict['ego'] = {}\\n\\n ego_id = -1\\n ego_lidar_pose = []\\n\\n # first find the ego vehicle's lidar pose\\n for cav_id, cav_content in base_data_dict.items():\\n if cav_content['ego']:\\n ego_id = cav_id\\n ego_lidar_pose = cav_content['params']['lidar_pose']\\n break\\n assert cav_id == list(base_data_dict.keys())[\\n 0], \\\"The first element in the OrderedDict must be ego\\\"\\n assert ego_id != -1\\n assert len(ego_lidar_pose) > 0\\n # this is used for v2vnet and disconet\\n pairwise_t_matrix = \\\\\\n self.get_pairwise_transformation(base_data_dict,\\n self.params['train_params'][\\n 'max_cav'])\\n\\n processed_features = []\\n object_stack = []\\n object_id_stack = []\\n\\n # prior knowledge for time delay correction and indicating data type\\n # (V2V vs V2i)\\n velocity = []\\n time_delay = []\\n infra = []\\n spatial_correction_matrix = []\\n\\n if self.visualize:\\n projected_lidar_stack = []\\n\\n # loop over all CAVs to process information\\n for cav_id, selected_cav_base in base_data_dict.items():\\n # check if the cav is within the communication range with ego\\n distance = \\\\\\n math.sqrt((selected_cav_base['params']['lidar_pose'][0] -\\n ego_lidar_pose[0]) ** 2 + (\\n selected_cav_base['params'][\\n 'lidar_pose'][1] - ego_lidar_pose[\\n 1]) ** 2)\\n if distance > v2xvit.data_utils.datasets.COM_RANGE:\\n continue\\n\\n selected_cav_processed, void_lidar = self.get_item_single_car(\\n selected_cav_base,\\n ego_lidar_pose)\\n\\n if void_lidar:\\n continue\\n\\n object_stack.append(selected_cav_processed['object_bbx_center'])\\n object_id_stack += selected_cav_processed['object_ids']\\n processed_features.append(\\n selected_cav_processed['processed_features'])\\n\\n velocity.append(selected_cav_processed['velocity'])\\n time_delay.append(float(selected_cav_base['time_delay']))\\n spatial_correction_matrix.append(\\n selected_cav_base['params']['spatial_correction_matrix'])\\n infra.append(1 if int(cav_id) < 0 else 0)\\n\\n if self.visualize:\\n projected_lidar_stack.append(\\n selected_cav_processed['projected_lidar'])\\n\\n # exclude all repetitive objects\\n unique_indices = \\\\\\n [object_id_stack.index(x) for x in set(object_id_stack)]\\n object_stack = np.vstack(object_stack)\\n object_stack = object_stack[unique_indices]\\n\\n # make sure bounding boxes across all frames have the same number\\n object_bbx_center = \\\\\\n np.zeros((self.params['postprocess']['max_num'], 7))\\n mask = np.zeros(self.params['postprocess']['max_num'])\\n object_bbx_center[:object_stack.shape[0], :] = object_stack\\n mask[:object_stack.shape[0]] = 1\\n\\n # merge preprocessed features from different cavs into the same dict\\n cav_num = len(processed_features)\\n merged_feature_dict = self.merge_features_to_dict(processed_features)\\n\\n # generate the anchor boxes\\n anchor_box = self.post_processor.generate_anchor_box()\\n\\n # generate targets label\\n label_dict = \\\\\\n self.post_processor.generate_label(\\n gt_box_center=object_bbx_center,\\n anchors=anchor_box,\\n mask=mask)\\n\\n # pad dv, dt, infra to max_cav\\n velocity = velocity + (self.max_cav - len(velocity)) * [0.]\\n time_delay = time_delay + (self.max_cav - len(time_delay)) * [0.]\\n infra = infra + (self.max_cav - len(infra)) * [0.]\\n spatial_correction_matrix = np.stack(spatial_correction_matrix)\\n padding_eye = np.tile(np.eye(4)[None],(self.max_cav - len(\\n spatial_correction_matrix),1,1))\\n spatial_correction_matrix = np.concatenate([spatial_correction_matrix, padding_eye], axis=0)\\n\\n processed_data_dict['ego'].update(\\n {'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': mask,\\n 'object_ids': [object_id_stack[i] for i in unique_indices],\\n 'anchor_box': anchor_box,\\n 'processed_lidar': merged_feature_dict,\\n 'label_dict': label_dict,\\n 'cav_num': cav_num,\\n 'velocity': velocity,\\n 'time_delay': time_delay,\\n 'infra': infra,\\n 'spatial_correction_matrix': spatial_correction_matrix,\\n 'pairwise_t_matrix': pairwise_t_matrix})\\n\\n if self.visualize:\\n processed_data_dict['ego'].update({'origin_lidar':\\n np.vstack(\\n projected_lidar_stack)})\\n return processed_data_dict\\n\\n @staticmethod\\n def get_pairwise_transformation(base_data_dict, max_cav):\\n \\\"\\\"\\\"\\n Get pair-wise transformation matrix across different agents.\\n This is only used for v2vnet and disconet. Currently we set\\n this as identity matrix as the pointcloud is projected to\\n ego vehicle first.\\n\\n Parameters\\n ----------\\n base_data_dict : dict\\n Key : cav id, item: transformation matrix to ego, lidar points.\\n\\n max_cav : int\\n The maximum number of cav, default 5\\n\\n Return\\n ------\\n pairwise_t_matrix : np.array\\n The pairwise transformation matrix across each cav.\\n shape: (L, L, 4, 4)\\n \\\"\\\"\\\"\\n pairwise_t_matrix = np.zeros((max_cav, max_cav, 4, 4))\\n # default are identity matrix\\n pairwise_t_matrix[:, :] = np.identity(4)\\n\\n return pairwise_t_matrix\\n\\n def get_item_single_car(self, selected_cav_base, ego_pose):\\n \\\"\\\"\\\"\\n Project the lidar and bbx to ego space first, and then do clipping.\\n\\n Parameters\\n ----------\\n selected_cav_base : dict\\n The dictionary contains a single CAV's raw information.\\n ego_pose : list\\n The ego vehicle lidar pose under world coordinate.\\n\\n Returns\\n -------\\n selected_cav_processed : dict\\n The dictionary contains the cav's processed information.\\n \\\"\\\"\\\"\\n selected_cav_processed = {}\\n\\n # calculate the transformation matrix\\n transformation_matrix = \\\\\\n selected_cav_base['params']['transformation_matrix']\\n\\n # retrieve objects under ego coordinates\\n object_bbx_center, object_bbx_mask, object_ids = \\\\\\n self.post_processor.generate_object_center([selected_cav_base],\\n ego_pose)\\n\\n # filter lidar\\n lidar_np = selected_cav_base['lidar_np']\\n lidar_np = shuffle_points(lidar_np)\\n # remove points that hit itself\\n lidar_np = mask_ego_points(lidar_np)\\n # project the lidar to ego space\\n lidar_np[:, :3] = \\\\\\n box_utils.project_points_by_matrix_torch(lidar_np[:, :3],\\n transformation_matrix)\\n lidar_np = mask_points_by_range(lidar_np,\\n self.params['preprocess'][\\n 'cav_lidar_range'])\\n # Check if filtered LiDAR points are not void\\n void_lidar = True if lidar_np.shape[0] < 1 else False\\n\\n processed_lidar = self.pre_processor.preprocess(lidar_np)\\n\\n # velocity\\n velocity = selected_cav_base['params']['ego_speed']\\n # normalize veloccity by average speed 30 km/h\\n velocity = velocity / 30\\n\\n selected_cav_processed.update(\\n {'object_bbx_center': object_bbx_center[object_bbx_mask == 1],\\n 'object_ids': object_ids,\\n 'projected_lidar': lidar_np,\\n 'processed_features': processed_lidar,\\n 'velocity': velocity})\\n\\n return selected_cav_processed, void_lidar\\n\\n @staticmethod\\n def merge_features_to_dict(processed_feature_list):\\n \\\"\\\"\\\"\\n Merge the preprocessed features from different cavs to the same\\n dictionary.\\n\\n Parameters\\n ----------\\n processed_feature_list : list\\n A list of dictionary containing all processed features from\\n different cavs.\\n\\n Returns\\n -------\\n merged_feature_dict: dict\\n key: feature names, value: list of features.\\n \\\"\\\"\\\"\\n\\n merged_feature_dict = OrderedDict()\\n\\n for i in range(len(processed_feature_list)):\\n for feature_name, feature in processed_feature_list[i].items():\\n if feature_name not in merged_feature_dict:\\n merged_feature_dict[feature_name] = []\\n if isinstance(feature, list):\\n merged_feature_dict[feature_name] += feature\\n else:\\n merged_feature_dict[feature_name].append(feature)\\n\\n return merged_feature_dict\\n\\n def collate_batch_train(self, batch):\\n # Intermediate fusion is different the other two\\n output_dict = {'ego': {}}\\n\\n object_bbx_center = []\\n object_bbx_mask = []\\n object_ids = []\\n processed_lidar_list = []\\n # used to record different scenario\\n record_len = []\\n label_dict_list = []\\n\\n # used for PriorEncoding\\n velocity = []\\n time_delay = []\\n infra = []\\n\\n # pairwise transformation matrix\\n pairwise_t_matrix_list = []\\n\\n # used for correcting the spatial transformation between delayed timestamp\\n # and current timestamp\\n spatial_correction_matrix_list = []\\n\\n if self.visualize:\\n origin_lidar = []\\n\\n for i in range(len(batch)):\\n ego_dict = batch[i]['ego']\\n object_bbx_center.append(ego_dict['object_bbx_center'])\\n object_bbx_mask.append(ego_dict['object_bbx_mask'])\\n object_ids.append(ego_dict['object_ids'])\\n\\n processed_lidar_list.append(ego_dict['processed_lidar'])\\n record_len.append(ego_dict['cav_num'])\\n label_dict_list.append(ego_dict['label_dict'])\\n\\n velocity.append(ego_dict['velocity'])\\n time_delay.append(ego_dict['time_delay'])\\n infra.append(ego_dict['infra'])\\n spatial_correction_matrix_list.append(\\n ego_dict['spatial_correction_matrix'])\\n pairwise_t_matrix_list.append(ego_dict['pairwise_t_matrix'])\\n\\n if self.visualize:\\n origin_lidar.append(ego_dict['origin_lidar'])\\n # convert to numpy, (B, max_num, 7)\\n object_bbx_center = torch.from_numpy(np.array(object_bbx_center))\\n object_bbx_mask = torch.from_numpy(np.array(object_bbx_mask))\\n\\n # example: {'voxel_features':[np.array([1,2,3]]),\\n # np.array([3,5,6]), ...]}\\n merged_feature_dict = self.merge_features_to_dict(processed_lidar_list)\\n processed_lidar_torch_dict = \\\\\\n self.pre_processor.collate_batch(merged_feature_dict)\\n # [2, 3, 4, ..., M]\\n record_len = torch.from_numpy(np.array(record_len, dtype=int))\\n label_torch_dict = \\\\\\n self.post_processor.collate_batch(label_dict_list)\\n\\n # (B, max_cav)\\n velocity = torch.from_numpy(np.array(velocity))\\n time_delay = torch.from_numpy(np.array(time_delay))\\n infra = torch.from_numpy(np.array(infra))\\n spatial_correction_matrix_list = \\\\\\n torch.from_numpy(np.array(spatial_correction_matrix_list))\\n # (B, max_cav, 3)\\n prior_encoding = \\\\\\n torch.stack([velocity, time_delay, infra], dim=-1).float()\\n # (B, max_cav)\\n pairwise_t_matrix = torch.from_numpy(np.array(pairwise_t_matrix_list))\\n\\n # object id is only used during inference, where batch size is 1.\\n # so here we only get the first element.\\n output_dict['ego'].update({'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': object_bbx_mask,\\n 'processed_lidar': processed_lidar_torch_dict,\\n 'record_len': record_len,\\n 'label_dict': label_torch_dict,\\n 'object_ids': object_ids[0],\\n 'prior_encoding': prior_encoding,\\n 'spatial_correction_matrix': spatial_correction_matrix_list,\\n 'pairwise_t_matrix': pairwise_t_matrix})\\n\\n if self.visualize:\\n origin_lidar = \\\\\\n np.array(downsample_lidar_minimum(pcd_np_list=origin_lidar))\\n origin_lidar = torch.from_numpy(origin_lidar)\\n output_dict['ego'].update({'origin_lidar': origin_lidar})\\n\\n return output_dict\\n\\n def collate_batch_test(self, batch):\\n assert len(batch) <= 1, \\\"Batch size 1 is required during testing!\\\"\\n output_dict = self.collate_batch_train(batch)\\n\\n # check if anchor box in the batch\\n if batch[0]['ego']['anchor_box'] is not None:\\n output_dict['ego'].update({'anchor_box':\\n torch.from_numpy(np.array(\\n batch[0]['ego'][\\n 'anchor_box']))})\\n\\n # save the transformation matrix (4, 4) to ego vehicle\\n transformation_matrix_torch = \\\\\\n torch.from_numpy(np.identity(4)).float()\\n output_dict['ego'].update({'transformation_matrix':\\n transformation_matrix_torch})\\n\\n return output_dict\\n\\n def post_process(self, data_dict, output_dict):\\n \\\"\\\"\\\"\\n Process the outputs of the model to 2D/3D bounding box.\\n\\n Parameters\\n ----------\\n data_dict : dict\\n The dictionary containing the origin input data of model.\\n\\n output_dict :dict\\n The dictionary containing the output of the model.\\n\\n Returns\\n -------\\n pred_box_tensor : torch.Tensor\\n The tensor of prediction bounding box after NMS.\\n gt_box_tensor : torch.Tensor\\n The tensor of gt bounding box.\\n \\\"\\\"\\\"\\n pred_box_tensor, pred_score = \\\\\\n self.post_processor.post_process(data_dict, output_dict)\\n gt_box_tensor = self.post_processor.generate_gt_bbx(data_dict)\\n\\n return pred_box_tensor, pred_score, gt_box_tensor\\n\\n\\n\\\"\\\"\\\"\\nThis is a dataset for early fusion visualization only.\\n\\\"\\\"\\\"\\nfrom collections import OrderedDict\\n\\nimport numpy as np\\nimport torch\\n\\nfrom v2xvit.utils import box_utils\\nfrom v2xvit.data_utils.post_processor import build_postprocessor\\nfrom v2xvit.data_utils.datasets import basedataset\\nfrom v2xvit.data_utils.pre_processor import build_preprocessor\\nfrom v2xvit.utils.pcd_utils import \\\\\\n mask_points_by_range, mask_ego_points, shuffle_points, \\\\\\n downsample_lidar_minimum\\n\\n\\nclass EarlyFusionVisDataset(basedataset.BaseDataset):\\n def __init__(self, params, visualize, train=True):\\n super(EarlyFusionVisDataset, self).__init__(params, visualize, train)\\n self.pre_processor = build_preprocessor(params['preprocess'],\\n train)\\n self.post_processor = build_postprocessor(params['postprocess'], train)\\n\\n def __getitem__(self, idx):\\n base_data_dict = self.retrieve_base_data(idx)\\n\\n processed_data_dict = OrderedDict()\\n processed_data_dict['ego'] = {}\\n\\n ego_id = -1\\n ego_lidar_pose = []\\n\\n # first find the ego vehicle's lidar pose\\n for cav_id, cav_content in base_data_dict.items():\\n if cav_content['ego']:\\n ego_id = cav_id\\n ego_lidar_pose = cav_content['params']['lidar_pose']\\n break\\n\\n assert ego_id != -1\\n assert len(ego_lidar_pose) > 0\\n\\n projected_lidar_stack = []\\n object_stack = []\\n object_id_stack = []\\n\\n # loop over all CAVs to process information\\n for cav_id, selected_cav_base in base_data_dict.items():\\n selected_cav_processed = self.get_item_single_car(\\n selected_cav_base,\\n ego_lidar_pose)\\n # all these lidar and object coordinates are projected to ego\\n # already.\\n projected_lidar_stack.append(\\n selected_cav_processed['projected_lidar'])\\n object_stack.append(selected_cav_processed['object_bbx_center'])\\n object_id_stack += selected_cav_processed['object_ids']\\n\\n # exclude all repetitive objects\\n unique_indices = \\\\\\n [object_id_stack.index(x) for x in set(object_id_stack)]\\n object_stack = np.vstack(object_stack)\\n object_stack = object_stack[unique_indices]\\n\\n # make sure bounding boxes across all frames have the same number\\n object_bbx_center = \\\\\\n np.zeros((self.params['postprocess']['max_num'], 7))\\n mask = np.zeros(self.params['postprocess']['max_num'])\\n object_bbx_center[:object_stack.shape[0], :] = object_stack\\n mask[:object_stack.shape[0]] = 1\\n\\n # convert list to numpy array, (N, 4)\\n projected_lidar_stack = np.vstack(projected_lidar_stack)\\n\\n # data augmentation\\n projected_lidar_stack, object_bbx_center, mask = \\\\\\n self.augment(projected_lidar_stack, object_bbx_center, mask)\\n\\n # we do lidar filtering in the stacked lidar\\n projected_lidar_stack = mask_points_by_range(projected_lidar_stack,\\n self.params['preprocess'][\\n 'cav_lidar_range'])\\n # augmentation may remove some of the bbx out of range\\n object_bbx_center_valid = object_bbx_center[mask == 1]\\n object_bbx_center_valid = \\\\\\n box_utils.mask_boxes_outside_range_numpy(object_bbx_center_valid,\\n self.params['preprocess'][\\n 'cav_lidar_range'],\\n self.params['postprocess'][\\n 'order']\\n )\\n mask[object_bbx_center_valid.shape[0]:] = 0\\n object_bbx_center[:object_bbx_center_valid.shape[0]] = \\\\\\n object_bbx_center_valid\\n object_bbx_center[object_bbx_center_valid.shape[0]:] = 0\\n\\n processed_data_dict['ego'].update(\\n {'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': mask,\\n 'object_ids': [object_id_stack[i] for i in unique_indices],\\n 'origin_lidar': projected_lidar_stack\\n })\\n\\n return processed_data_dict\\n\\n def get_item_single_car(self, selected_cav_base, ego_pose):\\n \\\"\\\"\\\"\\n Project the lidar and bbx to ego space first, and then do clipping.\\n\\n Parameters\\n ----------\\n selected_cav_base : dict\\n The dictionary contains a single CAV's raw information.\\n ego_pose : list\\n The ego vehicle lidar pose under world coordinate.\\n\\n Returns\\n -------\\n selected_cav_processed : dict\\n The dictionary contains the cav's processed information.\\n \\\"\\\"\\\"\\n selected_cav_processed = {}\\n\\n # calculate the transformation matrix\\n transformation_matrix = \\\\\\n selected_cav_base['params']['transformation_matrix']\\n\\n # retrieve objects under ego coordinates\\n object_bbx_center, object_bbx_mask, object_ids = \\\\\\n self.post_processor.generate_object_center([selected_cav_base],\\n ego_pose)\\n\\n # filter lidar\\n lidar_np = selected_cav_base['lidar_np']\\n lidar_np = shuffle_points(lidar_np)\\n # remove points that hit itself\\n lidar_np = mask_ego_points(lidar_np)\\n # project the lidar to ego space\\n lidar_np[:, :3] = \\\\\\n box_utils.project_points_by_matrix_torch(lidar_np[:, :3],\\n transformation_matrix)\\n\\n selected_cav_processed.update(\\n {'object_bbx_center': object_bbx_center[object_bbx_mask == 1],\\n 'object_ids': object_ids,\\n 'projected_lidar': lidar_np})\\n\\n return selected_cav_processed\\n\\n def collate_batch_train(self, batch):\\n \\\"\\\"\\\"\\n Customized collate function for pytorch dataloader during training\\n for late fusion dataset.\\n\\n Parameters\\n ----------\\n batch : dict\\n\\n Returns\\n -------\\n batch : dict\\n Reformatted batch.\\n \\\"\\\"\\\"\\n # during training, we only care about ego.\\n output_dict = {'ego': {}}\\n\\n object_bbx_center = []\\n object_bbx_mask = []\\n origin_lidar = []\\n\\n for i in range(len(batch)):\\n ego_dict = batch[i]['ego']\\n object_bbx_center.append(ego_dict['object_bbx_center'])\\n object_bbx_mask.append(ego_dict['object_bbx_mask'])\\n origin_lidar.append(ego_dict['origin_lidar'])\\n\\n # convert to numpy, (B, max_num, 7)\\n object_bbx_center = torch.from_numpy(np.array(object_bbx_center))\\n object_bbx_mask = torch.from_numpy(np.array(object_bbx_mask))\\n output_dict['ego'].update({'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': object_bbx_mask})\\n\\n origin_lidar = \\\\\\n np.array(downsample_lidar_minimum(pcd_np_list=origin_lidar))\\n origin_lidar = torch.from_numpy(origin_lidar)\\n output_dict['ego'].update({'origin_lidar': origin_lidar})\\n\\n return output_dict\\n\\n\\n\\\"\\\"\\\"\\nDataset class for early fusion\\n\\\"\\\"\\\"\\nimport math\\nfrom collections import OrderedDict\\n\\nimport numpy as np\\nimport torch\\n\\nimport v2xvit\\nfrom v2xvit.utils import box_utils\\nfrom v2xvit.data_utils.post_processor import build_postprocessor\\nfrom v2xvit.data_utils.datasets import basedataset\\nfrom v2xvit.data_utils.pre_processor import build_preprocessor\\nfrom v2xvit.hypes_yaml.yaml_utils import load_yaml\\nfrom v2xvit.utils.pcd_utils import \\\\\\n mask_points_by_range, mask_ego_points, shuffle_points, \\\\\\n downsample_lidar_minimum\\n\\n\\nclass EarlyFusionDataset(basedataset.BaseDataset):\\n def __init__(self, params, visualize, train=True):\\n super(EarlyFusionDataset, self).__init__(params, visualize, train)\\n self.pre_processor = build_preprocessor(params['preprocess'],\\n train)\\n self.post_processor = build_postprocessor(params['postprocess'], train)\\n\\n def __getitem__(self, idx):\\n base_data_dict = self.retrieve_base_data(idx, cur_ego_pose_flag=True)\\n\\n processed_data_dict = OrderedDict()\\n processed_data_dict['ego'] = {}\\n\\n ego_id = -1\\n ego_lidar_pose = []\\n\\n # first find the ego vehicle's lidar pose\\n for cav_id, cav_content in base_data_dict.items():\\n if cav_content['ego']:\\n ego_id = cav_id\\n ego_lidar_pose = cav_content['params']['lidar_pose']\\n break\\n\\n assert ego_id != -1\\n assert len(ego_lidar_pose) > 0\\n\\n projected_lidar_stack = []\\n object_stack = []\\n object_id_stack = []\\n\\n # loop over all CAVs to process information\\n for cav_id, selected_cav_base in base_data_dict.items():\\n # check if the cav is within the communication range with ego\\n distance = \\\\\\n math.sqrt((selected_cav_base['params']['lidar_pose'][0] -\\n ego_lidar_pose[0]) ** 2 + (\\n selected_cav_base['params'][\\n 'lidar_pose'][1] - ego_lidar_pose[\\n 1]) ** 2)\\n if distance > v2xvit.data_utils.datasets.COM_RANGE:\\n continue\\n\\n selected_cav_processed = self.get_item_single_car(\\n selected_cav_base,\\n ego_lidar_pose)\\n # all these lidar and object coordinates are projected to ego\\n # already.\\n projected_lidar_stack.append(\\n selected_cav_processed['projected_lidar'])\\n object_stack.append(selected_cav_processed['object_bbx_center'])\\n object_id_stack += selected_cav_processed['object_ids']\\n\\n # exclude all repetitive objects\\n unique_indices = \\\\\\n [object_id_stack.index(x) for x in set(object_id_stack)]\\n object_stack = np.vstack(object_stack)\\n object_stack = object_stack[unique_indices]\\n\\n # make sure bounding boxes across all frames have the same number\\n object_bbx_center = \\\\\\n np.zeros((self.params['postprocess']['max_num'], 7))\\n mask = np.zeros(self.params['postprocess']['max_num'])\\n object_bbx_center[:object_stack.shape[0], :] = object_stack\\n mask[:object_stack.shape[0]] = 1\\n\\n # convert list to numpy array, (N, 4)\\n projected_lidar_stack = np.vstack(projected_lidar_stack)\\n\\n # data augmentation\\n projected_lidar_stack, object_bbx_center, mask = \\\\\\n self.augment(projected_lidar_stack, object_bbx_center, mask)\\n\\n # we do lidar filtering in the stacked lidar\\n projected_lidar_stack = mask_points_by_range(projected_lidar_stack,\\n self.params['preprocess'][\\n 'cav_lidar_range'])\\n # augmentation may remove some of the bbx out of range\\n object_bbx_center_valid = object_bbx_center[mask == 1]\\n object_bbx_center_valid = \\\\\\n box_utils.mask_boxes_outside_range_numpy(object_bbx_center_valid,\\n self.params['preprocess'][\\n 'cav_lidar_range'],\\n self.params[\\n 'postprocess'][\\n 'order']\\n )\\n mask[object_bbx_center_valid.shape[0]:] = 0\\n object_bbx_center[:object_bbx_center_valid.shape[0]] = \\\\\\n object_bbx_center_valid\\n object_bbx_center[object_bbx_center_valid.shape[0]:] = 0\\n\\n # pre-process the lidar to voxel/bev/downsampled lidar\\n lidar_dict = self.pre_processor.preprocess(projected_lidar_stack)\\n\\n # generate the anchor boxes\\n anchor_box = self.post_processor.generate_anchor_box()\\n\\n # generate targets label\\n label_dict = \\\\\\n self.post_processor.generate_label(\\n gt_box_center=object_bbx_center,\\n anchors=anchor_box,\\n mask=mask)\\n\\n processed_data_dict['ego'].update(\\n {'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': mask,\\n 'object_ids': [object_id_stack[i] for i in unique_indices],\\n 'anchor_box': anchor_box,\\n 'processed_lidar': lidar_dict,\\n 'label_dict': label_dict})\\n\\n if self.visualize:\\n processed_data_dict['ego'].update({'origin_lidar':\\n projected_lidar_stack})\\n\\n return processed_data_dict\\n\\n def get_item_single_car(self, selected_cav_base, ego_pose):\\n \\\"\\\"\\\"\\n Project the lidar and bbx to ego space first, and then do clipping.\\n\\n Parameters\\n ----------\\n selected_cav_base : dict\\n The dictionary contains a single CAV's raw information.\\n ego_pose : list\\n The ego vehicle lidar pose under world coordinate.\\n\\n Returns\\n -------\\n selected_cav_processed : dict\\n The dictionary contains the cav's processed information.\\n \\\"\\\"\\\"\\n selected_cav_processed = {}\\n\\n # calculate the transformation matrix\\n transformation_matrix = selected_cav_base['params'][\\n 'transformation_matrix']\\n\\n # retrieve objects under ego coordinates\\n object_bbx_center, object_bbx_mask, object_ids = \\\\\\n self.post_processor.generate_object_center([selected_cav_base],\\n ego_pose)\\n\\n # filter lidar\\n lidar_np = selected_cav_base['lidar_np']\\n lidar_np = shuffle_points(lidar_np)\\n # remove points that hit itself\\n lidar_np = mask_ego_points(lidar_np)\\n # project the lidar to ego space\\n lidar_np[:, :3] = \\\\\\n box_utils.project_points_by_matrix_torch(lidar_np[:, :3],\\n transformation_matrix)\\n\\n selected_cav_processed.update(\\n {'object_bbx_center': object_bbx_center[object_bbx_mask == 1],\\n 'object_ids': object_ids,\\n 'projected_lidar': lidar_np})\\n\\n return selected_cav_processed\\n\\n def collate_batch_test(self, batch):\\n \\\"\\\"\\\"\\n Customized collate function for pytorch dataloader during testing\\n for late fusion dataset.\\n\\n Parameters\\n ----------\\n batch : dict\\n\\n Returns\\n -------\\n batch : dict\\n Reformatted batch.\\n \\\"\\\"\\\"\\n # currently, we only support batch size of 1 during testing\\n assert len(batch) <= 1, \\\"Batch size 1 is required during testing!\\\"\\n batch = batch[0]\\n\\n output_dict = {}\\n\\n for cav_id, cav_content in batch.items():\\n output_dict.update({cav_id: {}})\\n # shape: (1, max_num, 7)\\n object_bbx_center = \\\\\\n torch.from_numpy(np.array([cav_content['object_bbx_center']]))\\n object_bbx_mask = \\\\\\n torch.from_numpy(np.array([cav_content['object_bbx_mask']]))\\n object_ids = cav_content['object_ids']\\n\\n # the anchor box is the same for all bounding boxes usually, thus\\n # we don't need the batch dimension.\\n if cav_content['anchor_box'] is not None:\\n output_dict[cav_id].update({'anchor_box':\\n torch.from_numpy(np.array(\\n cav_content[\\n 'anchor_box']))})\\n if self.visualize:\\n origin_lidar = [cav_content['origin_lidar']]\\n\\n # processed lidar dictionary\\n processed_lidar_torch_dict = \\\\\\n self.pre_processor.collate_batch(\\n [cav_content['processed_lidar']])\\n # label dictionary\\n label_torch_dict = \\\\\\n self.post_processor.collate_batch([cav_content['label_dict']])\\n\\n # save the transformation matrix (4, 4) to ego vehicle\\n transformation_matrix_torch = \\\\\\n torch.from_numpy(np.identity(4)).float()\\n\\n output_dict[cav_id].update({'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': object_bbx_mask,\\n 'processed_lidar': processed_lidar_torch_dict,\\n 'label_dict': label_torch_dict,\\n 'object_ids': object_ids,\\n 'transformation_matrix': transformation_matrix_torch})\\n\\n if self.visualize:\\n origin_lidar = \\\\\\n np.array(\\n downsample_lidar_minimum(pcd_np_list=origin_lidar))\\n origin_lidar = torch.from_numpy(origin_lidar)\\n output_dict[cav_id].update({'origin_lidar': origin_lidar})\\n\\n return output_dict\\n\\n def post_process(self, data_dict, output_dict):\\n \\\"\\\"\\\"\\n Process the outputs of the model to 2D/3D bounding box.\\n\\n Parameters\\n ----------\\n data_dict : dict\\n The dictionary containing the origin input data of model.\\n\\n output_dict :dict\\n The dictionary containing the output of the model.\\n\\n Returns\\n -------\\n pred_box_tensor : torch.Tensor\\n The tensor of prediction bounding box after NMS.\\n gt_box_tensor : torch.Tensor\\n The tensor of gt bounding box.\\n \\\"\\\"\\\"\\n pred_box_tensor, pred_score = \\\\\\n self.post_processor.post_process(data_dict, output_dict)\\n gt_box_tensor = self.post_processor.generate_gt_bbx(data_dict)\\n\\n return pred_box_tensor, pred_score, gt_box_tensor\\n\\n\\n\\\"\\\"\\\"\\nBasedataset class for lidar data pre-processing\\n\\\"\\\"\\\"\\n\\nimport os\\nimport math\\nfrom collections import OrderedDict\\n\\nimport torch\\nimport numpy as np\\nfrom torch.utils.data import Dataset\\n\\nimport v2xvit.utils.pcd_utils as pcd_utils\\nfrom v2xvit.data_utils.augmentor.data_augmentor import DataAugmentor\\nfrom v2xvit.hypes_yaml.yaml_utils import load_yaml\\nfrom v2xvit.utils.pcd_utils import downsample_lidar_minimum\\nfrom v2xvit.utils.transformation_utils import x1_to_x2\\n\\n\\nclass BaseDataset(Dataset):\\n \\\"\\\"\\\"\\n Base dataset for all kinds of fusion. Mainly used to assign correct\\n index and add noise.\\n\\n Parameters\\n __________\\n params : dict\\n The dictionary contains all parameters for training/testing.\\n\\n visualize : false\\n If set to true, the dataset is used for visualization.\\n\\n Attributes\\n ----------\\n scenario_database : OrderedDict\\n A structured dictionary contains all file information.\\n\\n len_record : list\\n The list to record each scenario's data length. This is used to\\n retrieve the correct index during training.\\n\\n \\\"\\\"\\\"\\n\\n def __init__(self, params, visualize, train=True):\\n self.params = params\\n self.visualize = visualize\\n self.train = train\\n\\n self.pre_processor = None\\n self.post_processor = None\\n self.data_augmentor = DataAugmentor(params['data_augment'],\\n train)\\n\\n # if the training/testing include noisy setting\\n if 'wild_setting' in params:\\n self.seed = params['wild_setting']['seed']\\n # whether to add time delay\\n self.async_flag = params['wild_setting']['async']\\n self.async_mode = \\\\\\n 'sim' if 'async_mode' not in params['wild_setting'] \\\\\\n else params['wild_setting']['async_mode']\\n self.async_overhead = params['wild_setting']['async_overhead']\\n\\n # localization error\\n self.loc_err_flag = params['wild_setting']['loc_err']\\n self.xyz_noise_std = params['wild_setting']['xyz_std']\\n self.ryp_noise_std = params['wild_setting']['ryp_std']\\n\\n # transmission data size\\n self.data_size = \\\\\\n params['wild_setting']['data_size'] \\\\\\n if 'data_size' in params['wild_setting'] else 0\\n self.transmission_speed = \\\\\\n params['wild_setting']['transmission_speed'] \\\\\\n if 'transmission_speed' in params['wild_setting'] else 27\\n self.backbone_delay = \\\\\\n params['wild_setting']['backbone_delay'] \\\\\\n if 'backbone_delay' in params['wild_setting'] else 0\\n\\n else:\\n self.async_flag = False\\n self.async_overhead = 0 # ms\\n self.async_mode = 'sim'\\n self.loc_err_flag = False\\n self.xyz_noise_std = 0\\n self.ryp_noise_std = 0\\n self.data_size = 0 # Mb\\n self.transmission_speed = 27 # Mbps\\n self.backbone_delay = 0 # ms\\n\\n if self.train:\\n root_dir = params['root_dir']\\n else:\\n root_dir = params['validate_dir']\\n\\n if 'max_cav' not in params['train_params']:\\n self.max_cav = 7\\n else:\\n self.max_cav = params['train_params']['max_cav']\\n\\n # first load all paths of different scenarios\\n scenario_folders = sorted([os.path.join(root_dir, x)\\n for x in os.listdir(root_dir) if\\n os.path.isdir(os.path.join(root_dir, x))])\\n # Structure: {scenario_id : {cav_1 : {timestamp1 : {yaml: path,\\n # lidar: path, cameras:list of path}}}}\\n self.scenario_database = OrderedDict()\\n self.len_record = []\\n\\n # loop over all scenarios\\n for (i, scenario_folder) in enumerate(scenario_folders):\\n self.scenario_database.update({i: OrderedDict()})\\n\\n # at least 1 cav should show up\\n cav_list = sorted([x for x in os.listdir(scenario_folder)\\n if os.path.isdir(\\n os.path.join(scenario_folder, x))])\\n assert len(cav_list) > 0\\n\\n # roadside unit data's id is always negative, so here we want to\\n # make sure they will be in the end of the list as they shouldn't\\n # be ego vehicle.\\n if int(cav_list[0]) < 0:\\n cav_list = cav_list[1:] + [cav_list[0]]\\n\\n # loop over all CAV data\\n for (j, cav_id) in enumerate(cav_list):\\n if j > self.max_cav - 1:\\n print('too many cavs')\\n break\\n self.scenario_database[i][cav_id] = OrderedDict()\\n\\n # save all yaml files to the dictionary\\n cav_path = os.path.join(scenario_folder, cav_id)\\n\\n # use the frame number as key, the full path as the values\\n yaml_files = \\\\\\n sorted([os.path.join(cav_path, x)\\n for x in os.listdir(cav_path) if\\n x.endswith('.yaml')])\\n timestamps = self.extract_timestamps(yaml_files)\\n\\n for timestamp in timestamps:\\n self.scenario_database[i][cav_id][timestamp] = \\\\\\n OrderedDict()\\n\\n yaml_file = os.path.join(cav_path,\\n timestamp + '.yaml')\\n lidar_file = os.path.join(cav_path,\\n timestamp + '.pcd')\\n camera_files = self.load_camera_files(cav_path, timestamp)\\n\\n self.scenario_database[i][cav_id][timestamp]['yaml'] = \\\\\\n yaml_file\\n self.scenario_database[i][cav_id][timestamp]['lidar'] = \\\\\\n lidar_file\\n self.scenario_database[i][cav_id][timestamp]['camera0'] = \\\\\\n camera_files\\n # Assume all cavs will have the same timestamps length. Thus\\n # we only need to calculate for the first vehicle in the\\n # scene.\\n if j == 0:\\n self.scenario_database[i][cav_id]['ego'] = True\\n if not self.len_record:\\n self.len_record.append(len(timestamps))\\n else:\\n prev_last = self.len_record[-1]\\n self.len_record.append(prev_last + len(timestamps))\\n else:\\n self.scenario_database[i][cav_id]['ego'] = False\\n\\n def __len__(self):\\n return self.len_record[-1]\\n\\n def __getitem__(self, idx):\\n \\\"\\\"\\\"\\n Abstract method, needs to be define by the children class.\\n \\\"\\\"\\\"\\n pass\\n\\n def retrieve_base_data(self, idx, cur_ego_pose_flag=True):\\n \\\"\\\"\\\"\\n Given the index, return the corresponding data.\\n\\n Parameters\\n ----------\\n idx : int\\n Index given by dataloader.\\n\\n cur_ego_pose_flag : bool\\n Indicate whether to use current timestamp ego pose to calculate\\n transformation matrix.\\n\\n Returns\\n -------\\n data : dict\\n The dictionary contains loaded yaml params and lidar data for\\n each cav.\\n \\\"\\\"\\\"\\n # we loop the accumulated length list to see get the scenario index\\n scenario_index = 0\\n for i, ele in enumerate(self.len_record):\\n if idx < ele:\\n scenario_index = i\\n break\\n scenario_database = self.scenario_database[scenario_index]\\n\\n # check the timestamp index\\n timestamp_index = idx if scenario_index == 0 else \\\\\\n idx - self.len_record[scenario_index - 1]\\n # retrieve the corresponding timestamp key\\n timestamp_key = self.return_timestamp_key(scenario_database,\\n timestamp_index)\\n # calculate distance to ego for each cav for time delay estimation\\n ego_cav_content = \\\\\\n self.calc_dist_to_ego(scenario_database, timestamp_key)\\n\\n data = OrderedDict()\\n # load files for all CAVs\\n for cav_id, cav_content in scenario_database.items():\\n data[cav_id] = OrderedDict()\\n data[cav_id]['ego'] = cav_content['ego']\\n\\n # calculate delay for this vehicle\\n timestamp_delay = \\\\\\n self.time_delay_calculation(cav_content['ego'])\\n\\n if timestamp_index - timestamp_delay <= 0:\\n timestamp_delay = timestamp_index\\n timestamp_index_delay = max(0, timestamp_index - timestamp_delay)\\n timestamp_key_delay = self.return_timestamp_key(scenario_database,\\n timestamp_index_delay)\\n # add time delay to vehicle parameters\\n data[cav_id]['time_delay'] = timestamp_delay\\n # load the corresponding data into the dictionary\\n data[cav_id]['params'] = self.reform_param(cav_content,\\n ego_cav_content,\\n timestamp_key,\\n timestamp_key_delay,\\n cur_ego_pose_flag)\\n data[cav_id]['lidar_np'] = \\\\\\n pcd_utils.pcd_to_np(cav_content[timestamp_key_delay]['lidar'])\\n return data\\n\\n @staticmethod\\n def extract_timestamps(yaml_files):\\n \\\"\\\"\\\"\\n Given the list of the yaml files, extract the mocked timestamps.\\n\\n Parameters\\n ----------\\n yaml_files : list\\n The full path of all yaml files of ego vehicle\\n\\n Returns\\n -------\\n timestamps : list\\n The list containing timestamps only.\\n \\\"\\\"\\\"\\n timestamps = []\\n\\n for file in yaml_files:\\n res = file.split('/')[-1]\\n\\n timestamp = res.replace('.yaml', '')\\n timestamps.append(timestamp)\\n\\n return timestamps\\n\\n @staticmethod\\n def return_timestamp_key(scenario_database, timestamp_index):\\n \\\"\\\"\\\"\\n Given the timestamp index, return the correct timestamp key, e.g.\\n 2 --> '000078'.\\n\\n Parameters\\n ----------\\n scenario_database : OrderedDict\\n The dictionary contains all contents in the current scenario.\\n\\n timestamp_index : int\\n The index for timestamp.\\n\\n Returns\\n -------\\n timestamp_key : str\\n The timestamp key saved in the cav dictionary.\\n \\\"\\\"\\\"\\n # get all timestamp keys\\n timestamp_keys = list(scenario_database.items())[0][1]\\n # retrieve the correct index\\n timestamp_key = list(timestamp_keys.items())[timestamp_index][0]\\n\\n return timestamp_key\\n\\n def calc_dist_to_ego(self, scenario_database, timestamp_key):\\n \\\"\\\"\\\"\\n Calculate the distance to ego for each cav.\\n \\\"\\\"\\\"\\n ego_lidar_pose = None\\n ego_cav_content = None\\n # Find ego pose first\\n for cav_id, cav_content in scenario_database.items():\\n if cav_content['ego']:\\n ego_cav_content = cav_content\\n ego_lidar_pose = \\\\\\n load_yaml(cav_content[timestamp_key]['yaml'])['lidar_pose']\\n break\\n\\n assert ego_lidar_pose is not None\\n\\n # calculate the distance\\n for cav_id, cav_content in scenario_database.items():\\n cur_lidar_pose = \\\\\\n load_yaml(cav_content[timestamp_key]['yaml'])['lidar_pose']\\n distance = \\\\\\n math.sqrt((cur_lidar_pose[0] -\\n ego_lidar_pose[0]) ** 2 +\\n (cur_lidar_pose[1] - ego_lidar_pose[1]) ** 2)\\n cav_content['distance_to_ego'] = distance\\n scenario_database.update({cav_id: cav_content})\\n\\n return ego_cav_content\\n\\n def time_delay_calculation(self, ego_flag):\\n \\\"\\\"\\\"\\n Calculate the time delay for a certain vehicle.\\n\\n Parameters\\n ----------\\n ego_flag : boolean\\n Whether the current cav is ego.\\n\\n Return\\n ------\\n time_delay : int\\n The time delay quantization.\\n \\\"\\\"\\\"\\n # there is not time delay for ego vehicle\\n if ego_flag:\\n return 0\\n # time delay real mode\\n if self.async_mode == 'real':\\n # noise/time is in ms unit\\n overhead_noise = np.random.uniform(0, self.async_overhead)\\n tc = self.data_size / self.transmission_speed * 1000\\n time_delay = int(overhead_noise + tc + self.backbone_delay)\\n elif self.async_mode == 'sim':\\n time_delay = np.abs(self.async_overhead)\\n\\n time_delay = time_delay // 100\\n return time_delay if self.async_flag else 0\\n\\n def add_loc_noise(self, pose, xyz_std, ryp_std):\\n \\\"\\\"\\\"\\n Add localization noise to the pose.\\n\\n Parameters\\n ----------\\n pose : list\\n x,y,z,roll,yaw,pitch\\n\\n xyz_std : float\\n std of the gaussian noise on xyz\\n\\n ryp_std : float\\n std of the gaussian noise\\n \\\"\\\"\\\"\\n np.random.seed(self.seed)\\n xyz_noise = np.random.normal(0, xyz_std, 3)\\n ryp_std = np.random.normal(0, ryp_std, 3)\\n noise_pose = [pose[0] + xyz_noise[0],\\n pose[1] + xyz_noise[1],\\n pose[2] + xyz_noise[2],\\n pose[3],\\n pose[4] + ryp_std[1],\\n pose[5]]\\n return noise_pose\\n\\n def reform_param(self, cav_content, ego_content, timestamp_cur,\\n timestamp_delay, cur_ego_pose_flag):\\n \\\"\\\"\\\"\\n Reform the data params with current timestamp object groundtruth and\\n delay timestamp LiDAR pose.\\n\\n Parameters\\n ----------\\n cav_content : dict\\n Dictionary that contains all file paths in the current cav/rsu.\\n\\n ego_content : dict\\n Ego vehicle content.\\n\\n timestamp_cur : str\\n The current timestamp.\\n\\n timestamp_delay : str\\n The delayed timestamp.\\n\\n cur_ego_pose_flag : bool\\n Whether use current ego pose to calculate transformation matrix.\\n\\n Return\\n ------\\n The merged parameters.\\n \\\"\\\"\\\"\\n cur_params = load_yaml(cav_content[timestamp_cur]['yaml'])\\n delay_params = load_yaml(cav_content[timestamp_delay]['yaml'])\\n\\n cur_ego_params = load_yaml(ego_content[timestamp_cur]['yaml'])\\n delay_ego_params = load_yaml(ego_content[timestamp_delay]['yaml'])\\n\\n # we need to calculate the transformation matrix from cav to ego\\n # at the delayed timestamp\\n delay_cav_lidar_pose = delay_params['lidar_pose']\\n delay_ego_lidar_pose = delay_ego_params[\\\"lidar_pose\\\"]\\n\\n cur_ego_lidar_pose = cur_ego_params['lidar_pose']\\n cur_cav_lidar_pose = cur_params['lidar_pose']\\n\\n if not cav_content['ego'] and self.loc_err_flag:\\n delay_cav_lidar_pose = self.add_loc_noise(delay_cav_lidar_pose,\\n self.xyz_noise_std,\\n self.ryp_noise_std)\\n cur_cav_lidar_pose = self.add_loc_noise(cur_cav_lidar_pose,\\n self.xyz_noise_std,\\n self.ryp_noise_std)\\n\\n if cur_ego_pose_flag:\\n transformation_matrix = x1_to_x2(delay_cav_lidar_pose,\\n cur_ego_lidar_pose)\\n spatial_correction_matrix = np.eye(4)\\n else:\\n transformation_matrix = x1_to_x2(delay_cav_lidar_pose,\\n delay_ego_lidar_pose)\\n spatial_correction_matrix = x1_to_x2(delay_ego_lidar_pose,\\n cur_ego_lidar_pose)\\n # This is only used for late fusion, as it did the transformation\\n # in the postprocess, so we want the gt object transformation use\\n # the correct one\\n gt_transformation_matrix = x1_to_x2(cur_cav_lidar_pose,\\n cur_ego_lidar_pose)\\n\\n # we always use current timestamp's gt bbx to gain a fair evaluation\\n delay_params['vehicles'] = cur_params['vehicles']\\n delay_params['transformation_matrix'] = transformation_matrix\\n delay_params['gt_transformation_matrix'] = \\\\\\n gt_transformation_matrix\\n delay_params['spatial_correction_matrix'] = spatial_correction_matrix\\n\\n return delay_params\\n\\n @staticmethod\\n def load_camera_files(cav_path, timestamp):\\n \\\"\\\"\\\"\\n Retrieve the paths to all camera files.\\n\\n Parameters\\n ----------\\n cav_path : str\\n The full file path of current cav.\\n\\n timestamp : str\\n Current timestamp\\n\\n Returns\\n -------\\n camera_files : list\\n The list containing all camera png file paths.\\n \\\"\\\"\\\"\\n camera0_file = os.path.join(cav_path,\\n timestamp + '_camera0.png')\\n camera1_file = os.path.join(cav_path,\\n timestamp + '_camera1.png')\\n camera2_file = os.path.join(cav_path,\\n timestamp + '_camera2.png')\\n camera3_file = os.path.join(cav_path,\\n timestamp + '_camera3.png')\\n return [camera0_file, camera1_file, camera2_file, camera3_file]\\n\\n def project_points_to_bev_map(self, points, ratio=0.1):\\n \\\"\\\"\\\"\\n Project points to BEV occupancy map with default ratio=0.1.\\n\\n Parameters\\n ----------\\n points : np.ndarray\\n (N, 3) / (N, 4)\\n\\n ratio : float\\n Discretization parameters. Default is 0.1.\\n\\n Returns\\n -------\\n bev_map : np.ndarray\\n BEV occupancy map including projected points\\n with shape (img_row, img_col).\\n\\n \\\"\\\"\\\"\\n return self.pre_processor.project_points_to_bev_map(points, ratio)\\n\\n def augment(self, lidar_np, object_bbx_center, object_bbx_mask):\\n \\\"\\\"\\\"\\n Data augmentation operation.\\n \\\"\\\"\\\"\\n tmp_dict = {'lidar_np': lidar_np,\\n 'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': object_bbx_mask}\\n tmp_dict = self.data_augmentor.forward(tmp_dict)\\n\\n lidar_np = tmp_dict['lidar_np']\\n object_bbx_center = tmp_dict['object_bbx_center']\\n object_bbx_mask = tmp_dict['object_bbx_mask']\\n\\n return lidar_np, object_bbx_center, object_bbx_mask\\n\\n def collate_batch_train(self, batch):\\n \\\"\\\"\\\"\\n Customized collate function for pytorch dataloader during training\\n for late fusion dataset.\\n\\n Parameters\\n ----------\\n batch : dict\\n\\n Returns\\n -------\\n batch : dict\\n Reformatted batch.\\n \\\"\\\"\\\"\\n # during training, we only care about ego.\\n output_dict = {'ego': {}}\\n\\n object_bbx_center = []\\n object_bbx_mask = []\\n processed_lidar_list = []\\n label_dict_list = []\\n\\n if self.visualize:\\n origin_lidar = []\\n\\n for i in range(len(batch)):\\n ego_dict = batch[i]['ego']\\n object_bbx_center.append(ego_dict['object_bbx_center'])\\n object_bbx_mask.append(ego_dict['object_bbx_mask'])\\n processed_lidar_list.append(ego_dict['processed_lidar'])\\n label_dict_list.append(ego_dict['label_dict'])\\n\\n if self.visualize:\\n origin_lidar.append(ego_dict['origin_lidar'])\\n\\n # convert to numpy, (B, max_num, 7)\\n object_bbx_center = torch.from_numpy(np.array(object_bbx_center))\\n object_bbx_mask = torch.from_numpy(np.array(object_bbx_mask))\\n\\n processed_lidar_torch_dict = \\\\\\n self.pre_processor.collate_batch(processed_lidar_list)\\n label_torch_dict = \\\\\\n self.post_processor.collate_batch(label_dict_list)\\n output_dict['ego'].update({'object_bbx_center': object_bbx_center,\\n 'object_bbx_mask': object_bbx_mask,\\n 'processed_lidar': processed_lidar_torch_dict,\\n 'label_dict': label_torch_dict})\\n if self.visualize:\\n origin_lidar = \\\\\\n np.array(downsample_lidar_minimum(pcd_np_list=origin_lidar))\\n origin_lidar = torch.from_numpy(origin_lidar)\\n output_dict['ego'].update({'origin_lidar': origin_lidar})\\n\\n return output_dict\\n\\n def visualize_result(self, pred_box_tensor,\\n gt_tensor,\\n pcd,\\n show_vis,\\n save_path,\\n dataset=None):\\n self.post_processor.visualize(pred_box_tensor,\\n gt_tensor,\\n pcd,\\n show_vis,\\n save_path,\\n dataset=dataset)\\n\\n\\nfrom v2xvit.data_utils.datasets.late_fusion_dataset import LateFusionDataset\\nfrom v2xvit.data_utils.datasets.early_fusion_dataset import EarlyFusionDataset\\nfrom v2xvit.data_utils.datasets.intermediate_fusion_dataset import IntermediateFusionDataset\\n\\n__all__ = {\\n 'LateFusionDataset': LateFusionDataset,\\n 'EarlyFusionDataset': EarlyFusionDataset,\\n 'IntermediateFusionDataset': IntermediateFusionDataset\\n}\\n\\n# the final range for evaluation\\nGT_RANGE = [-140, -40, -3, 140, 40, 1]\\n# The communication range for cavs\\nCOM_RANGE = 70\\n\\n\\ndef build_dataset(dataset_cfg, visualize=False, train=True):\\n dataset_name = dataset_cfg['fusion']['core_method']\\n error_message = f\\\"{dataset_name} is not found. \\\" \\\\\\n f\\\"Please add your processor file's name in opencood/\\\" \\\\\\n f\\\"data_utils/datasets/init.py\\\"\\n assert dataset_name in ['LateFusionDataset', 'EarlyFusionDataset',\\n 'IntermediateFusionDataset'], error_message\\n\\n dataset = __all__[dataset_name](\\n params=dataset_cfg,\\n visualize=visualize,\\n train=train\\n )\\n\\n return dataset\\n\\n\\n\\\"\\\"\\\"\\nClass for data augmentation\\n\\\"\\\"\\\"\\nfrom functools import partial\\n\\nfrom v2xvit.data_utils.augmentor import augment_utils\\n\\n\\nclass DataAugmentor(object):\\n \\\"\\\"\\\"\\n Data Augmentor.\\n\\n Parameters\\n ----------\\n augment_config : list\\n A list of augmentation configuration.\\n\\n Attributes\\n ----------\\n data_augmentor_queue : list\\n The list of data augmented functions.\\n \\\"\\\"\\\"\\n\\n def __init__(self, augment_config, train=True):\\n self.data_augmentor_queue = []\\n self.train = train\\n\\n for cur_cfg in augment_config:\\n cur_augmentor = getattr(self, cur_cfg['NAME'])(config=cur_cfg)\\n self.data_augmentor_queue.append(cur_augmentor)\\n\\n def random_world_flip(self, data_dict=None, config=None):\\n if data_dict is None:\\n return partial(self.random_world_flip, config=config)\\n\\n gt_boxes, gt_mask, points = data_dict['object_bbx_center'], \\\\\\n data_dict['object_bbx_mask'], \\\\\\n data_dict['lidar_np']\\n gt_boxes_valid = gt_boxes[gt_mask == 1]\\n\\n for cur_axis in config['ALONG_AXIS_LIST']:\\n assert cur_axis in ['x', 'y']\\n gt_boxes_valid, points = getattr(augment_utils,\\n 'random_flip_along_%s' % cur_axis)(\\n gt_boxes_valid, points,\\n )\\n\\n gt_boxes[:gt_boxes_valid.shape[0], :] = gt_boxes_valid\\n\\n data_dict['object_bbx_center'] = gt_boxes\\n data_dict['object_bbx_mask'] = gt_mask\\n data_dict['lidar_np'] = points\\n\\n return data_dict\\n\\n def random_world_rotation(self, data_dict=None, config=None):\\n if data_dict is None:\\n return partial(self.random_world_rotation, config=config)\\n\\n rot_range = config['WORLD_ROT_ANGLE']\\n if not isinstance(rot_range, list):\\n rot_range = [-rot_range, rot_range]\\n\\n gt_boxes, gt_mask, points = data_dict['object_bbx_center'], \\\\\\n data_dict['object_bbx_mask'], \\\\\\n data_dict['lidar_np']\\n gt_boxes_valid = gt_boxes[gt_mask == 1]\\n gt_boxes_valid, points = augment_utils.global_rotation(\\n gt_boxes_valid, points, rot_range=rot_range\\n )\\n gt_boxes[:gt_boxes_valid.shape[0], :] = gt_boxes_valid\\n\\n data_dict['object_bbx_center'] = gt_boxes\\n data_dict['object_bbx_mask'] = gt_mask\\n data_dict['lidar_np'] = points\\n\\n return data_dict\\n\\n def random_world_scaling(self, data_dict=None, config=None):\\n if data_dict is None:\\n return partial(self.random_world_scaling, config=config)\\n\\n gt_boxes, gt_mask, points = data_dict['object_bbx_center'], \\\\\\n data_dict['object_bbx_mask'], \\\\\\n data_dict['lidar_np']\\n gt_boxes_valid = gt_boxes[gt_mask == 1]\\n\\n gt_boxes_valid, points = augment_utils.global_scaling(\\n gt_boxes_valid, points, config['WORLD_SCALE_RANGE']\\n )\\n gt_boxes[:gt_boxes_valid.shape[0], :] = gt_boxes_valid\\n\\n data_dict['object_bbx_center'] = gt_boxes\\n data_dict['object_bbx_mask'] = gt_mask\\n data_dict['lidar_np'] = points\\n\\n return data_dict\\n\\n def forward(self, data_dict):\\n \\\"\\\"\\\"\\n Args:\\n data_dict:\\n points: (N, 3 + C_in)\\n gt_boxes: optional, (N, 7) [x, y, z, dx, dy, dz, heading]\\n gt_names: optional, (N), string\\n ...\\n\\n Returns:\\n \\\"\\\"\\\"\\n if self.train:\\n for cur_augmentor in self.data_augmentor_queue:\\n data_dict = cur_augmentor(data_dict=data_dict)\\n\\n return data_dict\\n\\n\\nimport numpy as np\\n\\nfrom v2xvit.utils import common_utils\\n\\n\\ndef random_flip_along_x(gt_boxes, points):\\n \\\"\\\"\\\"\\n Args:\\n gt_boxes: (N, 7 + C), [x, y, z, dx, dy, dz, heading, [vx], [vy]]\\n points: (M, 3 + C)\\n Returns:\\n \\\"\\\"\\\"\\n enable = np.random.choice([False, True], replace=False, p=[0.5, 0.5])\\n if enable:\\n gt_boxes[:, 1] = -gt_boxes[:, 1]\\n gt_boxes[:, 6] = -gt_boxes[:, 6]\\n points[:, 1] = -points[:, 1]\\n\\n if gt_boxes.shape[1] > 7:\\n gt_boxes[:, 8] = -gt_boxes[:, 8]\\n\\n return gt_boxes, points\\n\\n\\ndef random_flip_along_y(gt_boxes, points):\\n \\\"\\\"\\\"\\n Args:\\n gt_boxes: (N, 7 + C), [x, y, z, dx, dy, dz, heading, [vx], [vy]]\\n points: (M, 3 + C)\\n Returns:\\n \\\"\\\"\\\"\\n enable = np.random.choice([False, True], replace=False, p=[0.5, 0.5])\\n if enable:\\n gt_boxes[:, 0] = -gt_boxes[:, 0]\\n gt_boxes[:, 6] = -(gt_boxes[:, 6] + np.pi)\\n points[:, 0] = -points[:, 0]\\n\\n if gt_boxes.shape[1] > 7:\\n gt_boxes[:, 7] = -gt_boxes[:, 7]\\n\\n return gt_boxes, points\\n\\n\\ndef global_rotation(gt_boxes, points, rot_range):\\n \\\"\\\"\\\"\\n Args:\\n gt_boxes: (N, 7 + C), [x, y, z, dx, dy, dz, heading, [vx], [vy]]\\n points: (M, 3 + C),\\n rot_range: [min, max]\\n Returns:\\n \\\"\\\"\\\"\\n noise_rotation = np.random.uniform(rot_range[0],\\n rot_range[1])\\n points = common_utils.rotate_points_along_z(points[np.newaxis, :, :],\\n np.array([noise_rotation]))[0]\\n\\n gt_boxes[:, 0:3] = \\\\\\n common_utils.rotate_points_along_z(gt_boxes[np.newaxis, :, 0:3],\\n np.array([noise_rotation]))[0]\\n gt_boxes[:, 6] += noise_rotation\\n\\n if gt_boxes.shape[1] > 7:\\n gt_boxes[:, 7:9] = common_utils.rotate_points_along_z(\\n np.hstack((gt_boxes[:, 7:9], np.zeros((gt_boxes.shape[0], 1))))[\\n np.newaxis, :, :],\\n np.array([noise_rotation]))[0][:, 0:2]\\n\\n return gt_boxes, points\\n\\n\\ndef global_scaling(gt_boxes, points, scale_range):\\n \\\"\\\"\\\"\\n Args:\\n gt_boxes: (N, 7), [x, y, z, dx, dy, dz, heading]\\n points: (M, 3 + C),\\n scale_range: [min, max]\\n Returns:\\n \\\"\\\"\\\"\\n if scale_range[1] - scale_range[0] < 1e-3:\\n return gt_boxes, points\\n noise_scale = np.random.uniform(scale_range[0], scale_range[1])\\n points[:, :3] *= noise_scale\\n gt_boxes[:, :6] *= noise_scale\\n\\n return gt_boxes, points\\n\\n\\n\\n\\nimport re\\nimport yaml\\nimport os\\nimport math\\n\\nimport numpy as np\\n\\n\\ndef load_yaml(file, opt=None):\\n \\\"\\\"\\\"\\n Load yaml file and return a dictionary.\\n\\n Parameters\\n ----------\\n file : string\\n yaml file path.\\n\\n opt : argparser\\n Argparser.\\n Returns\\n -------\\n param : dict\\n A dictionary that contains defined parameters.\\n \\\"\\\"\\\"\\n if opt and opt.model_dir:\\n file = os.path.join(opt.model_dir, 'config.yaml')\\n\\n stream = open(file, 'r')\\n loader = yaml.Loader\\n loader.add_implicit_resolver(\\n u'tag:yaml.org,2002:float',\\n re.compile(u'''^(?:\\n [-+]?(?:[0-9][0-9_]*)\\\\\\\\.[0-9_]*(?:[eE][-+]?[0-9]+)?\\n |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)\\n |\\\\\\\\.[0-9_]+(?:[eE][-+][0-9]+)?\\n |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\\\\\\\.[0-9_]*\\n |[-+]?\\\\\\\\.(?:inf|Inf|INF)\\n |\\\\\\\\.(?:nan|NaN|NAN))$''', re.X),\\n list(u'-+0123456789.'))\\n param = yaml.load(stream, Loader=loader)\\n if \\\"yaml_parser\\\" in param:\\n param = eval(param[\\\"yaml_parser\\\"])(param)\\n\\n return param\\n\\n\\ndef load_voxel_params(param):\\n \\\"\\\"\\\"\\n Based on the lidar range and resolution of voxel, calcuate the anchor box\\n and target resolution.\\n\\n Parameters\\n ----------\\n param : dict\\n Original loaded parameter dictionary.\\n\\n Returns\\n -------\\n param : dict\\n Modified parameter dictionary with new attribute `anchor_args[W][H][L]`\\n \\\"\\\"\\\"\\n anchor_args = param['postprocess']['anchor_args']\\n cav_lidar_range = anchor_args['cav_lidar_range']\\n voxel_size = param['preprocess']['args']['voxel_size']\\n\\n vw = voxel_size[0]\\n vh = voxel_size[1]\\n vd = voxel_size[2]\\n\\n anchor_args['vw'] = vw\\n anchor_args['vh'] = vh\\n anchor_args['vd'] = vd\\n\\n anchor_args['W'] = int((cav_lidar_range[3] - cav_lidar_range[0]) / vw)\\n anchor_args['H'] = int((cav_lidar_range[4] - cav_lidar_range[1]) / vh)\\n anchor_args['D'] = int((cav_lidar_range[5] - cav_lidar_range[2]) / vd)\\n\\n param['postprocess'].update({'anchor_args': anchor_args})\\n # sometimes we just want to visualize the data without implementing model\\n if 'model' in param:\\n param['model']['args']['W'] = anchor_args['W']\\n param['model']['args']['H'] = anchor_args['H']\\n param['model']['args']['D'] = anchor_args['D']\\n return param\\n\\n\\ndef load_point_pillar_params(param):\\n \\\"\\\"\\\"\\n Based on the lidar range and resolution of voxel, calcuate the anchor box\\n and target resolution.\\n\\n Parameters\\n ----------\\n param : dict\\n Original loaded parameter dictionary.\\n\\n Returns\\n -------\\n param : dict\\n Modified parameter dictionary with new attribute.\\n \\\"\\\"\\\"\\n cav_lidar_range = param['preprocess']['cav_lidar_range']\\n voxel_size = param['preprocess']['args']['voxel_size']\\n\\n grid_size = (np.array(cav_lidar_range[3:6]) - np.array(\\n cav_lidar_range[0:3])) / \\\\\\n np.array(voxel_size)\\n grid_size = np.round(grid_size).astype(np.int64)\\n param['model']['args']['point_pillar_scatter']['grid_size'] = grid_size\\n\\n anchor_args = param['postprocess']['anchor_args']\\n\\n vw = voxel_size[0]\\n vh = voxel_size[1]\\n vd = voxel_size[2]\\n\\n anchor_args['vw'] = vw\\n anchor_args['vh'] = vh\\n anchor_args['vd'] = vd\\n\\n anchor_args['W'] = math.ceil((cav_lidar_range[3] - cav_lidar_range[0]) / vw)\\n anchor_args['H'] = math.ceil((cav_lidar_range[4] - cav_lidar_range[1]) / vh)\\n anchor_args['D'] = math.ceil((cav_lidar_range[5] - cav_lidar_range[2]) / vd)\\n\\n param['postprocess'].update({'anchor_args': anchor_args})\\n\\n return param\\n\\ndef load_second_params(param):\\n \\\"\\\"\\\"\\n Based on the lidar range and resolution of voxel, calcuate the anchor box\\n and target resolution.\\n\\n Parameters\\n ----------\\n param : dict\\n Original loaded parameter dictionary.\\n\\n Returns\\n -------\\n param : dict\\n Modified parameter dictionary with new attribute.\\n \\\"\\\"\\\"\\n cav_lidar_range = param['preprocess']['cav_lidar_range']\\n voxel_size = param['preprocess']['args']['voxel_size']\\n\\n grid_size = (np.array(cav_lidar_range[3:6]) - np.array(\\n cav_lidar_range[0:3])) / \\\\\\n np.array(voxel_size)\\n grid_size = np.round(grid_size).astype(np.int64)\\n param['model']['args']['grid_size'] = grid_size\\n\\n anchor_args = param['postprocess']['anchor_args']\\n\\n vw = voxel_size[0]\\n vh = voxel_size[1]\\n vd = voxel_size[2]\\n\\n anchor_args['vw'] = vw\\n anchor_args['vh'] = vh\\n anchor_args['vd'] = vd\\n\\n anchor_args['W'] = math.ceil((cav_lidar_range[3] - cav_lidar_range[0]) / vw)\\n anchor_args['H'] = math.ceil((cav_lidar_range[4] - cav_lidar_range[1]) / vh)\\n anchor_args['D'] = math.ceil((cav_lidar_range[5] - cav_lidar_range[2]) / vd)\\n\\n param['postprocess'].update({'anchor_args': anchor_args})\\n\\n return param\\n\\ndef load_bev_params(param):\\n \\\"\\\"\\\"\\n Load bev related geometry parameters s.t. boundary, resolutions, input\\n shape, target shape etc.\\n\\n Parameters\\n ----------\\n param : dict\\n Original loaded parameter dictionary.\\n\\n Returns\\n -------\\n param : dict\\n Modified parameter dictionary with new attribute `geometry_param`.\\n\\n \\\"\\\"\\\"\\n res = param[\\\"preprocess\\\"][\\\"args\\\"][\\\"res\\\"]\\n L1, W1, H1, L2, W2, H2 = param[\\\"preprocess\\\"][\\\"cav_lidar_range\\\"]\\n downsample_rate = param[\\\"preprocess\\\"][\\\"args\\\"][\\\"downsample_rate\\\"]\\n\\n def f(low, high, r):\\n return int((high - low) / r)\\n\\n input_shape = (\\n int((f(L1, L2, res))),\\n int((f(W1, W2, res))),\\n int((f(H1, H2, res)) + 1)\\n )\\n label_shape = (\\n int(input_shape[0] / downsample_rate),\\n int(input_shape[1] / downsample_rate),\\n 7\\n )\\n geometry_param = {\\n 'L1': L1,\\n 'L2': L2,\\n 'W1': W1,\\n 'W2': W2,\\n 'H1': H1,\\n 'H2': H2,\\n \\\"downsample_rate\\\": downsample_rate,\\n \\\"input_shape\\\": input_shape,\\n \\\"label_shape\\\": label_shape,\\n \\\"res\\\": res\\n }\\n param[\\\"preprocess\\\"][\\\"geometry_param\\\"] = geometry_param\\n param[\\\"postprocess\\\"][\\\"geometry_param\\\"] = geometry_param\\n param[\\\"model\\\"][\\\"args\\\"][\\\"geometry_param\\\"] = geometry_param\\n return param\\n\\n\\ndef save_yaml(data, save_name):\\n \\\"\\\"\\\"\\n Save the dictionary into a yaml file.\\n\\n Parameters\\n ----------\\n data : dict\\n The dictionary contains all data.\\n\\n save_name : string\\n Full path of the output yaml file.\\n \\\"\\\"\\\"\\n\\n with open(save_name, 'w') as outfile:\\n yaml.dump(data, outfile, default_flow_style=False)\",\"difficulty\":\"easy\",\"domain\":\"Code Repository Understanding\",\"length\":\"short\",\"question\":\"Which realistic factor in collaborative perception does this algorithm model mainly address?\",\"sub_domain\":\"Code repo QA\"}","display_format":"text","language":"","answer_status":"published","assets":[],"source_url":"https://huggingface.co/datasets/zai-org/LongBench-v2","history":"initial import","indexing_mode":"noindex","subproblems":[],"grids":[]}