diff --git a/model_examples/BEVFusion/bevfusion.patch b/model_examples/BEVFusion/bevfusion.patch index b2a41ed2e632dc295e4ef4bb45032f267f2b3026..64e42ce4d4b0c7d9e17296608c42f0ad646a50ac 100644 --- a/model_examples/BEVFusion/bevfusion.patch +++ b/model_examples/BEVFusion/bevfusion.patch @@ -12,264 +12,42 @@ index 56e8440b..b3a6382a 100644 if ground_plane is not None: xyz = sampled_gt_bboxes[:, :3] diff --git a/mmdet3d/models/layers/sparse_block.py b/mmdet3d/models/layers/sparse_block.py -index 6ed7c8f4..6a5ba828 100644 +index 6ed7c8f4..13f69b0d 100644 --- a/mmdet3d/models/layers/sparse_block.py +++ b/mmdet3d/models/layers/sparse_block.py -@@ -1,224 +1,349 @@ --# Copyright (c) OpenMMLab. All rights reserved. --from typing import Optional, Tuple, Union -- --from mmcv.cnn import build_conv_layer, build_norm_layer +@@ -2,17 +2,22 @@ + from typing import Optional, Tuple, Union + + from mmcv.cnn import build_conv_layer, build_norm_layer -from mmdet.models.backbones.resnet import BasicBlock, Bottleneck --from torch import nn -- --from mmdet3d.utils import OptConfigType --from .spconv import IS_SPCONV2_AVAILABLE -- --if IS_SPCONV2_AVAILABLE: -- from spconv.pytorch import SparseConvTensor, SparseModule, SparseSequential --else: -- from mmcv.ops import SparseConvTensor, SparseModule, SparseSequential -- -- --def replace_feature(out: SparseConvTensor, -- new_features: SparseConvTensor) -> SparseConvTensor: -- if 'replace_feature' in out.__dir__(): -- # spconv 2.x behaviour -- return out.replace_feature(new_features) -- else: -- out.features = new_features -- return out -- -- --class SparseBottleneck(Bottleneck, SparseModule): -- """Sparse bottleneck block for PartA^2. -- -- Bottleneck block implemented with submanifold sparse convolution. -- -- Args: -- inplanes (int): Inplanes of block. -- planes (int): Planes of block. -- stride (int or Tuple[int]): Stride of the first block. Defaults to 1. -- downsample (Module, optional): Down sample module for block. -- Defaults to None. -- indice_key (str): Indice key for spconv. Default to None. -- conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -- convolution layer. Defaults to None. -- norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -- normalization layer. Defaults to None. -- """ -- -- expansion = 4 -- -- def __init__(self, -- inplanes: int, -- planes: int, -- stride: Union[int, Tuple[int]] = 1, -- downsample: nn.Module = None, -- indice_key=None, -- conv_cfg: OptConfigType = None, -- norm_cfg: OptConfigType = None) -> None: -- -- SparseModule.__init__(self) -- if conv_cfg is None: -- conv_cfg = dict(type='SubMConv3d') -- conv_cfg.setdefault('indice_key', indice_key) -- if norm_cfg is None: -- norm_cfg = dict(type='BN1d') -- Bottleneck.__init__( -- self, -- inplanes, -- planes, -- stride=stride, -- downsample=downsample, -- conv_cfg=conv_cfg, -- norm_cfg=norm_cfg) -- -- def forward(self, x: SparseConvTensor) -> SparseConvTensor: -- identity = x.features -- -- out = self.conv1(x) -- out = replace_feature(out, self.bn1(out.features)) -- out = replace_feature(out, self.relu(out.features)) -- -- out = self.conv2(out) -- out = replace_feature(out, self.bn2(out.features)) -- out = replace_feature(out, self.relu(out.features)) -- -- out = self.conv3(out) -- out = replace_feature(out, self.bn3(out.features)) -- -- if self.downsample is not None: -- identity = self.downsample(x).features -- -- out = replace_feature(out, out.features + identity) -- out = replace_feature(out, self.relu(out.features)) -- -- return out -- -- --class SparseBasicBlock(BasicBlock, SparseModule): -- """Sparse basic block for PartA^2. -- -- Sparse basic block implemented with submanifold sparse convolution. -- -- Args: -- inplanes (int): Inplanes of block. -- planes (int): Planes of block. -- stride (int or Tuple[int]): Stride of the first block. Defaults to 1. -- downsample (Module, optional): Down sample module for block. -- Defaults to None. -- indice_key (str): Indice key for spconv. Default to None. -- conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -- convolution layer. Defaults to None. -- norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -- normalization layer. Defaults to None. -- """ -- -- expansion = 1 -- -- def __init__(self, -- inplanes: int, -- planes: int, -- stride: Union[int, Tuple[int]] = 1, -- downsample: nn.Module = None, -- indice_key: Optional[str] = None, -- conv_cfg: OptConfigType = None, -- norm_cfg: OptConfigType = None) -> None: -- SparseModule.__init__(self) -- if conv_cfg is None: -- conv_cfg = dict(type='SubMConv3d') -- conv_cfg.setdefault('indice_key', indice_key) -- if norm_cfg is None: -- norm_cfg = dict(type='BN1d') -- BasicBlock.__init__( -- self, -- inplanes, -- planes, -- stride=stride, -- downsample=downsample, -- conv_cfg=conv_cfg, -- norm_cfg=norm_cfg) -- -- def forward(self, x: SparseConvTensor) -> SparseConvTensor: -- identity = x.features -- -- assert x.features.dim() == 2, f'x.features.dim()={x.features.dim()}' -- out = self.conv1(x) -- out = replace_feature(out, self.norm1(out.features)) -- out = replace_feature(out, self.relu(out.features)) -- -- out = self.conv2(out) -- out = replace_feature(out, self.norm2(out.features)) -- -- if self.downsample is not None: -- identity = self.downsample(x).features -- -- out = replace_feature(out, out.features + identity) -- out = replace_feature(out, self.relu(out.features)) -- -- return out -- -- --def make_sparse_convmodule(in_channels: int, -- out_channels: int, -- kernel_size: Union[int, Tuple[int]], -- indice_key: Optional[str] = None, -- stride: Union[int, Tuple[int]] = 1, -- padding: Union[int, Tuple[int]] = 0, -- conv_type: str = 'SubMConv3d', -- norm_cfg: OptConfigType = None, -- order: Tuple[str] = ('conv', 'norm', 'act'), -- **kwargs) -> SparseSequential: -- """Make sparse convolution module. -- -- Args: -- in_channels (int): The number of input channels. -- out_channels (int): The number of out channels. -- kernel_size (int | Tuple[int]): Kernel size of convolution. -- indice_key (str): The indice key used for sparse tensor. -- stride (int or tuple[int]): The stride of convolution. -- padding (int or tuple[int]): The padding number of input. -- conv_type (str): Sparse conv type in spconv. Defaults to 'SubMConv3d'. -- norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -- normalization layer. Defaults to None. -- order (Tuple[str]): The order of conv/norm/activation layers. It is a -- sequence of "conv", "norm" and "act". Common examples are -- ("conv", "norm", "act") and ("act", "conv", "norm"). -- Defaults to ('conv', 'norm', 'act'). -- -- Returns: -- spconv.SparseSequential: sparse convolution module. -- """ -- assert isinstance(order, tuple) and len(order) <= 3 -- assert set(order) | {'conv', 'norm', 'act'} == {'conv', 'norm', 'act'} -- -- conv_cfg = dict(type=conv_type, indice_key=indice_key) -- if norm_cfg is None: -- norm_cfg = dict(type='BN1d') -- -- layers = list() -- for layer in order: -- if layer == 'conv': -- if conv_type not in [ -- 'SparseInverseConv3d', 'SparseInverseConv2d', -- 'SparseInverseConv1d' -- ]: -- layers.append( -- build_conv_layer( -- conv_cfg, -- in_channels, -- out_channels, -- kernel_size, -- stride=stride, -- padding=padding, -- bias=False)) -- else: -- layers.append( -- build_conv_layer( -- conv_cfg, -- in_channels, -- out_channels, -- kernel_size, -- bias=False)) -- elif layer == 'norm': -- layers.append(build_norm_layer(norm_cfg, out_channels)[1]) -- elif layer == 'act': -- layers.append(nn.ReLU(inplace=True)) -- -- layers = SparseSequential(*layers) -- return layers -+# Copyright (c) OpenMMLab. All rights reserved. -+from typing import Optional, Tuple, Union -+ -+from mmcv.cnn import build_conv_layer, build_norm_layer +import inspect +from mmengine.model import BaseModule +import torch -+from torch import nn + from torch import nn +import torch.utils.checkpoint as cp -+ -+from mmdet3d.utils import OptConfigType + + from mmdet3d.utils import OptConfigType +-from .spconv import IS_SPCONV2_AVAILABLE +from mx_driving.spconv import SparseSequential, SubMConv3d, SparseConv3d, SparseModule +from mmdet.models.backbones.resnet import BasicBlock, Bottleneck +from mmengine.registry import Registry +from mx_driving.spconv import SparseConvTensor -+ -+ + +-if IS_SPCONV2_AVAILABLE: +- from spconv.pytorch import SparseConvTensor, SparseModule, SparseSequential +-else: +- from mmcv.ops import SparseConvTensor, SparseModule, SparseSequential + +MODELS = Registry('Sparse conv layer') +MODELS.register_module('SubMConv3d', module=SubMConv3d) +MODELS.register_module('SparseConv3d', module=SparseConv3d) -+ -+def replace_feature(out: SparseConvTensor, -+ new_features: SparseConvTensor) -> SparseConvTensor: -+ if 'replace_feature' in out.__dir__(): -+ # spconv 2.x behaviour -+ return out.replace_feature(new_features) -+ else: -+ out.features = new_features -+ return out -+ + + def replace_feature(out: SparseConvTensor, + new_features: SparseConvTensor) -> SparseConvTensor: +@@ -23,6 +28,87 @@ def replace_feature(out: SparseConvTensor, + out.features = new_features + return out + +class BasicBlock(BaseModule): + expansion = 1 + @@ -351,205 +129,31 @@ index 6ed7c8f4..6a5ba828 100644 + out = self.relu(out) + + return out -+ -+class SparseBottleneck(Bottleneck, SparseModule): -+ """Sparse bottleneck block for PartA^2. -+ -+ Bottleneck block implemented with submanifold sparse convolution. -+ -+ Args: -+ inplanes (int): Inplanes of block. -+ planes (int): Planes of block. -+ stride (int or Tuple[int]): Stride of the first block. Defaults to 1. -+ downsample (Module, optional): Down sample module for block. -+ Defaults to None. -+ indice_key (str): Indice key for spconv. Default to None. -+ conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -+ convolution layer. Defaults to None. -+ norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -+ normalization layer. Defaults to None. -+ """ -+ -+ expansion = 4 -+ -+ def __init__(self, -+ inplanes: int, -+ planes: int, -+ stride: Union[int, Tuple[int]] = 1, -+ downsample: nn.Module = None, -+ indice_key=None, -+ conv_cfg: OptConfigType = None, -+ norm_cfg: OptConfigType = None) -> None: -+ -+ SparseModule.__init__(self) -+ if conv_cfg is None: -+ conv_cfg = dict(type='SubMConv3d') -+ conv_cfg.setdefault('indice_key', indice_key) -+ if norm_cfg is None: -+ norm_cfg = dict(type='BN1d') -+ Bottleneck.__init__( -+ self, -+ inplanes, -+ planes, -+ stride=stride, -+ downsample=downsample, -+ conv_cfg=conv_cfg, -+ norm_cfg=norm_cfg) -+ -+ def forward(self, x: SparseConvTensor) -> SparseConvTensor: -+ identity = x.features -+ -+ out = self.conv1(x) -+ out = replace_feature(out, self.bn1(out.features)) -+ out = replace_feature(out, self.relu(out.features)) -+ -+ out = self.conv2(out) -+ out = replace_feature(out, self.bn2(out.features)) -+ out = replace_feature(out, self.relu(out.features)) -+ -+ out = self.conv3(out) -+ out = replace_feature(out, self.bn3(out.features)) -+ -+ if self.downsample is not None: -+ identity = self.downsample(x).features -+ -+ out = replace_feature(out, out.features + identity) -+ out = replace_feature(out, self.relu(out.features)) -+ -+ return out -+ -+ -+class SparseBasicBlock(BasicBlock, SparseModule): -+ """Sparse basic block for PartA^2. -+ -+ Sparse basic block implemented with submanifold sparse convolution. -+ -+ Args: -+ inplanes (int): Inplanes of block. -+ planes (int): Planes of block. -+ stride (int or Tuple[int]): Stride of the first block. Defaults to 1. -+ downsample (Module, optional): Down sample module for block. -+ Defaults to None. -+ indice_key (str): Indice key for spconv. Default to None. -+ conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -+ convolution layer. Defaults to None. -+ norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -+ normalization layer. Defaults to None. -+ """ -+ -+ expansion = 1 -+ -+ def __init__(self, -+ inplanes: int, -+ planes: int, -+ stride: Union[int, Tuple[int]] = 1, -+ downsample: nn.Module = None, -+ indice_key: Optional[str] = None, -+ conv_cfg: OptConfigType = None, -+ norm_cfg: OptConfigType = None) -> None: -+ SparseModule.__init__(self) -+ if conv_cfg is None: -+ conv_cfg = dict(type='SubMConv3d') -+ conv_cfg.setdefault('indice_key', indice_key) -+ if norm_cfg is None: -+ norm_cfg = dict(type='BN1d') -+ BasicBlock.__init__( -+ self, -+ inplanes, -+ planes, -+ stride=stride, -+ downsample=downsample, -+ conv_cfg=conv_cfg, -+ norm_cfg=norm_cfg) -+ -+ def forward(self, x: SparseConvTensor) -> SparseConvTensor: -+ identity = x.features -+ -+ assert x.features.dim() == 2, f'x.features.dim()={x.features.dim()}' -+ out = self.conv1(x) -+ out = replace_feature(out, self.norm1(out.features)) -+ out = replace_feature(out, self.relu(out.features)) -+ -+ out = self.conv2(out) -+ out = replace_feature(out, self.norm2(out.features)) -+ -+ if self.downsample is not None: -+ identity = self.downsample(x).features -+ -+ out = replace_feature(out, out.features + identity) -+ out = replace_feature(out, self.relu(out.features)) -+ -+ return out -+ -+ -+def make_sparse_convmodule(in_channels: int, -+ out_channels: int, -+ kernel_size: Union[int, Tuple[int]], -+ indice_key: Optional[str] = None, -+ stride: Union[int, Tuple[int]] = 1, -+ padding: Union[int, Tuple[int]] = 0, -+ conv_type: str = 'SubMConv3d', -+ norm_cfg: OptConfigType = None, -+ order: Tuple[str] = ('conv', 'norm', 'act'), -+ **kwargs) -> SparseSequential: -+ """Make sparse convolution module. -+ -+ Args: -+ in_channels (int): The number of input channels. -+ out_channels (int): The number of out channels. -+ kernel_size (int | Tuple[int]): Kernel size of convolution. -+ indice_key (str): The indice key used for sparse tensor. -+ stride (int or tuple[int]): The stride of convolution. -+ padding (int or tuple[int]): The padding number of input. -+ conv_type (str): Sparse conv type in spconv. Defaults to 'SubMConv3d'. -+ norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for -+ normalization layer. Defaults to None. -+ order (Tuple[str]): The order of conv/norm/activation layers. It is a -+ sequence of "conv", "norm" and "act". Common examples are -+ ("conv", "norm", "act") and ("act", "conv", "norm"). -+ Defaults to ('conv', 'norm', 'act'). -+ -+ Returns: -+ spconv.SparseSequential: sparse convolution module. -+ """ -+ assert isinstance(order, tuple) and len(order) <= 3 -+ assert set(order) | {'conv', 'norm', 'act'} == {'conv', 'norm', 'act'} -+ -+ conv_cfg = dict(type=conv_type, indice_key=indice_key) -+ if norm_cfg is None: -+ norm_cfg = dict(type='BN1d') -+ -+ layers = list() -+ for layer in order: -+ if layer == 'conv': -+ if conv_type not in [ -+ 'SparseInverseConv3d', 'SparseInverseConv2d', -+ 'SparseInverseConv1d' -+ ]: -+ layers.append( + + class SparseBottleneck(Bottleneck, SparseModule): + """Sparse bottleneck block for PartA^2. +@@ -199,7 +285,7 @@ def make_sparse_convmodule(in_channels: int, + 'SparseInverseConv1d' + ]: + layers.append( +- build_conv_layer( + build_sparse_conv_layer( -+ conv_cfg, -+ in_channels, -+ out_channels, -+ kernel_size, -+ stride=stride, -+ padding=padding, -+ bias=False)) -+ else: -+ layers.append( + conv_cfg, + in_channels, + out_channels, +@@ -209,7 +295,7 @@ def make_sparse_convmodule(in_channels: int, + bias=False)) + else: + layers.append( +- build_conv_layer( + build_sparse_conv_layer( -+ conv_cfg, -+ in_channels, -+ out_channels, -+ kernel_size, -+ bias=False)) -+ elif layer == 'norm': -+ layers.append(build_norm_layer(norm_cfg, out_channels)[1]) -+ elif layer == 'act': -+ layers.append(nn.ReLU(inplace=True)) -+ -+ layers = SparseSequential(*layers) -+ return layers + conv_cfg, + in_channels, + out_channels, +@@ -222,3 +308,42 @@ def make_sparse_convmodule(in_channels: int, + + layers = SparseSequential(*layers) + return layers + + +def build_sparse_conv_layer(cfg, *args, **kwargs): @@ -698,10 +302,15 @@ index 9f56934e..12f579eb 100644 # hard voxelize f, c, n = ret diff --git a/projects/BEVFusion/bevfusion/depth_lss.py b/projects/BEVFusion/bevfusion/depth_lss.py -index 6cc0cc16..d0cb4e0d 100644 +index 6cc0cc16..55fd6d62 100644 --- a/projects/BEVFusion/bevfusion/depth_lss.py +++ b/projects/BEVFusion/bevfusion/depth_lss.py -@@ -3,9 +3,11 @@ from typing import Tuple +@@ -1,11 +1,26 @@ + # modify from https://github.com/mit-han-lab/bevfusion ++import os ++import ctypes ++import warnings + from typing import Tuple import torch from torch import nn @@ -710,21 +319,40 @@ index 6cc0cc16..d0cb4e0d 100644 from mmdet3d.registry import MODELS -from .ops import bev_pool -+from mx_driving import bev_pool_v3, npu_index_select ++from mx_driving import bev_pool_v3 ++ ++try: ++ from mx_driving import npu_index_select ++ npu_index_select_available = True ++except ImportError: ++ npu_index_select_available = False ++ ++lib_path = os.path.join(os.environ.get('ASCEND_HOME_PATH', '/usr/local/Ascend'), 'lib64', 'libopapi.so') ++lib = ctypes.CDLL(lib_path) ++aclnnIndexAddV2_available = hasattr(lib, "aclnnIndexAddV2") # 检查是否支持aclnnIndexAddV2 def gen_dx_bx(xbound, ybound, zbound): -@@ -137,10 +139,11 @@ class BaseViewTransform(nn.Module): +@@ -137,10 +152,19 @@ class BaseViewTransform(nn.Module): & (geom_feats[:, 1] < self.nx[1]) & (geom_feats[:, 2] >= 0) & (geom_feats[:, 2] < self.nx[2])) - x = x[kept] - geom_feats = geom_feats[kept] +- +- x = bev_pool(x, geom_feats, B, self.nx[2], self.nx[0], self.nx[1]) + valid_indices = torch.nonzero(kept).view(-1) -+ x = npu_index_select(x, 0, valid_indices) ++ ++ if npu_index_select_available and aclnnIndexAddV2_available: ++ x = npu_index_select(x, 0, valid_indices) ++ else: ++ # Fallback to torch.index_select if npu_index_select fails ++ warnings.warn( ++ "'npu_index_select' is unavailable. Please update the mx_driving and CANN packages to the latest version to enable this operator for better performance. The model will still run without it, but performance may be degraded." ++ ) ++ x = torch.index_select(x, 0, valid_indices) + geom_feats = torch.index_select(geom_feats, 0, valid_indices) - -- x = bev_pool(x, geom_feats, B, self.nx[2], self.nx[0], self.nx[1]) ++ + x = bev_pool_v3(None, x, None, None, geom_feats.to(torch.int32), [B, self.nx[2], self.nx[0], self.nx[1], C]) # collapse Z