You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
I'm getting a SpconvOps error when running project UniDistill.
Version:spconv2.1.25+cumm0.2.9. I also tried ● spconv-cu114-2.1.24, spconv-cu114-2.1.25, spconv-cu111-2.1.25, spconv-cu114-2.1.18 all didn’t work.
Traceback (most recent call last):
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 685, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 777, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1199, in _run
self._dispatch()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1279, in _dispatch
self.training_type_plugin.start_training(self)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py", line 202, in start_training
self._results = trainer.run_stage()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1289, in run_stage
return self._run_train()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1319, in _run_train
self.fit_loop.run()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/base.py", line 145, in run
self.advance(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/fit_loop.py", line 234, in advance
self.epoch_loop.run(data_fetcher)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/base.py", line 145, in run
self.advance(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 193, in advance
batch_output = self.batch_loop.run(batch, batch_idx)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/base.py", line 145, in run
self.advance(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
outputs = self.optimizer_loop.run(split_batch, optimizers, batch_idx)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/base.py", line 145, in run
self.advance(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 219, in advance
self.optimizer_idx,
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 266, in _run_optimization
self._optimizer_step(optimizer, opt_idx, batch_idx, closure)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 386, in _optimizer_step
using_lbfgs=is_lbfgs,
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/core/lightning.py", line 1652, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/core/optimizer.py", line 164, in step
trainer.accelerator.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/accelerators/accelerator.py", line 339, in optimizer_step
self.precision_plugin.optimizer_step(model, optimizer, opt_idx, closure, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 163, in optimizer_step
optimizer.step(closure=closure, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/optim/lr_scheduler.py", line 65, in wrapper
return wrapped(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/optim/optimizer.py", line 88, in wrapper
return func(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 28, in decorate_context
return func(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/optim/adamw.py", line 65, in step
loss = closure()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 148, in _wrap_closure
closure_result = closure()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 160, in __call__
self._result = self.closure(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 142, in closure
step_output = self._step_fn()
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 435, in _training_step
training_step_output = self.trainer.accelerator.training_step(step_kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/accelerators/accelerator.py", line 219, in training_step
return self.training_type_plugin.training_step(*step_kwargs.values())
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/plugins/training_type/ddp.py", line 439, in training_step
return self.model(*args, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/parallel/distributed.py", line 799, in forward
output = self.module(*inputs[0], **kwargs[0])
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/pytorch_lightning/overrides/base.py", line 81, in forward
output = self.module.training_step(*inputs, **kwargs)
File "/root/UniDistill/unidistill/exps/multisensor_fusion/nuscenes/BEVFusion/BEVFusion_nuscenes_base_exp.py", line 376, in training_step
ret_dict, tf_dict, _, _, _, _ = self(points, imgs, metas, gt_boxes)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/root/UniDistill/unidistill/exps/multisensor_fusion/nuscenes/BEVFusion/BEVFusion_nuscenes_base_exp.py", line 360, in forward
return self.model(points, imgs, metas, gt_boxes)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/root/UniDistill/unidistill/exps/multisensor_fusion/nuscenes/BEVFusion/BEVFusion_nuscenes_centerhead_fusion_exp.py", line 144, in forward
lidar_output = self.lidar_encoder(lidar_points)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/root/UniDistill/unidistill/exps/multisensor_fusion/nuscenes/BEVFusion/BEVFusion_nuscenes_base_exp.py", line 76, in forward
voxels, voxel_coords, voxel_num_points = self.voxelizer(lidar_points)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/root/UniDistill/unidistill/data/det3d/preprocess/voxelization.py", line 54, in forward
voxel_output = self.voxel_generator(p)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/spconv/pytorch/utils.py", line 88, in __call__
res = self.generate_voxel_with_id(pc, clear_voxels, empty_mean)
File "/usr/local/envs/unidistill/lib/python3.6/site-packages/spconv/pytorch/utils.py", line 139, in generate_voxel_with_id
empty_mean, clear_voxels, stream)
RuntimeError: zero_ /tmp/pip-build-env-jm2xsh18/overlay/lib/python3.6/site-packages/cumm/include/tensorview/tensor.h 220
cuda failed with error 1 invalid argument. use CUDA_LAUNCH_BLOCKING=1 to get correct traceback.
It's urgent. Please help!
The text was updated successfully, but these errors were encountered:
I'm getting a SpconvOps error when running project UniDistill.
Version:spconv2.1.25+cumm0.2.9. I also tried ● spconv-cu114-2.1.24, spconv-cu114-2.1.25, spconv-cu111-2.1.25, spconv-cu114-2.1.18 all didn’t work.
Related issues:megvii-research/CVPR2023-UniDistill#9
It's urgent. Please help!
The text was updated successfully, but these errors were encountered: