Hi, I got an error message "AttributeError: 'ModelEMA' object has no attribute 'eval'" when training to epoch4, so I changed Line 62 and 79 in mmdet3d/core/hook/eval_hook.py to runner.ema_model.ema_model according to your reply to a similar question in the discussion forum. Then I got a new error message "RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cpu!"
In order to trigger the evaluation conveniently, I set the interval to 1 in configs/cotr/cotr-bevdetocc-r50-4d-stereo-24e.py, so the evaluation changed from the original interval4 to interval1.
The error message is as follows.How do I resolve the issue “RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cpu!”?
2025-07-05 11:43:57,919 - mmdet - INFO - Saving checkpoint at 1 epochs
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
2025-07-05 11:44:04,653 - mmdet - INFO - Saving ema checkpoint at /data5/xieying/COTR_DAOcc/work_dirs/cotr-bevdetocc-r50-4d-stereo-24e/epoch_1_ema.pth
[ ] 0/6019, elapsed: 0s, ETA:After BatchNorm1d output type: <class 'torch.Tensor'>
Traceback (most recent call last):
File "tools/train_occ.py", line 263, in
main()
File "tools/train_occ.py", line 252, in main
train_occ_model(
File "/data5/xieying/COTR_DAOcc/mmdet3d/apis/train_occ.py", line 351, in train_occ_model
train_detector(
File "/data5/xieying/COTR_DAOcc/mmdet3d/apis/train_occ.py", line 326, in train_detector
runner.run(data_loaders, cfg.workflow)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 130, in run
epoch_runner(data_loaders[i], **kwargs)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 56, in train
self.call_hook('after_train_epoch')
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/base_runner.py", line 309, in call_hook
getattr(hook, fn_name)(self)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/hooks/evaluation.py", line 267, in after_train_epoch
self._do_evaluate(runner)
File "/data5/xieying/COTR_DAOcc/mmdet3d/core/hook/eval_hooks.py", line 111, in _do_evaluate
results = multi_gpu_test(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmdet/apis/test.py", line 109, in multi_gpu_test
result = model(return_loss=False, rescale=True, **data)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 116, in new_func
return old_func(*args, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/base.py", line 62, in forward
return self.forward_test(**kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/bevdet.py", line 171, in forward_test
return self.simple_test(points[0], img_metas[0], img_inputs[0],
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 471, in simple_test
img_feats,pts_feats, depth, img_metas, cam_params = self.extract_feat(
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 383, in extract_feat
img_feats, depth, img_metas, cam_params = self.extract_img_feat(img, img_metas, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 166, in extract_img_feat
self.prepare_bev_feat(*inputs_curr)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 113, in prepare_bev_feat
bev_feat, depth = self.img_view_transformer(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 803, in forward
x = self.depth_net(x, mlp_input, stereo_metas)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 639, in forward
cost_volumn = self.calculate_cost_volumn(stereo_metas)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 580, in calculate_cost_volumn
grid = self.gen_grid(metas, B, N, D, H, W, hi, wi).to(curr.dtype)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 545, in gen_grid
points = frustum - metas['post_trans'].view(B, N, 1, 1, 1, 3)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cpu!
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
After BatchNorm1d output type: <class 'torch.Tensor'>
Traceback (most recent call last):
File "tools/train_occ.py", line 263, in
main()
File "tools/train_occ.py", line 252, in main
train_occ_model(
File "/data5/xieying/COTR_DAOcc/mmdet3d/apis/train_occ.py", line 351, in train_occ_model
train_detector(
File "/data5/xieying/COTR_DAOcc/mmdet3d/apis/train_occ.py", line 326, in train_detector
runner.run(data_loaders, cfg.workflow)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 130, in run
epoch_runner(data_loaders[i], **kwargs)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 56, in train
self.call_hook('after_train_epoch')
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/base_runner.py", line 309, in call_hook
getattr(hook, fn_name)(self)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/hooks/evaluation.py", line 267, in after_train_epoch
self._do_evaluate(runner)
File "/data5/xieying/COTR_DAOcc/mmdet3d/core/hook/eval_hooks.py", line 111, in _do_evaluate
results = multi_gpu_test(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmdet/apis/test.py", line 109, in multi_gpu_test
result = model(return_loss=False, rescale=True, **data)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 116, in new_func
return old_func(*args, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/base.py", line 62, in forward
return self.forward_test(**kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/bevdet.py", line 171, in forward_test
return self.simple_test(points[0], img_metas[0], img_inputs[0],
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 471, in simple_test
img_feats,pts_feats, depth, img_metas, cam_params = self.extract_feat(
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 383, in extract_feat
img_feats, depth, img_metas, cam_params = self.extract_img_feat(img, img_metas, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 166, in extract_img_feat
self.prepare_bev_feat(*inputs_curr)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 113, in prepare_bev_feat
bev_feat, depth = self.img_view_transformer(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 803, in forward
x = self.depth_net(x, mlp_input, stereo_metas)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 639, in forward
cost_volumn = self.calculate_cost_volumn(stereo_metas)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 580, in calculate_cost_volumn
grid = self.gen_grid(metas, B, N, D, H, W, hi, wi).to(curr.dtype)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 545, in gen_grid
points = frustum - metas['post_trans'].view(B, N, 1, 1, 1, 3)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!
WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 49991 closing signal SIGTERM
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 1 (pid: 49992) of binary: /opt/conda/envs/cotr/bin/python
Traceback (most recent call last):
File "/opt/conda/envs/cotr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/opt/conda/envs/cotr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launch.py", line 193, in
main()
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launch.py", line 189, in main
launch(args)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launch.py", line 174, in launch
run(args)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/run.py", line 710, in run
elastic_launch(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 259, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
tools/train_occ.py FAILED
Failures:
<NO_OTHER_FAILURES>
Root Cause (first observed failure):
[0]:
time : 2025-07-05_11:44:10
host : 841d68780d16
rank : 1 (local_rank: 1)
exitcode : 1 (pid: 49992)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
Hi, I got an error message "AttributeError: 'ModelEMA' object has no attribute 'eval'" when training to epoch4, so I changed Line 62 and 79 in mmdet3d/core/hook/eval_hook.py to runner.ema_model.ema_model according to your reply to a similar question in the discussion forum. Then I got a new error message "RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cpu!"
In order to trigger the evaluation conveniently, I set the interval to 1 in configs/cotr/cotr-bevdetocc-r50-4d-stereo-24e.py, so the evaluation changed from the original interval4 to interval1.
The error message is as follows.How do I resolve the issue “RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cpu!”?
2025-07-05 11:43:57,919 - mmdet - INFO - Saving checkpoint at 1 epochs
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
2025-07-05 11:44:04,653 - mmdet - INFO - Saving ema checkpoint at /data5/xieying/COTR_DAOcc/work_dirs/cotr-bevdetocc-r50-4d-stereo-24e/epoch_1_ema.pth
[ ] 0/6019, elapsed: 0s, ETA:After BatchNorm1d output type: <class 'torch.Tensor'>
Traceback (most recent call last):
File "tools/train_occ.py", line 263, in
main()
File "tools/train_occ.py", line 252, in main
train_occ_model(
File "/data5/xieying/COTR_DAOcc/mmdet3d/apis/train_occ.py", line 351, in train_occ_model
train_detector(
File "/data5/xieying/COTR_DAOcc/mmdet3d/apis/train_occ.py", line 326, in train_detector
runner.run(data_loaders, cfg.workflow)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 130, in run
epoch_runner(data_loaders[i], **kwargs)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 56, in train
self.call_hook('after_train_epoch')
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/base_runner.py", line 309, in call_hook
getattr(hook, fn_name)(self)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/hooks/evaluation.py", line 267, in after_train_epoch
self._do_evaluate(runner)
File "/data5/xieying/COTR_DAOcc/mmdet3d/core/hook/eval_hooks.py", line 111, in _do_evaluate
results = multi_gpu_test(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmdet/apis/test.py", line 109, in multi_gpu_test
result = model(return_loss=False, rescale=True, **data)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 116, in new_func
return old_func(*args, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/base.py", line 62, in forward
return self.forward_test(**kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/bevdet.py", line 171, in forward_test
return self.simple_test(points[0], img_metas[0], img_inputs[0],
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 471, in simple_test
img_feats,pts_feats, depth, img_metas, cam_params = self.extract_feat(
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 383, in extract_feat
img_feats, depth, img_metas, cam_params = self.extract_img_feat(img, img_metas, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 166, in extract_img_feat
self.prepare_bev_feat(*inputs_curr)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 113, in prepare_bev_feat
bev_feat, depth = self.img_view_transformer(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 803, in forward
x = self.depth_net(x, mlp_input, stereo_metas)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 639, in forward
cost_volumn = self.calculate_cost_volumn(stereo_metas)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 580, in calculate_cost_volumn
grid = self.gen_grid(metas, B, N, D, H, W, hi, wi).to(curr.dtype)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 545, in gen_grid
points = frustum - metas['post_trans'].view(B, N, 1, 1, 1, 3)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:1 and cpu!
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
/data5/xieying/COTR_DAOcc/mmdet3d/datasets/pipelines/loading.py:1081: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:201.)
gt_boxes, gt_labels = torch.Tensor(gt_boxes), torch.tensor(gt_labels)
After BatchNorm1d output type: <class 'torch.Tensor'>
Traceback (most recent call last):
File "tools/train_occ.py", line 263, in
main()
File "tools/train_occ.py", line 252, in main
train_occ_model(
File "/data5/xieying/COTR_DAOcc/mmdet3d/apis/train_occ.py", line 351, in train_occ_model
train_detector(
File "/data5/xieying/COTR_DAOcc/mmdet3d/apis/train_occ.py", line 326, in train_detector
runner.run(data_loaders, cfg.workflow)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 130, in run
epoch_runner(data_loaders[i], **kwargs)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/epoch_based_runner.py", line 56, in train
self.call_hook('after_train_epoch')
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/base_runner.py", line 309, in call_hook
getattr(hook, fn_name)(self)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/hooks/evaluation.py", line 267, in after_train_epoch
self._do_evaluate(runner)
File "/data5/xieying/COTR_DAOcc/mmdet3d/core/hook/eval_hooks.py", line 111, in _do_evaluate
results = multi_gpu_test(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmdet/apis/test.py", line 109, in multi_gpu_test
result = model(return_loss=False, rescale=True, **data)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/mmcv/runner/fp16_utils.py", line 116, in new_func
return old_func(*args, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/base.py", line 62, in forward
return self.forward_test(**kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/bevdet.py", line 171, in forward_test
return self.simple_test(points[0], img_metas[0], img_inputs[0],
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 471, in simple_test
img_feats,pts_feats, depth, img_metas, cam_params = self.extract_feat(
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 383, in extract_feat
img_feats, depth, img_metas, cam_params = self.extract_img_feat(img, img_metas, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 166, in extract_img_feat
self.prepare_bev_feat(*inputs_curr)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/detectors/cotr.py", line 113, in prepare_bev_feat
bev_feat, depth = self.img_view_transformer(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 803, in forward
x = self.depth_net(x, mlp_input, stereo_metas)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 639, in forward
cost_volumn = self.calculate_cost_volumn(stereo_metas)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 580, in calculate_cost_volumn
grid = self.gen_grid(metas, B, N, D, H, W, hi, wi).to(curr.dtype)
File "/data5/xieying/COTR_DAOcc/mmdet3d/models/necks/view_transformer.py", line 545, in gen_grid
points = frustum - metas['post_trans'].view(B, N, 1, 1, 1, 3)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!
WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 49991 closing signal SIGTERM
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 1 (pid: 49992) of binary: /opt/conda/envs/cotr/bin/python
Traceback (most recent call last):
File "/opt/conda/envs/cotr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/opt/conda/envs/cotr/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launch.py", line 193, in
main()
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launch.py", line 189, in main
launch(args)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launch.py", line 174, in launch
run(args)
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/run.py", line 710, in run
elastic_launch(
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 131, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/opt/conda/envs/cotr/lib/python3.8/site-packages/torch/distributed/launcher/api.py", line 259, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
tools/train_occ.py FAILED
Failures:
<NO_OTHER_FAILURES>
Root Cause (first observed failure):
[0]:
time : 2025-07-05_11:44:10
host : 841d68780d16
rank : 1 (local_rank: 1)
exitcode : 1 (pid: 49992)
error_file: <N/A>
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html