File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 447, in init_distributed_mode
torch.cuda.set_device(1)
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/cuda/init.py", line 263, in set_device
torch._C._cuda_setDevice(device)
RuntimeError: CUDA error: invalid device ordinal
Traceback (most recent call last):
Traceback (most recent call last):
File "main_track.py", line 374, in
File "main_track.py", line 374, in
main(args)main(args)
File "main_track.py", line 151, in main
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 447, in init_distributed_mode
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 447, in init_distributed_mode
torch.cuda.set_device(1)torch.cuda.set_device(1)
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/cuda/init.py", line 263, in set_device
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/cuda/init.py", line 263, in set_device
torch._C._cuda_setDevice(device)torch._C._cuda_setDevice(device)
RuntimeErrorRuntimeError: : CUDA error: invalid device ordinalCUDA error: invalid device ordinal
sh track_exps/crowdhuman_train.sh
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
| distributed init (rank 2): env://
| distributed init (rank 5): env://
| distributed init (rank 1): env://
| distributed init (rank 6): env://
| distributed init (rank 3): env://
| distributed init (rank 0): env://
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 436, in init_process_group
store, rank, world_size = next(rendezvous_iterator)
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/rendezvous.py", line 179, in _env_rendezvous_handler
store = TCPStore(master_addr, master_port, world_size, start_daemon, timeout)
RuntimeError: Address already in use
| distributed init (rank 7): env://
| distributed init (rank 4): env://
Traceback (most recent call last):
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/runpy.py", line 193, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/launch.py", line 260, in
main()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/launch.py", line 255, in main
raise subprocess.CalledProcessError(returncode=process.returncode,
subprocess.CalledProcessError: Command '['/home/fuc/anaconda3/envs/MOT/bin/python3', '-u', 'main_track.py', '--output_dir', './output_crowdhuman', '--dataset_file', 'crowdhuman', '--coco_path', 'crowdhuman', '--batch_size', '1', '--with_box_refine', '--num_queries', '500', '--epochs', '150', '--lr_drop', '100']' returned non-zero exit status 1.
(MOT) fuc@fuc-computer:~/code/MOTcode/TransTrack$ sh track_exps/crowdhuman_train.sh
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
| distributed init (rank 0): env://
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 436, in init_process_group
store, rank, world_size = next(rendezvous_iterator)
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/rendezvous.py", line 179, in _env_rendezvous_handler
store = TCPStore(master_addr, master_port, world_size, start_daemon, timeout)
RuntimeError: Address already in use
Traceback (most recent call last):
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/runpy.py", line 193, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/launch.py", line 260, in
main()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/launch.py", line 255, in main
raise subprocess.CalledProcessError(returncode=process.returncode,
subprocess.CalledProcessError: Command '['/home/fuc/anaconda3/envs/MOT/bin/python3', '-u', 'main_track.py', '--output_dir', './output_crowdhuman', '--dataset_file', 'crowdhuman', '--coco_path', 'crowdhuman', '--batch_size', '1', '--with_box_refine', '--num_queries', '500', '--epochs', '150', '--lr_drop', '100']' returned non-zero exit status 1.
| distributed init (rank 4): env://
| distributed init (rank 3): env://
(MOT) fuc@fuc-computer:~/code/MOTcode/TransTrack$ | distributed init (rank 5): env://
| distributed init (rank 2): env://
| distributed init (rank 7): env://
| distributed init (rank 6): env://
| distributed init (rank 1): env://
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
Traceback (most recent call last):
File "main_track.py", line 374, in
main(args)
File "main_track.py", line 151, in main
utils.init_distributed_mode(args)
File "/home/fuc/code/MOTcode/TransTrack/util/misc.py", line 451, in init_distributed_mode
torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
barrier()
File "/home/fuc/anaconda3/envs/MOT/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607370172916/work/torch/lib/c10d/ProcessGroupNCCL.cpp:784, unhandled system error, NCCL version 2.7.8
