(maskpls) nvidia@DESKTOP-BCMI1AU:~/occ/MaskPLS/mask_pls$ python3 -X faulthandler scripts/evaluate_model.py --w weights/mask_pls_kitti.ckpt
/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/MinkowskiEngine-0.5.4-py3.8-linux-x86_64.egg/MinkowskiEngine/__init__.py:36: UserWarning: The environment variable `OMP_NUM_THREADS` not set. MinkowskiEngine will automatically set `OMP_NUM_THREADS=16`. If you want to set `OMP_NUM_THREADS` manually, please export it on the command line before running a python script. e.g. `export OMP_NUM_THREADS=12; python your_program.py`. It is recommended to set it below 24.
warnings.warn(
[KeOps] Compiling cuda jit compiler engine ... OK
[pyKeOps] Compiling nvrtc binder for python ... OK
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/trainer/data_loading.py:116: UserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 28 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
rank_zero_warn(
Validating: 0%| | 0/4071 [00:00<?, ?it/s]Fatal Python error: Segmentation fault
Thread 0x00007f583ce55700 (most recent call first):
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/threading.py", line 306 in wait
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/threading.py", line 558 in wait
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/tqdm/_monitor.py", line 60 in run
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/threading.py", line 890 in _bootstrap
Current thread 0x00007f591173f740 (most recent call first):
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/keopscore/binders/nvrtc/Gpu_link_compile.py", line 67 in generate_code
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/keopscore/binders/LinkCompile.py", line 101 in get_dll_and_params
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/keopscore/get_keops_dll.py", line 124 in get_keops_dll_impl
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/keopscore/utils/Cache.py", line 27 in __call__
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pykeops/common/keops_io/LoadKeOps.py", line 126 in init
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pykeops/common/keops_io/LoadKeOps.py", line 18 in __init__
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pykeops/common/keops_io/LoadKeOps_nvrtc.py", line 15 in __init__
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/keopscore/utils/Cache.py", line 68 in __call__
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pykeops/torch/generic/generic_red.py", line 78 in forward
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pykeops/torch/generic/generic_red.py", line 624 in __call__
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pykeops/common/lazy_tensor.py", line 937 in __call__
File "/home/nvidia/occ/MaskPLS/mask_pls/utils/interpolate.py", line 44 in kNN
File "/home/nvidia/occ/MaskPLS/mask_pls/utils/interpolate.py", line 24 in forward
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102 in _call_impl
File "/home/nvidia/occ/MaskPLS/mask_pls/models/mink.py", line 138 in <listcomp>
File "/home/nvidia/occ/MaskPLS/mask_pls/models/mink.py", line 137 in <listcomp>
File "/home/nvidia/occ/MaskPLS/mask_pls/models/mink.py", line 136 in forward
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102 in _call_impl
File "/home/nvidia/occ/MaskPLS/mask_pls/models/mask_model.py", line 35 in forward
File "/home/nvidia/occ/MaskPLS/mask_pls/models/mask_model.py", line 89 in evaluation_step
File "/home/nvidia/occ/MaskPLS/mask_pls/models/mask_model.py", line 65 in validation_step
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py", line 219 in validation_step
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py", line 236 in validation_step
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 217 in _evaluation_step
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 122 in advance
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/loops/base.py", line 145 in run
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 110 in advance
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/loops/base.py", line 145 in run
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1329 in _run_evaluate
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1281 in run_stage
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py", line 206 in start_evaluating
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1270 in _dispatch
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1194 in _run
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 859 in _validate_impl
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 682 in _call_and_handle_interrupt
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 816 in validate
File "scripts/evaluate_model.py", line 52 in main
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/click/core.py", line 760 in invoke
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/click/core.py", line 1404 in invoke
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/click/core.py", line 1055 in main
File "/home/nvidia/miniforge3/envs/maskpls/lib/python3.8/site-packages/click/core.py", line 1130 in __call__
File "scripts/evaluate_model.py", line 73 in <module>
