Skip to content

Commit 62d6963

Browse files
Avoid race due to deleted imports when cleaning up objects at interpreter shutdown (#973)
* is_shutting_down * Update cuda_bindings/cuda/bindings/utils/__init__.py * safe_close() * is_finalizing * refactor * add note
1 parent f9b4930 commit 62d6963

File tree

5 files changed

+36
-18
lines changed

5 files changed

+36
-18
lines changed

cuda_bindings/cuda/bindings/utils/__init__.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE
3-
43
from typing import Any, Callable
54

65
from ._ptx_utils import get_minimal_required_cuda_ver_from_ptx_ver, get_ptx_ver

cuda_core/cuda/core/experimental/_event.pyx

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ from cuda.core.experimental._utils.cuda_utils import (
1818
driver,
1919
handle_return,
2020
)
21-
21+
import sys
2222
if TYPE_CHECKING:
2323
import cuda.bindings
2424
from cuda.core.experimental._device import Device
@@ -108,15 +108,20 @@ cdef class Event:
108108
self._ctx_handle = ctx_handle
109109
return self
110110

111-
cpdef close(self):
112-
"""Destroy the event."""
111+
cdef _shutdown_safe_close(self, is_shutting_down=sys.is_finalizing):
112+
if is_shutting_down and is_shutting_down():
113+
return
113114
if self._handle is not None:
114115
err, = driver.cuEventDestroy(self._handle)
115116
self._handle = None
116117
raise_if_driver_error(err)
117118

119+
cpdef close(self):
120+
"""Destroy the event."""
121+
self._shutdown_safe_close(is_shutting_down=None)
122+
118123
def __del__(self):
119-
self.close()
124+
self._shutdown_safe_close()
120125

121126
def __isub__(self, other):
122127
return NotImplemented

cuda_core/cuda/core/experimental/_memory.pyx

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ from cuda.core.experimental._utils.cuda_utils cimport (
99
_check_driver_error as raise_if_driver_error,
1010
check_or_create_options,
1111
)
12+
import sys
1213

1314
from dataclasses import dataclass
1415
from typing import TypeVar, Union, TYPE_CHECKING
@@ -69,7 +70,16 @@ cdef class Buffer:
6970
return self
7071

7172
def __del__(self):
72-
self.close()
73+
self._shutdown_safe_close()
74+
75+
cdef _shutdown_safe_close(self, stream: Stream = None, is_shutting_down=sys.is_finalizing):
76+
if is_shutting_down and is_shutting_down():
77+
return
78+
if self._ptr and self._mr is not None:
79+
self._mr.deallocate(self._ptr, self._size, stream)
80+
self._ptr = 0
81+
self._mr = None
82+
self._ptr_obj = None
7383

7484
cpdef close(self, stream: Stream = None):
7585
"""Deallocate this buffer asynchronously on the given stream.
@@ -83,11 +93,7 @@ cdef class Buffer:
8393
The stream object to use for asynchronous deallocation. If None,
8494
the behavior depends on the underlying memory resource.
8595
"""
86-
if self._ptr and self._mr is not None:
87-
self._mr.deallocate(self._ptr, self._size, stream)
88-
self._ptr = 0
89-
self._mr = None
90-
self._ptr_obj = None
96+
self._shutdown_safe_close(stream, is_shutting_down=None)
9197

9298
@property
9399
def handle(self) -> DevicePointerT:

cuda_core/cuda/core/experimental/_stream.pyx

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ from cuda.core.experimental._utils.cuda_utils cimport (
88
_check_driver_error as raise_if_driver_error,
99
check_or_create_options,
1010
)
11+
import sys
1112

1213
import cython
1314
import os
@@ -186,22 +187,28 @@ cdef class Stream:
186187
return self
187188

188189
def __del__(self):
189-
self.close()
190+
self._shutdown_safe_close()
190191

191-
cpdef close(self):
192-
"""Destroy the stream.
192+
cdef _shutdown_safe_close(self, is_shutting_down=sys.is_finalizing):
193+
if is_shutting_down and is_shutting_down():
194+
return
193195

194-
Destroy the stream if we own it. Borrowed foreign stream
195-
object will instead have their references released.
196-
197-
"""
198196
if self._owner is None:
199197
if self._handle and not self._builtin:
200198
handle_return(driver.cuStreamDestroy(self._handle))
201199
else:
202200
self._owner = None
203201
self._handle = None
204202

203+
cpdef close(self):
204+
"""Destroy the stream.
205+
206+
Destroy the stream if we own it. Borrowed foreign stream
207+
object will instead have their references released.
208+
209+
"""
210+
self._shutdown_safe_close(is_shutting_down=None)
211+
205212
def __cuda_stream__(self) -> tuple[int, int]:
206213
"""Return an instance of a __cuda_stream__ protocol."""
207214
return (0, int(self.handle))

cuda_core/docs/source/release/0.X.Y-notes.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,4 @@ Fixes and enhancements
4747
- Fixed a bug in :class:`GraphBuilder.add_child` where dependencies extracted from capturing stream were passed inconsistently with num_dependencies parameter (addresses issue #843).
4848
- Make :class:`Buffer` creation more performant.
4949
- Enabled :class:`MemoryResource` subclasses to accept :class:`Device` objects, in addition to previously supported device ordinals.
50+
- Fixed a bug in :class:`Stream` and other classes where object cleanup would error during interpreter shutdown.

0 commit comments

Comments
 (0)