Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add manual cuda deps search logic (#90411) #90426

Merged
merged 1 commit into from Dec 8, 2022
Merged

Add manual cuda deps search logic (#90411) #90426

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
28 changes: 27 additions & 1 deletion torch/__init__.py
View file
Open in desktop
Expand Up @@ -141,6 +141,24 @@
kernel32.SetErrorMode(prev_error_mode)


def _preload_cuda_deps():
""" Preloads cudnn/cublas deps if they could not be found otherwise """
# Should only be called on Linux if default path resolution have failed
assert platform.system() == 'Linux', 'Should only be called on Linux'
for path in sys.path:
nvidia_path = os.path.join(path, 'nvidia')
if not os.path.exists(nvidia_path):
continue
cublas_path = os.path.join(nvidia_path, 'cublas', 'lib', 'libcublas.so.11')
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are very specific lib versions. Isn't that a problem?

cudnn_path = os.path.join(nvidia_path, 'cudnn', 'lib', 'libcudnn.so.8')
if not os.path.exists(cublas_path) or not os.path.exists(cudnn_path):
continue
break

ctypes.CDLL(cublas_path)
ctypes.CDLL(cudnn_path)


# See Note [Global dependencies]
def _load_global_deps():
if platform.system() == 'Windows' or sys.executable == 'torch_deploy':
Expand All @@ -150,7 +168,15 @@ def _load_global_deps():
here = os.path.abspath(__file__)
lib_path = os.path.join(os.path.dirname(here), 'lib', lib_name)

ctypes.CDLL(lib_path, mode=ctypes.RTLD_GLOBAL)
try:
ctypes.CDLL(lib_path, mode=ctypes.RTLD_GLOBAL)
except OSError as err:
# Can only happen of wheel with cublas as PYPI deps
# As PyTorch is not purelib, but nvidia-cublas-cu11 is
if 'libcublas.so.11' not in err.args[0]:
raise err
_preload_cuda_deps()
ctypes.CDLL(lib_path, mode=ctypes.RTLD_GLOBAL)


if (USE_RTLD_GLOBAL_WITH_LIBTORCH or os.getenv('TORCH_USE_RTLD_GLOBAL')) and \
Expand Down