From 18f8cd09e5df163612fe41f8cba876d8c8db5234 Mon Sep 17 00:00:00 2001 From: Roy Williams Date: Fri, 23 Apr 2021 23:03:01 -0400 Subject: [PATCH] Make libprotobuf symbols local on OSX (#8346) @gnossen gave a great overview in https://github.com/grpc/grpc/pull/24992 of the overall problem. If a python process using both protobuf _and_ another native library linking in libprotobuf frequently can cause crashes. This seems to frequently affect tensorflow as well: tensorflow/tensorflow#8394, tensorflow/tensorflow#9525 (comment) tensorflow/tensorflow#24976, tensorflow/tensorflow#35573, https://github.com/tensorflow/tensorflow/blob/v2.0.0/tensorflow/contrib/makefile/rename_protobuf.sh, tensorflow/tensorflow#16104 Testing locally this fixes both crashes when linking in multiple versions of protobuf and fixes `DescriptorPool` clashes as well (e.g. Python and Native code import different versions of the same message). --- python/setup.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/setup.py b/python/setup.py index 77d9cd1f6ce..4a6161c21fa 100755 --- a/python/setup.py +++ b/python/setup.py @@ -210,6 +210,18 @@ def get_option_from_sys_argv(option_str): extra_compile_args = [] + message_extra_link_args = None + api_implementation_link_args = None + if "darwin" in sys.platform: + if sys.version_info[0] == 2: + message_init_symbol = 'init_message' + api_implementation_init_symbol = 'init_api_implementation' + else: + message_init_symbol = 'PyInit__message' + api_implementation_init_symbol = 'PyInit__api_implementation' + message_extra_link_args = ['-Wl,-exported_symbol,_%s' % message_init_symbol] + api_implementation_link_args = ['-Wl,-exported_symbol,_%s' % api_implementation_init_symbol] + if sys.platform != 'win32': extra_compile_args.append('-Wno-write-strings') extra_compile_args.append('-Wno-invalid-offsetof') @@ -260,6 +272,7 @@ def get_option_from_sys_argv(option_str): include_dirs=[".", "../src"], libraries=libraries, extra_objects=extra_objects, + extra_link_args=message_extra_link_args, library_dirs=['../src/.libs'], extra_compile_args=extra_compile_args, ), @@ -267,6 +280,7 @@ def get_option_from_sys_argv(option_str): "google.protobuf.internal._api_implementation", glob.glob('google/protobuf/internal/api_implementation.cc'), extra_compile_args=extra_compile_args + ['-DPYTHON_PROTO2_CPP_IMPL_V2'], + extra_link_args=api_implementation_link_args, ), ]) os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'