diff --git a/setup.py b/setup.py
index 445a448ebc52..601a6304aa24 100644
--- a/setup.py
+++ b/setup.py
@@ -105,21 +105,40 @@ def add_optimization_flags(self):
         """
 
         env = os.environ.copy()
-        if not setupext.config.getboolean('libs', 'enable_lto', fallback=True):
-            return env
         if sys.platform == 'win32':
             return env
-
+        enable_lto = setupext.config.getboolean('libs', 'enable_lto',
+                                                fallback=None)
+
+        if 'CFLAGS' in os.environ:
+            if '-fno-lto' in os.environ['CFLAGS']:
+                if enable_lto is True:
+                    raise ValueError('Configuration enable_lto=True, but '
+                                     'CFLAGS contains -fno-lto')
+                enable_lto = False
         cppflags = []
         if 'CPPFLAGS' in os.environ:
             cppflags.append(os.environ['CPPFLAGS'])
+            if '-fno-lto' in os.environ['CPPFLAGS']:
+                if enable_lto is True:
+                    raise ValueError('Configuration enable_lto=True, but '
+                                     'CPPFLAGS contains -fno-lto')
+                enable_lto = False
         cxxflags = []
         if 'CXXFLAGS' in os.environ:
             cxxflags.append(os.environ['CXXFLAGS'])
+            if '-fno-lto' in os.environ['CXXFLAGS']:
+                if enable_lto is True:
+                    raise ValueError('Configuration enable_lto=True, but '
+                                     'CXXFLAGS contains -fno-lto')
+                enable_lto = False
         ldflags = []
         if 'LDFLAGS' in os.environ:
             ldflags.append(os.environ['LDFLAGS'])
 
+        if enable_lto is False:
+            return env
+
         if has_flag(self.compiler, '-fvisibility=hidden'):
             for ext in self.extensions:
                 ext.extra_compile_args.append('-fvisibility=hidden')