diff --git a/setup.py b/setup.py
index c49dbe96575..2d02cf129ca 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,7 @@
 import struct
 import subprocess
 import sys
+import sysconfig
 import warnings
 
 from pybind11.setup_helpers import ParallelCompile
@@ -571,8 +572,16 @@ def build_extensions(self) -> None:
                 for d in os.environ[k].split(os.path.pathsep):
                     _add_directory(library_dirs, d)
 
-        _add_directory(library_dirs, os.path.join(sys.prefix, "lib"))
-        _add_directory(include_dirs, os.path.join(sys.prefix, "include"))
+        _add_directory(
+            library_dirs,
+            (sys.prefix == sys.base_prefix and sysconfig.get_config_var("LIBDIR"))
+            or os.path.join(sys.prefix, "lib"),
+        )
+        _add_directory(
+            include_dirs,
+            (sys.prefix == sys.base_prefix and sysconfig.get_config_var("INCLUDEDIR"))
+            or os.path.join(sys.prefix, "include"),
+        )
 
         #
         # add platform directories