diff --git a/scripts/install/install_build_dependencies.py b/scripts/install/install_build_dependencies.py
index ca1688faa..2e35e2de4 100755
--- a/scripts/install/install_build_dependencies.py
+++ b/scripts/install/install_build_dependencies.py
@@ -76,13 +76,17 @@ def download_pybind11(path):
     md5sum = '90c4946e87c64d8d8fc0ae4edf35d780'
     out_dir_name = 'pybind11-2.11.0'
 
-    download_and_extract_tgz(url, md5sum, path)
+    with tempfile.TemporaryDirectory() as temp_dir:
+        download_and_extract_tgz(url, md5sum, temp_dir)
 
-    target_path = Path(path) / 'pybind11'
-    if target_path.exists():
-        shutil.rmtree(target_path)
+        target_path = Path(path) / 'pybind11'
+        if target_path.exists():
+            shutil.rmtree(target_path)
+
+        os.makedirs(path, exist_ok=True)
+        shutil.move(str(Path(temp_dir) / out_dir_name / 'include/pybind11'),
+                    str(Path(path) / 'pybind11/pybind11'))
 
-    shutil.move(str(Path(path) / out_dir_name), str(Path(path) / 'pybind11'))
 
     return str(target_path)
 
diff --git a/setup.py b/setup.py
index d0372da90..03bac969b 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,6 @@
 import sys
 
 import numpy
-import pybind11
 np_include_dir = numpy.get_include()
 
 install_reqs = [
@@ -93,6 +92,19 @@ def get_include_path(library_name):
     return full_path
 
 
+def get_pybind11_include_path():
+    # If we can import pybind11, use that include path
+    try:
+        import pybind11
+    except ImportError:
+        pass
+    else:
+        return pybind11.get_include()
+
+    # Otherwise, we will download the source and include that
+    return get_include_path('pybind11')
+
+
 def get_cpp_extensions():
     cpp_transform_pkgdir = Path('hexrd') / 'transforms/cpp_sublibrary'
     src_files = [str(cpp_transform_pkgdir / 'src/inverse_distortion.cpp')]
@@ -106,7 +118,7 @@ def get_cpp_extensions():
     include_dirs = [
         get_include_path('eigen3'),
         get_include_path('xsimd'),
-        pybind11.get_include(),
+        get_pybind11_include_path(),
         numpy.get_include(),
     ]