diff --git a/.ci/setup.sh b/.ci/setup.sh
index fd6bda6fb746..f013a9b551ce 100755
--- a/.ci/setup.sh
+++ b/.ci/setup.sh
@@ -44,6 +44,8 @@ else  # Linux
             libicu66 \
             libssl1.1 \
             libunwind8 \
+            libxau-dev \
+            libxrender1 \
             locales \
             netcat \
             unzip \
@@ -81,6 +83,8 @@ else  # Linux
         apt-get update
         apt-get install --no-install-recommends -y \
             curl \
+            libxau-dev \
+            libxrender1 \
             lsb-release \
             software-properties-common
         if [[ $COMPILER == "clang" ]]; then
diff --git a/.ci/test.sh b/.ci/test.sh
index 701a9b6ba675..66745a0b6de5 100755
--- a/.ci/test.sh
+++ b/.ci/test.sh
@@ -103,8 +103,7 @@ conda install -q -y -n $CONDA_ENV cloudpickle dask distributed joblib matplotlib
 conda install -q -y \
     -n $CONDA_ENV \
     -c conda-forge \
-        python-graphviz \
-        xorg-libxau
+        python-graphviz
 
 if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
     # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)