From a65ad8b0d4ab394c00f21fdc1e59b11823374420 Mon Sep 17 00:00:00 2001
From: Earlopain <14981592+Earlopain@users.noreply.github.com>
Date: Fri, 8 Dec 2023 11:29:30 +0100
Subject: [PATCH] Stop counting fds after enough free ones are encountered

On systems with incredibly high `RLIMIT_NOFILE` (either intentionally or
by accident) this can take quite a large amount of time.

Closes #600
---
 include/rfb/rfb.h          |  3 +++
 src/libvncserver/main.c    |  1 +
 src/libvncserver/sockets.c | 16 ++++++++++------
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/include/rfb/rfb.h b/include/rfb/rfb.h
index 79a446f1b..83be2425f 100644
--- a/include/rfb/rfb.h
+++ b/include/rfb/rfb.h
@@ -363,6 +363,9 @@ typedef struct _rfbScreenInfo
 	of file descriptors LibVNCServer uses before denying new client connections.
 	It is set to 0.5 per default. */
     float fdQuota;
+    /** The amount of free file descriptors after which checking for more file
+    descriptors is not necessary anymore. Set to 100_000 by default. */
+    int fdSufficientFreeCount;
 #ifdef LIBVNCSERVER_HAVE_LIBPTHREAD
     pthread_t listener_thread;
     int pipe_notify_listener_thread[2];
diff --git a/src/libvncserver/main.c b/src/libvncserver/main.c
index 1efa83879..beb10d8be 100644
--- a/src/libvncserver/main.c
+++ b/src/libvncserver/main.c
@@ -956,6 +956,7 @@ rfbScreenInfoPtr rfbGetScreen(int* argc,char** argv,
 #endif
 
    screen->fdQuota = 0.5;
+   screen->fdSufficientFreeCount = 100000;
 
    screen->httpInitDone=FALSE;
    screen->httpEnableProxyConnect=FALSE;
diff --git a/src/libvncserver/sockets.c b/src/libvncserver/sockets.c
index c18493cbd..67bf56898 100644
--- a/src/libvncserver/sockets.c
+++ b/src/libvncserver/sockets.c
@@ -478,7 +478,7 @@ rfbProcessNewConnection(rfbScreenInfoPtr rfbScreen)
     rfbSocket chosen_listen_sock = RFB_INVALID_SOCKET;
 #if defined LIBVNCSERVER_HAVE_SYS_RESOURCE_H && defined LIBVNCSERVER_HAVE_FCNTL_H
     struct rlimit rlim;
-    size_t maxfds, curfds, i;
+    size_t maxfds, curopenfds, curfreefds, i;
 #endif
     /* Do another select() call to find out which listen socket
        has an incoming connection pending. We know that at least 
@@ -512,13 +512,17 @@ rfbProcessNewConnection(rfbScreenInfoPtr rfbScreen)
 	maxfds = rlim.rlim_cur;
 
     /* get the number of currently open fds as per https://stackoverflow.com/a/7976880/361413 */
-    curfds = 0;
-    for(i = 0; i < maxfds; ++i)
+    curopenfds = 0;
+    curfreefds = 0;
+    for(i = 0; i < maxfds, curfreefds < rfbScreen->fdSufficientFreeCount; ++i) {
 	if(fcntl(i, F_GETFD) != -1)
-	    ++curfds;
+	    ++curopenfds;
+    else
+        ++curfreefds;
+    }
 
-    if(curfds > maxfds * rfbScreen->fdQuota) {
-	rfbErr("rfbProcessNewconnection: open fd count of %lu exceeds quota %.1f of limit %lu, denying connection\n", curfds, rfbScreen->fdQuota, maxfds);
+    if(curfreefds < rfbScreen->fdSufficientFreeCount && curopenfds > maxfds * rfbScreen->fdQuota) {
+	rfbErr("rfbProcessNewconnection: open fd count of %lu exceeds quota %.1f of limit %lu, denying connection\n", curopenfds, rfbScreen->fdQuota, maxfds);
 	sock = accept(chosen_listen_sock, NULL, NULL);
 	rfbCloseSocket(sock);
 	return FALSE;