From cf7516fcbcf573ba40d8ea692d5a380f41547496 Mon Sep 17 00:00:00 2001
From: spacewander <spacewanderlzx@gmail.com>
Date: Sat, 20 Apr 2019 11:35:31 +0800
Subject: [PATCH] optimize: added an NGINX core patch to ensure unused
 listening fds are closed when 'reuseport' is used.

When `reuseport` is enabled in the `listen` directive, Nginx will create
a listening fd for each worker process in the master process.

These fds will be inherited by the worker processes, but most of them
are unused. For example, considering we have 32 listening ip:port
configurations and 64 worker processes, each worker process will inherit
2048 (32 * 64) listening fds, but only 32 fds are used. By closing the
unused fds, this change could save up to 2016 (32 * 63) fds in a worker
process.

It doesn't affect the listening socket, since there is only one used fd
which associates to the socket with or without this change.

Co-authored-by: Thibault Charbonnier <thibaultcha@me.com>
---
 ...nx-1.17.1-reuseport_close_unused_fds.patch | 38 +++++++++
 t/002-reuseport-close-unused-fds.t            | 83 +++++++++++++++++++
 util/mirror-tarballs                          |  4 +
 3 files changed, 125 insertions(+)
 create mode 100644 patches/nginx-1.17.1-reuseport_close_unused_fds.patch
 create mode 100644 t/002-reuseport-close-unused-fds.t

diff --git a/patches/nginx-1.17.1-reuseport_close_unused_fds.patch b/patches/nginx-1.17.1-reuseport_close_unused_fds.patch
new file mode 100644
index 0000000..ff4a36f
--- /dev/null
+++ b/patches/nginx-1.17.1-reuseport_close_unused_fds.patch
@@ -0,0 +1,38 @@
+diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
+--- a/src/core/ngx_connection.c
++++ b/src/core/ngx_connection.c
+@@ -1118,6 +1118,12 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle)
+     ls = cycle->listening.elts;
+     for (i = 0; i < cycle->listening.nelts; i++) {
+ 
++#if (NGX_HAVE_REUSEPORT)
++        if (ls[i].fd == (ngx_socket_t) -1) {
++            continue;
++        }
++#endif
++
+         c = ls[i].connection;
+ 
+         if (c) {
+diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c
+--- a/src/event/ngx_event.c
++++ b/src/event/ngx_event.c
+@@ -775,6 +775,18 @@ ngx_event_process_init(ngx_cycle_t *cycle)
+ 
+ #if (NGX_HAVE_REUSEPORT)
+         if (ls[i].reuseport && ls[i].worker != ngx_worker) {
++            ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,
++                           "closing unused fd:%d listening on %V",
++                           ls[i].fd, &ls[i].addr_text);
++
++            if (ngx_close_socket(ls[i].fd) == -1) {
++                ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
++                              ngx_close_socket_n " %V failed",
++                              &ls[i].addr_text);
++            }
++
++            ls[i].fd = (ngx_socket_t) -1;
++
+             continue;
+         }
+ #endif
diff --git a/t/002-reuseport-close-unused-fds.t b/t/002-reuseport-close-unused-fds.t
new file mode 100644
index 0000000..a0290cc
--- /dev/null
+++ b/t/002-reuseport-close-unused-fds.t
@@ -0,0 +1,83 @@
+# vim:set ft= ts=4 sw=4 et fdm=marker:
+
+use Test::Nginx::Socket::Lua;
+
+master_on();
+workers(2);
+log_level('debug');
+
+repeat_each(1);
+
+plan tests => repeat_each() * (blocks() * 3);
+
+no_long_string();
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: closes unused file descriptors when 'reuseport' is enabled
+--- http_config
+    server {
+        listen 127.0.0.1:12345 reuseport;
+    }
+--- config
+    location /t {
+        return 200;
+    }
+--- request
+GET /t
+--- ignore_response_body
+--- no_error_log
+[error]
+--- grep_error_log eval: qr/\[debug\] .*? closing unused fd:\d+ listening on 127\.0\.0\.1:12345/
+--- grep_error_log_out eval
+qr/\[debug\] .*? closing unused fd:\d+ listening on 127\.0\.0\.1:12345
+\[debug\] .*? closing unused fd:\d+ listening on 127\.0\.0\.1:12345
+/
+
+
+
+=== TEST 2: closes multiple unused file descriptors when 'reuseport' is enabled
+--- http_config
+    server {
+        listen 127.0.0.1:12345 reuseport;
+    }
+
+    server {
+        listen 127.0.0.2:12345 reuseport;
+    }
+--- config
+    location /t {
+        return 200;
+    }
+--- request
+GET /t
+--- ignore_response_body
+--- no_error_log
+[error]
+--- grep_error_log eval: qr/\[debug\] .*? closing unused fd:\d+ listening on 127\.0\.0\.\d+:12345/
+--- grep_error_log_out eval
+qr/\[debug\] .*? closing unused fd:\d+ listening on 127\.0\.0\.1:12345
+\[debug\] .*? closing unused fd:\d+ listening on 127\.0\.0\.2:12345
+\[debug\] .*? closing unused fd:\d+ listening on 127\.0\.0\.1:12345
+\[debug\] .*? closing unused fd:\d+ listening on 127\.0\.0\.2:12345
+/
+
+
+
+=== TEST 3: does not close any fd if 'reuseport' is not used
+--- http_config
+    server {
+        listen 127.0.0.1:12345;
+    }
+--- config
+    location /t {
+        return 200;
+    }
+--- request
+GET /t
+--- ignore_response_body
+--- no_error_log
+[error]
+closing unused fd
diff --git a/util/mirror-tarballs b/util/mirror-tarballs
index cb6bb9b..8c08c91 100755
--- a/util/mirror-tarballs
+++ b/util/mirror-tarballs
@@ -437,6 +437,10 @@ echo "$info_txt applying the socket_cloexec patch for nginx"
 patch -p1 < $root/patches/nginx-$main_ver-socket_cloexec.patch || exit 1
 echo
 
+echo "$info_txt applying the reuseport_close_unused_fds patch for nginx"
+patch -p1 < $root/patches/nginx-$main_ver-reuseport_close_unused_fds.patch || exit 1
+echo
+
 cp $root/html/index.html docs/html/ || exit 1
 cp $root/html/50x.html docs/html/ || exit 1