# HG changeset patch
# User Sepherosa Ziehau <sepherosa@gmail.com>
# Date 1375418949 -28800
#      Fri Aug 02 12:49:09 2013 +0800
# Node ID a7f0011f531d74bf38330709c155c58255e5eec4
# Parent  ae3fd1ca62e0942a6c83a37f30df386622462c80
# Parent  1b83130dd94a46aeeb20abca9c83d41e1a2bd149
SO_REUSEPORT

diff -r ae3fd1ca62e0 -r a7f0011f531d src/core/nginx.c
--- a/src/core/nginx.c	Wed Jul 31 23:40:46 2013 +0400
+++ b/src/core/nginx.c	Fri Aug 02 12:49:09 2013 +0800
@@ -23,6 +23,8 @@
     void *conf);
 static char *ngx_set_worker_processes(ngx_conf_t *cf, ngx_command_t *cmd,
     void *conf);
+static char *ngx_set_so_reuseport(ngx_conf_t *cf, ngx_command_t *cmd,
+    void *conf);
 
 
 static ngx_conf_enum_t  ngx_debug_points[] = {
@@ -76,6 +78,13 @@
       0,
       NULL },
 
+    { ngx_string("so_reuseport"),
+      NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1,
+      ngx_set_so_reuseport,
+      0,
+      0,
+      NULL },
+
     { ngx_string("debug_points"),
       NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_enum_slot,
@@ -586,25 +595,31 @@
         return NGX_INVALID_PID;
     }
 
-    var = ngx_alloc(sizeof(NGINX_VAR)
-                    + cycle->listening.nelts * (NGX_INT32_LEN + 1) + 2,
-                    cycle->log);
-    if (var == NULL) {
-        ngx_free(env);
-        return NGX_INVALID_PID;
+    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
+
+    if (!ccf->so_reuseport) {
+        var = ngx_alloc(sizeof(NGINX_VAR)
+                        + cycle->listening.nelts * (NGX_INT32_LEN + 1) + 2,
+                        cycle->log);
+        if (var == NULL) {
+            ngx_free(env);
+            return NGX_INVALID_PID;
+        }
+
+        p = ngx_cpymem(var, NGINX_VAR "=", sizeof(NGINX_VAR));
+
+        ls = cycle->listening.elts;
+        for (i = 0; i < cycle->listening.nelts; i++) {
+            p = ngx_sprintf(p, "%ud;", ls[i].fd);
+        }
+
+        *p = '\0';
+
+        env[n++] = var;
+    } else {
+        var = NULL;
     }
 
-    p = ngx_cpymem(var, NGINX_VAR "=", sizeof(NGINX_VAR));
-
-    ls = cycle->listening.elts;
-    for (i = 0; i < cycle->listening.nelts; i++) {
-        p = ngx_sprintf(p, "%ud;", ls[i].fd);
-    }
-
-    *p = '\0';
-
-    env[n++] = var;
-
 #if (NGX_SETPROCTITLE_USES_ENV)
 
     /* allocate the spare 300 bytes for the new binary process title */
@@ -630,8 +645,6 @@
 
     ctx.envp = (char *const *) env;
 
-    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
-
     if (ngx_rename_file(ccf->pid.data, ccf->oldpid.data) == NGX_FILE_ERROR) {
         ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
                       ngx_rename_file_n " %s to %s failed "
@@ -639,7 +652,8 @@
                       ccf->pid.data, ccf->oldpid.data, argv[0]);
 
         ngx_free(env);
-        ngx_free(var);
+        if (var != NULL)
+            ngx_free(var);
 
         return NGX_INVALID_PID;
     }
@@ -658,7 +672,8 @@
     }
 
     ngx_free(env);
-    ngx_free(var);
+    if (var != NULL)
+        ngx_free(var);
 
     return pid;
 }
@@ -1361,3 +1376,26 @@
 
     return NGX_CONF_OK;
 }
+
+
+static char *
+ngx_set_so_reuseport(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+#ifdef SO_REUSEPORT
+    ngx_str_t        *value;
+    ngx_core_conf_t  *ccf;
+
+    ccf = (ngx_core_conf_t *) conf;
+
+    value = (ngx_str_t *) cf->args->elts;
+
+    if (ngx_strcmp(value[1].data, "on") == 0) {
+        ccf->so_reuseport = 1;
+    } else if (ngx_strcmp(value[1].data, "off") == 0) {
+        ccf->so_reuseport = 0;
+    } else {
+        return "invalid value";
+    }
+#endif
+    return NGX_CONF_OK;
+}
diff -r ae3fd1ca62e0 -r a7f0011f531d src/core/ngx_connection.c
--- a/src/core/ngx_connection.c	Wed Jul 31 23:40:46 2013 +0400
+++ b/src/core/ngx_connection.c	Fri Aug 02 12:49:09 2013 +0800
@@ -269,18 +269,26 @@
 ngx_int_t
 ngx_open_listening_sockets(ngx_cycle_t *cycle)
 {
-    int               reuseaddr;
+    int               reuseaddr, reuseport;
     ngx_uint_t        i, tries, failed;
     ngx_err_t         err;
     ngx_log_t        *log;
     ngx_socket_t      s;
     ngx_listening_t  *ls;
+    ngx_core_conf_t  *ccf;
 
     reuseaddr = 1;
+    reuseport = 0;
 #if (NGX_SUPPRESS_WARN)
     failed = 0;
 #endif
 
+    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
+    if (ccf->so_reuseport) {
+        reuseaddr = 0;
+        reuseport = 1;
+    }
+
     log = cycle->log;
 
     /* TODO: configurable try number */
@@ -318,21 +326,43 @@
                 return NGX_ERROR;
             }
 
-            if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
-                           (const void *) &reuseaddr, sizeof(int))
-                == -1)
-            {
-                ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
-                              "setsockopt(SO_REUSEADDR) %V failed",
-                              &ls[i].addr_text);
+            if (reuseaddr) {
+                if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR,
+                               (const void *) &reuseaddr, sizeof(int))
+                    == -1)
+                {
+                    ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+                                  "setsockopt(SO_REUSEADDR) %V failed",
+                                  &ls[i].addr_text);
 
-                if (ngx_close_socket(s) == -1) {
+                    if (ngx_close_socket(s) == -1) {
+                        ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+                                      ngx_close_socket_n " %V failed",
+                                      &ls[i].addr_text);
+                    }
+
+                    return NGX_ERROR;
+                }
+            }
+            if (reuseport) {
+#ifdef SO_REUSEPORT
+                if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT,
+                               (const void *) &reuseport, sizeof(int))
+                    == -1)
+                {
                     ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
-                                  ngx_close_socket_n " %V failed",
+                                  "setsockopt(SO_REUSEPORT) %V failed",
                                   &ls[i].addr_text);
+
+                    if (ngx_close_socket(s) == -1) {
+                        ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno,
+                                      ngx_close_socket_n " %V failed",
+                                      &ls[i].addr_text);
+                    }
+
+                    return NGX_ERROR;
                 }
-
-                return NGX_ERROR;
+#endif
             }
 
 #if (NGX_HAVE_INET6 && defined IPV6_V6ONLY)
@@ -462,6 +492,38 @@
 
 
 void
+ngx_shutdown_listening_sockets(ngx_cycle_t *cycle)
+{
+    ngx_uint_t        i;
+    ngx_listening_t  *ls;
+
+    ls = cycle->listening.elts;
+    for (i = 0; i < cycle->listening.nelts; i++) {
+
+        if (ls[i].ignore) {
+            continue;
+        }
+
+        if (ls[i].fd == -1) {
+            continue;
+        }
+
+        if (ls[i].inherited) {
+            continue;
+        }
+
+        if (ngx_close_socket(ls[i].fd) == -1) {
+            ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
+                          ngx_close_socket_n " %V failed", &ls[i].addr_text);
+        }
+        ls[i].fd = (ngx_socket_t) -1;
+
+        ls[i].listen = 0;
+    }
+}
+
+
+void
 ngx_configure_listening_sockets(ngx_cycle_t *cycle)
 {
     int                        value;
diff -r ae3fd1ca62e0 -r a7f0011f531d src/core/ngx_connection.h
--- a/src/core/ngx_connection.h	Wed Jul 31 23:40:46 2013 +0400
+++ b/src/core/ngx_connection.h	Fri Aug 02 12:49:09 2013 +0800
@@ -187,6 +187,7 @@
 ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle);
 void ngx_configure_listening_sockets(ngx_cycle_t *cycle);
 void ngx_close_listening_sockets(ngx_cycle_t *cycle);
+void ngx_shutdown_listening_sockets(ngx_cycle_t *cycle);
 void ngx_close_connection(ngx_connection_t *c);
 ngx_int_t ngx_connection_local_sockaddr(ngx_connection_t *c, ngx_str_t *s,
     ngx_uint_t port);
diff -r ae3fd1ca62e0 -r a7f0011f531d src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c	Wed Jul 31 23:40:46 2013 +0400
+++ b/src/core/ngx_cycle.c	Fri Aug 02 12:49:09 2013 +0800
@@ -569,7 +569,8 @@
         goto failed;
     }
 
-    if (!ngx_test_config) {
+    if (!ngx_test_config &&
+        (ngx_process == NGX_PROCESS_SINGLE || !ccf->so_reuseport)) {
         ngx_configure_listening_sockets(cycle);
     }
 
diff -r ae3fd1ca62e0 -r a7f0011f531d src/core/ngx_cycle.h
--- a/src/core/ngx_cycle.h	Wed Jul 31 23:40:46 2013 +0400
+++ b/src/core/ngx_cycle.h	Fri Aug 02 12:49:09 2013 +0800
@@ -103,6 +103,8 @@
      ngx_array_t              env;
      char                   **environment;
 
+     unsigned                 so_reuseport:1;
+
 #if (NGX_THREADS)
      ngx_int_t                worker_threads;
      size_t                   thread_stack_size;
diff -r ae3fd1ca62e0 -r a7f0011f531d src/event/ngx_event.c
--- a/src/event/ngx_event.c	Wed Jul 31 23:40:46 2013 +0400
+++ b/src/event/ngx_event.c	Fri Aug 02 12:49:09 2013 +0800
@@ -598,7 +598,8 @@
     ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
     ecf = ngx_event_get_conf(cycle->conf_ctx, ngx_event_core_module);
 
-    if (ccf->master && ccf->worker_processes > 1 && ecf->accept_mutex) {
+    if (ccf->master && ccf->worker_processes > 1 && ecf->accept_mutex &&
+        !ccf->so_reuseport) {
         ngx_use_accept_mutex = 1;
         ngx_accept_mutex_held = 0;
         ngx_accept_mutex_delay = ecf->accept_mutex_delay;
diff -r ae3fd1ca62e0 -r a7f0011f531d src/os/unix/ngx_process_cycle.c
--- a/src/os/unix/ngx_process_cycle.c	Wed Jul 31 23:40:46 2013 +0400
+++ b/src/os/unix/ngx_process_cycle.c	Fri Aug 02 12:49:09 2013 +0800
@@ -212,6 +212,9 @@
 
             ls = cycle->listening.elts;
             for (n = 0; n < cycle->listening.nelts; n++) {
+                if (ls[n].fd == -1)
+                    continue;
+
                 if (ngx_close_socket(ls[n].fd) == -1) {
                     ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
                                   ngx_close_socket_n " %V failed",
@@ -350,8 +353,9 @@
 static void
 ngx_start_worker_processes(ngx_cycle_t *cycle, ngx_int_t n, ngx_int_t type)
 {
-    ngx_int_t      i;
-    ngx_channel_t  ch;
+    ngx_int_t        i;
+    ngx_channel_t    ch;
+    ngx_core_conf_t *ccf;
 
     ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start worker processes");
 
@@ -359,11 +363,16 @@
 
     ch.command = NGX_CMD_OPEN_CHANNEL;
 
+    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
+
     for (i = 0; i < n; i++) {
 
         ngx_spawn_process(cycle, ngx_worker_process_cycle,
                           (void *) (intptr_t) i, "worker process", type);
 
+        if (i == 0 && ccf->so_reuseport)
+            ngx_shutdown_listening_sockets(cycle);
+
         ch.pid = ngx_processes[ngx_process_slot].pid;
         ch.slot = ngx_process_slot;
         ch.fd = ngx_processes[ngx_process_slot].channel[0];
@@ -853,13 +862,18 @@
     ngx_core_conf_t  *ccf;
     ngx_listening_t  *ls;
 
+    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
+
+    if (ccf->so_reuseport) {
+        ngx_open_listening_sockets(cycle);
+        ngx_configure_listening_sockets(cycle);
+    }
+
     if (ngx_set_environment(cycle, NULL) == NULL) {
         /* fatal */
         exit(2);
     }
 
-    ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module);
-
     if (worker >= 0 && ccf->priority != 0) {
         if (setpriority(PRIO_PROCESS, 0, ccf->priority) == -1) {
             ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
