From 9fe35c1c8b64aeebe2a6f4fff9f2acb5eccc7f0e Mon Sep 17 00:00:00 2001 From: WangKai Date: Tue, 27 Mar 2018 14:48:29 +0800 Subject: [PATCH] support intel qat --- patches/nginx-1.13.6-support-intel-qat.patch | 2046 ++++++++++++++++++ 1 file changed, 2046 insertions(+) create mode 100644 patches/nginx-1.13.6-support-intel-qat.patch diff --git a/patches/nginx-1.13.6-support-intel-qat.patch b/patches/nginx-1.13.6-support-intel-qat.patch new file mode 100644 index 0000000..d39945c --- /dev/null +++ b/patches/nginx-1.13.6-support-intel-qat.patch @@ -0,0 +1,2046 @@ +diff --git a/src/auto/cc/gcc b/src/auto/cc/gcc +index a5ca336..52de374 100644 +--- a/src/auto/cc/gcc ++++ b/src/auto/cc/gcc +@@ -48,9 +48,9 @@ esac + + # optimizations + +-#NGX_GCC_OPT="-O2" ++NGX_GCC_OPT="-O2 -D_FORTIFY_SOURCE=2" + #NGX_GCC_OPT="-Os" +-NGX_GCC_OPT="-O" ++#NGX_GCC_OPT="-O" + + #CFLAGS="$CFLAGS -fomit-frame-pointer" + +@@ -174,6 +174,10 @@ CFLAGS="$CFLAGS -g" + # DragonFly's gcc3 generates DWARF + #CFLAGS="$CFLAGS -g -gstabs" + ++# required by Intel SDL ++CFLAGS="$CFLAGS -fstack-protector -fPIE -fPIC -Wformat -Wformat-security" ++CORE_LINK="-z noexecstack -z relro -z now -pie" ++ + if [ ".$CPP" = "." ]; then + CPP="$CC -E" + fi +diff --git a/src/src/core/nginx.c b/src/src/core/nginx.c +index aa4986b..12ca7be 100644 +--- a/src/src/core/nginx.c ++++ b/src/src/core/nginx.c +@@ -182,6 +182,10 @@ ngx_module_t ngx_core_module = { + static ngx_uint_t ngx_show_help; + static ngx_uint_t ngx_show_version; + static ngx_uint_t ngx_show_configure; ++/* indicate that nginx start without ngx_ssl_init() ++ * which will involve OpenSSL configuration file to ++ * start OpenSSL engine */ ++static ngx_uint_t ngx_no_ssl_init; + static u_char *ngx_prefix; + static u_char *ngx_conf_file; + static u_char *ngx_conf_params; +@@ -200,6 +204,7 @@ main(int argc, char *const *argv) + ngx_cycle_t *cycle, init_cycle; + ngx_conf_dump_t *cd; + ngx_core_conf_t *ccf; ++ ngx_int_t ret_int_num; + + ngx_debug_init(); + +@@ -236,7 +241,8 @@ main(int argc, char *const *argv) + + /* STUB */ + #if (NGX_OPENSSL) +- ngx_ssl_init(log); ++ if(!ngx_no_ssl_init) ++ ngx_ssl_init(log); + #endif + + /* +@@ -246,6 +252,7 @@ main(int argc, char *const *argv) + + ngx_memzero(&init_cycle, sizeof(ngx_cycle_t)); + init_cycle.log = log; ++ init_cycle.no_ssl_init = ngx_no_ssl_init; + ngx_cycle = &init_cycle; + + init_cycle.pool = ngx_create_pool(1024, log); +@@ -324,7 +331,9 @@ main(int argc, char *const *argv) + } + + if (ngx_signal) { +- return ngx_signal_process(cycle, ngx_signal); ++ ret_int_num = ngx_signal_process(cycle, ngx_signal); ++ ngx_destroy_pool(cycle->pool); ++ return ret_int_num; + } + + ngx_os_status(cycle->log); +@@ -344,7 +353,7 @@ main(int argc, char *const *argv) + } + + if (!ngx_inherited && ccf->daemon) { +- if (ngx_daemon(cycle->log) != NGX_OK) { ++ if (ngx_daemon(cycle) != NGX_OK) { + return 1; + } + +@@ -773,11 +782,13 @@ ngx_get_options(int argc, char *const *argv) + + case 't': + ngx_test_config = 1; ++ ngx_no_ssl_init = 1; + break; + + case 'T': + ngx_test_config = 1; + ngx_dump_config = 1; ++ ngx_no_ssl_init = 1; + break; + + case 'q': +@@ -827,6 +838,7 @@ ngx_get_options(int argc, char *const *argv) + return NGX_ERROR; + + case 's': ++ ngx_no_ssl_init = 1; + if (*p) { + ngx_signal = (char *) p; + +@@ -852,6 +864,7 @@ ngx_get_options(int argc, char *const *argv) + + default: + ngx_log_stderr(0, "invalid option: \"%c\"", *(p - 1)); ++ ngx_no_ssl_init = 1; + return NGX_ERROR; + } + } +diff --git a/src/src/core/ngx_buf.h b/src/src/core/ngx_buf.h +index 12781a7..bcc5912 100644 +--- a/src/src/core/ngx_buf.h ++++ b/src/src/core/ngx_buf.h +@@ -137,7 +137,7 @@ typedef struct { + && !ngx_buf_in_memory(b) && !b->in_file && !b->flush && !b->last_buf) + + #define ngx_buf_size(b) \ +- (ngx_buf_in_memory(b) ? (off_t) (b->last - b->pos): \ ++ (ngx_buf_in_memory(b) ? (off_t) (unsigned) (b->last - b->pos): \ + (b->file_last - b->file_pos)) + + ngx_buf_t *ngx_create_temp_buf(ngx_pool_t *pool, size_t size); +diff --git a/src/src/core/ngx_conf_file.h b/src/src/core/ngx_conf_file.h +index 213611f..9596341 100644 +--- a/src/src/core/ngx_conf_file.h ++++ b/src/src/core/ngx_conf_file.h +@@ -129,6 +129,7 @@ struct ngx_conf_s { + + ngx_conf_handler_pt handler; + char *handler_conf; ++ ngx_flag_t no_ssl_init; + }; + + +diff --git a/src/src/core/ngx_connection.c b/src/src/core/ngx_connection.c +index 9a74758..f39d358 100644 +--- a/src/src/core/ngx_connection.c ++++ b/src/src/core/ngx_connection.c +@@ -986,7 +986,14 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle) + * for closed shared listening sockets unless + * the events was explicitly deleted + */ +- ++#if (NGX_SSL) ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif + ngx_del_event(c->read, NGX_READ_EVENT, 0); + + } else { +@@ -1035,6 +1042,9 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log) + { + ngx_uint_t instance; + ngx_event_t *rev, *wev; ++#if (NGX_SSL) ++ ngx_event_t *aev; ++#endif + ngx_connection_t *c; + + /* disable warning: Win32 SOCKET is u_int while UNIX socket is int */ +@@ -1071,11 +1081,18 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log) + + rev = c->read; + wev = c->write; ++#if (NGX_SSL) ++ aev = c->async; ++#endif + + ngx_memzero(c, sizeof(ngx_connection_t)); + + c->read = rev; + c->write = wev; ++#if (NGX_SSL) ++ c->async = aev; ++#endif ++ + c->fd = s; + c->log = log; + +@@ -1083,17 +1100,32 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log) + + ngx_memzero(rev, sizeof(ngx_event_t)); + ngx_memzero(wev, sizeof(ngx_event_t)); ++#if (NGX_SSL) ++ ngx_memzero(aev, sizeof(ngx_event_t)); ++#endif + + rev->instance = !instance; + wev->instance = !instance; ++#if (NGX_SSL) ++ aev->instance = !instance; ++#endif + + rev->index = NGX_INVALID_INDEX; + wev->index = NGX_INVALID_INDEX; ++#if (NGX_SSL) ++ aev->index = NGX_INVALID_INDEX; ++#endif + + rev->data = c; + wev->data = c; ++#if (NGX_SSL) ++ aev->data = c; ++#endif + + wev->write = 1; ++#if (NGX_SSL) ++ aev->async = 1; ++#endif + + return c; + } +@@ -1132,11 +1164,32 @@ ngx_close_connection(ngx_connection_t *c) + ngx_del_timer(c->write); + } + ++#if (NGX_SSL) ++ if (c->async->timer_set) { ++ ngx_del_timer(c->async); ++ } ++ ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif ++ + if (!c->shared) { + if (ngx_del_conn) { + ngx_del_conn(c, NGX_CLOSE_EVENT); + + } else { ++#if (NGX_SSL) ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif + if (c->read->active || c->read->disabled) { + ngx_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT); + } +@@ -1155,8 +1208,17 @@ ngx_close_connection(ngx_connection_t *c) + ngx_delete_posted_event(c->write); + } + ++#if (NGX_SSL) ++ if (c->async->posted) { ++ ngx_delete_posted_event(c->async); ++ } ++#endif ++ + c->read->closed = 1; + c->write->closed = 1; ++#if (NGX_SSL) ++ c->async->closed = 1; ++#endif + + ngx_reusable_connection(c, 0); + +@@ -1166,6 +1228,9 @@ ngx_close_connection(ngx_connection_t *c) + + fd = c->fd; + c->fd = (ngx_socket_t) -1; ++#if (NGX_SSL) ++ c->async_fd = (ngx_socket_t) -1; ++#endif + + if (c->shared) { + return; +diff --git a/src/src/core/ngx_connection.h b/src/src/core/ngx_connection.h +index e4dfe58..e746698 100644 +--- a/src/src/core/ngx_connection.h ++++ b/src/src/core/ngx_connection.h +@@ -122,9 +122,14 @@ struct ngx_connection_s { + void *data; + ngx_event_t *read; + ngx_event_t *write; ++#if (NGX_SSL) ++ ngx_event_t *async; ++#endif + + ngx_socket_t fd; +- ++#if (NGX_SSL) ++ ngx_socket_t async_fd; ++#endif + ngx_recv_pt recv; + ngx_send_pt send; + ngx_recv_chain_pt recv_chain; +@@ -149,6 +154,7 @@ struct ngx_connection_s { + + #if (NGX_SSL || NGX_COMPAT) + ngx_ssl_connection_t *ssl; ++ ngx_flag_t asynch; + #endif + + struct sockaddr *local_sockaddr; +@@ -181,7 +187,9 @@ struct ngx_connection_s { + unsigned tcp_nopush:2; /* ngx_connection_tcp_nopush_e */ + + unsigned need_last_buf:1; +- ++#if (NGX_SSL) ++ unsigned num_async_fds:8; ++#endif + #if (NGX_HAVE_AIO_SENDFILE || NGX_COMPAT) + unsigned busy_count:2; + #endif +diff --git a/src/src/core/ngx_cycle.c b/src/src/core/ngx_cycle.c +index 675a506..37e7eac 100644 +--- a/src/src/core/ngx_cycle.c ++++ b/src/src/core/ngx_cycle.c +@@ -81,6 +81,7 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) + cycle->pool = pool; + cycle->log = log; + cycle->old_cycle = old_cycle; ++ cycle->no_ssl_init = old_cycle->no_ssl_init; + + cycle->conf_prefix.len = old_cycle->conf_prefix.len; + cycle->conf_prefix.data = ngx_pstrdup(pool, &old_cycle->conf_prefix); +@@ -261,6 +262,7 @@ ngx_init_cycle(ngx_cycle_t *old_cycle) + conf.log = log; + conf.module_type = NGX_CORE_MODULE; + conf.cmd_type = NGX_MAIN_CONF; ++ conf.no_ssl_init = cycle->no_ssl_init; + + #if 0 + log->log_level = NGX_LOG_DEBUG_ALL; +@@ -962,6 +964,7 @@ ngx_create_pidfile(ngx_str_t *name, ngx_log_t *log) + len = ngx_snprintf(pid, NGX_INT64_LEN + 2, "%P%N", ngx_pid) - pid; + + if (ngx_write_file(&file, pid, len, 0) == NGX_ERROR) { ++ ngx_close_file(file.fd); + return NGX_ERROR; + } + } +diff --git a/src/src/core/ngx_cycle.h b/src/src/core/ngx_cycle.h +index c342365..fb37394 100644 +--- a/src/src/core/ngx_cycle.h ++++ b/src/src/core/ngx_cycle.h +@@ -79,6 +79,9 @@ struct ngx_cycle_s { + ngx_connection_t *connections; + ngx_event_t *read_events; + ngx_event_t *write_events; ++#if (NGX_SSL) ++ ngx_event_t *async_events; ++#endif + + ngx_cycle_t *old_cycle; + +@@ -88,6 +91,7 @@ struct ngx_cycle_s { + ngx_str_t prefix; + ngx_str_t lock_file; + ngx_str_t hostname; ++ ngx_flag_t no_ssl_init; + + ngx_log_intercept_pt intercept_error_log_handler; + void *intercept_error_log_data; +diff --git a/src/src/core/ngx_string.h b/src/src/core/ngx_string.h +index 882ae7c..e0a32fa 100644 +--- a/src/src/core/ngx_string.h ++++ b/src/src/core/ngx_string.h +@@ -96,6 +96,16 @@ void *ngx_memcpy(void *dst, const void *src, size_t n); + + #else + ++#if (NGX_SECURE_MEM) ++ ++#define _MIN_(a,b) (((a)<(b))?(a):(b)) ++#define MEMCPY_S(dest, src, dest_sz, src_sz) \ ++ memcpy((void *)(dest), (void *) (src), (size_t)_MIN_(dest_sz, src_sz)) ++#define ngx_memcpy(dst, src, n) (void) MEMCPY_S(dst, src, n, n) ++#define ngx_cpymem(dst, src, n) (((u_char *) MEMCPY_S(dst, src, n, n)) + (n)) ++ ++#else ++ + /* + * gcc3, msvc, and icc7 compile memcpy() to the inline "rep movs". + * gcc3 compiles memcpy(d, s, 4) to the inline "mov"es. +@@ -104,8 +114,9 @@ void *ngx_memcpy(void *dst, const void *src, size_t n); + #define ngx_memcpy(dst, src, n) (void) memcpy(dst, src, n) + #define ngx_cpymem(dst, src, n) (((u_char *) memcpy(dst, src, n)) + (n)) + +-#endif ++#endif /* NGX_SECURE_MEM */ + ++#endif /* NGX_MEMCPY_LIMIT */ + + #if ( __INTEL_COMPILER >= 800 ) + +diff --git a/src/src/event/modules/ngx_devpoll_module.c b/src/src/event/modules/ngx_devpoll_module.c +index ee9f854..9dd2df3 100644 +--- a/src/src/event/modules/ngx_devpoll_module.c ++++ b/src/src/event/modules/ngx_devpoll_module.c +@@ -94,6 +94,8 @@ static ngx_event_module_t ngx_devpoll_module_ctx = { + ngx_devpoll_process_events, /* process the events */ + ngx_devpoll_init, /* init the events */ + ngx_devpoll_done, /* done the events */ ++ NULL, /* add an async conn */ ++ NULL /* del an async conn */ + } + + }; +diff --git a/src/src/event/modules/ngx_epoll_module.c b/src/src/event/modules/ngx_epoll_module.c +index 76aee08..ab5f7e3 100644 +--- a/src/src/event/modules/ngx_epoll_module.c ++++ b/src/src/event/modules/ngx_epoll_module.c +@@ -122,6 +122,11 @@ static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler); + #endif + static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, + ngx_uint_t flags); ++#if (NGX_SSL) ++static ngx_int_t ngx_epoll_add_async_connection(ngx_connection_t *c); ++static ngx_int_t ngx_epoll_del_async_connection(ngx_connection_t *c, ++ ngx_uint_t flags); ++#endif + + #if (NGX_HAVE_FILE_AIO) + static void ngx_epoll_eventfd_handler(ngx_event_t *ev); +@@ -196,6 +201,13 @@ static ngx_event_module_t ngx_epoll_module_ctx = { + ngx_epoll_process_events, /* process the events */ + ngx_epoll_init, /* init the events */ + ngx_epoll_done, /* done the events */ ++#if (NGX_SSL) ++ ngx_epoll_add_async_connection, /* add an async conn */ ++ ngx_epoll_del_async_connection /* del an async conn */ ++#else ++ NULL, /* add an async conn */ ++ NULL /* del an async conn */ ++#endif + } + }; + +@@ -626,7 +638,7 @@ ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags) + + if (epoll_ctl(ep, op, c->fd, &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, +- "epoll_ctl(%d, %d) failed", op, c->fd); ++ "socket add event epoll_ctl(%d, %d) failed", op, c->fd); + return NGX_ERROR; + } + +@@ -687,7 +699,7 @@ ngx_epoll_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags) + + if (epoll_ctl(ep, op, c->fd, &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_errno, +- "epoll_ctl(%d, %d) failed", op, c->fd); ++ "socket del event epoll_ctl(%d, %d) failed", op, c->fd); + return NGX_ERROR; + } + +@@ -710,7 +722,7 @@ ngx_epoll_add_connection(ngx_connection_t *c) + + if (epoll_ctl(ep, EPOLL_CTL_ADD, c->fd, &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, +- "epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->fd); ++ "socket add_conn epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->fd); + return NGX_ERROR; + } + +@@ -748,7 +760,7 @@ ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags) + + if (epoll_ctl(ep, op, c->fd, &ee) == -1) { + ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, +- "epoll_ctl(%d, %d) failed", op, c->fd); ++ "socket del conn epoll_ctl(%d, %d) failed", op, c->fd); + return NGX_ERROR; + } + +@@ -758,6 +770,53 @@ ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags) + return NGX_OK; + } + ++#if (NGX_SSL) ++static ngx_int_t ++ngx_epoll_add_async_connection(ngx_connection_t *c) ++{ ++ struct epoll_event ee; ++ ++ ee.events = EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP; ++ ee.data.ptr = (void *) ((uintptr_t) c | (c->async->async << 1) | c->async->instance); ++ ++ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "epoll add async connection: fd:%d ev:%08XD", c->async_fd, ee.events); ++ if (epoll_ctl(ep, EPOLL_CTL_ADD, c->async_fd, &ee) == -1) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, ++ "async add conn epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->async_fd); ++ return NGX_ERROR; ++ } ++ ++ c->async->active = 1; ++ ++ return NGX_OK; ++} ++ ++ ++static ngx_int_t ++ngx_epoll_del_async_connection(ngx_connection_t *c, ngx_uint_t flags) ++{ ++ int op; ++ struct epoll_event ee; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "epoll del async connection: fd:%d", c->async_fd); ++ ++ op = EPOLL_CTL_DEL; ++ ee.events = 0; ++ ee.data.ptr = NULL; ++ if (epoll_ctl(ep, op, c->async_fd, &ee) == -1) { ++ ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, ++ "async del conn epoll_ctl(%d, %d) failed", op, c->async_fd); ++ c->async_fd = -1; ++ return NGX_ERROR; ++ } ++ c->async_fd = -1; ++ c->async->active = 0; ++ ++ return NGX_OK; ++} ++#endif + + #if (NGX_HAVE_EVENTFD) + +@@ -791,6 +850,10 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) + ngx_event_t *rev, *wev; + ngx_queue_t *queue; + ngx_connection_t *c; ++#if (NGX_SSL) ++ ngx_int_t async; ++ ngx_event_t *aev; ++#endif + + /* NGX_TIMER_INFINITE == INFTIM */ + +@@ -837,7 +900,12 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) + c = event_list[i].data.ptr; + + instance = (uintptr_t) c & 1; ++#if (NGX_SSL) ++ async = ((uintptr_t) c & 2) >> 1; ++ c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~3); ++#else + c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~1); ++#endif + + rev = c->read; + +@@ -880,7 +948,11 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) + } + #endif + ++#if (NGX_SSL) ++ if ((revents & EPOLLIN) && rev->active && !async) { ++#else + if ((revents & EPOLLIN) && rev->active) { ++#endif + + #if (NGX_HAVE_EPOLLRDHUP) + if (revents & EPOLLRDHUP) { +@@ -905,7 +977,11 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) + + wev = c->write; + ++#if (NGX_SSL) ++ if ((revents & EPOLLOUT) && wev->active && !async) { ++#else + if ((revents & EPOLLOUT) && wev->active) { ++#endif + + if (c->fd == -1 || wev->instance != instance) { + +@@ -931,6 +1007,33 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) + wev->handler(wev); + } + } ++ ++#if (NGX_SSL) ++ aev = c->async; ++ ++ if ((revents & EPOLLIN) && aev->active && async) { ++ ++ if (c->async_fd == -1 || aev->instance!= instance) { ++ /* ++ * the stale event from a file descriptor ++ * that was just closed in this iteration ++ */ ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, ++ "epoll: stale event %p", c); ++ continue; ++ } ++ ++ aev->ready = 1; ++ ++ if (flags & NGX_POST_EVENTS) { ++ ngx_post_event(aev, &ngx_posted_events); ++ ++ } else { ++ aev->handler(aev); ++ } ++ } ++#endif + } + + return NGX_OK; +diff --git a/src/src/event/modules/ngx_eventport_module.c b/src/src/event/modules/ngx_eventport_module.c +index e723f92..ef25fd0 100644 +--- a/src/src/event/modules/ngx_eventport_module.c ++++ b/src/src/event/modules/ngx_eventport_module.c +@@ -185,6 +185,8 @@ static ngx_event_module_t ngx_eventport_module_ctx = { + ngx_eventport_process_events, /* process the events */ + ngx_eventport_init, /* init the events */ + ngx_eventport_done, /* done the events */ ++ NULL, /* add an async conn */ ++ NULL /* del an async conn */ + } + + }; +diff --git a/src/src/event/modules/ngx_kqueue_module.c b/src/src/event/modules/ngx_kqueue_module.c +index 9c7244c..c775077 100644 +--- a/src/src/event/modules/ngx_kqueue_module.c ++++ b/src/src/event/modules/ngx_kqueue_module.c +@@ -92,7 +92,9 @@ static ngx_event_module_t ngx_kqueue_module_ctx = { + #endif + ngx_kqueue_process_events, /* process the events */ + ngx_kqueue_init, /* init the events */ +- ngx_kqueue_done /* done the events */ ++ ngx_kqueue_done, /* done the events */ ++ NULL, /* add an async conn */ ++ NULL /* del an async conn */ + } + + }; +diff --git a/src/src/event/modules/ngx_poll_module.c b/src/src/event/modules/ngx_poll_module.c +index 4e03dab..c679d92 100644 +--- a/src/src/event/modules/ngx_poll_module.c ++++ b/src/src/event/modules/ngx_poll_module.c +@@ -42,7 +42,9 @@ static ngx_event_module_t ngx_poll_module_ctx = { + NULL, /* trigger a notify */ + ngx_poll_process_events, /* process the events */ + ngx_poll_init, /* init the events */ +- ngx_poll_done /* done the events */ ++ ngx_poll_done, /* done the events */ ++ NULL, /* add an async conn */ ++ NULL /* del an async conn */ + } + + }; +diff --git a/src/src/event/modules/ngx_select_module.c b/src/src/event/modules/ngx_select_module.c +index 0644621..4e64a32 100644 +--- a/src/src/event/modules/ngx_select_module.c ++++ b/src/src/event/modules/ngx_select_module.c +@@ -50,7 +50,9 @@ static ngx_event_module_t ngx_select_module_ctx = { + NULL, /* trigger a notify */ + ngx_select_process_events, /* process the events */ + ngx_select_init, /* init the events */ +- ngx_select_done /* done the events */ ++ ngx_select_done, /* done the events */ ++ NULL, /* add an async conn */ ++ NULL /* del an async conn */ + } + + }; +diff --git a/src/src/event/modules/ngx_win32_select_module.c b/src/src/event/modules/ngx_win32_select_module.c +index a98a83f..738a96f 100644 +--- a/src/src/event/modules/ngx_win32_select_module.c ++++ b/src/src/event/modules/ngx_win32_select_module.c +@@ -51,7 +51,9 @@ static ngx_event_module_t ngx_select_module_ctx = { + NULL, /* trigger a notify */ + ngx_select_process_events, /* process the events */ + ngx_select_init, /* init the events */ +- ngx_select_done /* done the events */ ++ ngx_select_done, /* done the events */ ++ NULL, /* add an async conn */ ++ NULL /* del an async conn */ + } + + }; +diff --git a/src/src/event/ngx_event.c b/src/src/event/ngx_event.c +index 4853945..39cc7e5 100644 +--- a/src/src/event/ngx_event.c ++++ b/src/src/event/ngx_event.c +@@ -170,7 +170,7 @@ static ngx_event_module_t ngx_event_core_module_ctx = { + ngx_event_core_create_conf, /* create configuration */ + ngx_event_core_init_conf, /* init configuration */ + +- { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL } ++ { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL } + }; + + +@@ -608,6 +608,9 @@ ngx_event_process_init(ngx_cycle_t *cycle) + { + ngx_uint_t m, i; + ngx_event_t *rev, *wev; ++#if (NGX_SSL) ++ ngx_event_t *aev; ++#endif + ngx_listening_t *ls; + ngx_connection_t *c, *next, *old; + ngx_core_conf_t *ccf; +@@ -751,6 +754,20 @@ ngx_event_process_init(ngx_cycle_t *cycle) + wev[i].closed = 1; + } + ++#if (NGX_SSL) ++ cycle->async_events = ngx_alloc(sizeof(ngx_event_t) * cycle->connection_n, ++ cycle->log); ++ if (cycle->async_events == NULL) { ++ return NGX_ERROR; ++ } ++ ++ aev = cycle->async_events; ++ for (i = 0; i < cycle->connection_n; i++) { ++ aev[i].closed = 1; ++ aev[i].instance = 1; ++ } ++#endif ++ + i = cycle->connection_n; + next = NULL; + +@@ -761,6 +778,10 @@ ngx_event_process_init(ngx_cycle_t *cycle) + c[i].read = &cycle->read_events[i]; + c[i].write = &cycle->write_events[i]; + c[i].fd = (ngx_socket_t) -1; ++#if (NGX_SSL) ++ c[i].async = &cycle->async_events[i]; ++ c[i].async_fd = (ngx_socket_t) -1; ++#endif + + next = &c[i]; + } while (i); +diff --git a/src/src/event/ngx_event.h b/src/src/event/ngx_event.h +index 19fec68..dc856b1 100644 +--- a/src/src/event/ngx_event.h ++++ b/src/src/event/ngx_event.h +@@ -32,6 +32,10 @@ struct ngx_event_s { + + unsigned write:1; + ++#if (NGX_SSL) ++ unsigned async:1; ++#endif ++ + unsigned accept:1; + + /* used to detect the stale events in kqueue and epoll */ +@@ -108,6 +112,9 @@ struct ngx_event_s { + #endif + + ngx_event_handler_pt handler; ++#if (NGX_SSL) ++ ngx_event_handler_pt saved_handler; ++#endif + + + #if (NGX_HAVE_IOCP) +@@ -191,6 +198,9 @@ typedef struct { + + ngx_int_t (*init)(ngx_cycle_t *cycle, ngx_msec_t timer); + void (*done)(ngx_cycle_t *cycle); ++ ++ ngx_int_t (*add_async_conn)(ngx_connection_t *c); ++ ngx_int_t (*del_async_conn)(ngx_connection_t *c, ngx_uint_t flags); + } ngx_event_actions_t; + + +@@ -415,6 +425,8 @@ extern ngx_uint_t ngx_use_epoll_rdhup; + #define ngx_del_event ngx_event_actions.del + #define ngx_add_conn ngx_event_actions.add_conn + #define ngx_del_conn ngx_event_actions.del_conn ++#define ngx_add_async_conn ngx_event_actions.add_async_conn ++#define ngx_del_async_conn ngx_event_actions.del_async_conn + + #define ngx_notify ngx_event_actions.notify + +diff --git a/src/src/event/ngx_event_accept.c b/src/src/event/ngx_event_accept.c +index 7756370..dc22320 100644 +--- a/src/src/event/ngx_event_accept.c ++++ b/src/src/event/ngx_event_accept.c +@@ -249,6 +249,9 @@ ngx_event_accept(ngx_event_t *ev) + + rev->log = log; + wev->log = log; ++#if (NGX_SSL) ++ c->async->log = log; ++#endif + + /* + * TODO: MT: - ngx_atomic_fetch_add() +@@ -735,6 +738,15 @@ ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all) + + #endif + ++#if (NGX_SSL) ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif ++ + if (ngx_del_event(c->read, NGX_READ_EVENT, NGX_DISABLE_EVENT) + == NGX_ERROR) + { +diff --git a/src/src/event/ngx_event_openssl.c b/src/src/event/ngx_event_openssl.c +index 7ca1abc..347607d 100644 +--- a/src/src/event/ngx_event_openssl.c ++++ b/src/src/event/ngx_event_openssl.c +@@ -69,6 +69,10 @@ static void *ngx_openssl_create_conf(ngx_cycle_t *cycle); + static char *ngx_openssl_engine(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); + static void ngx_openssl_exit(ngx_cycle_t *cycle); + ++static void ngx_ssl_handshake_async_handler(ngx_event_t * aev); ++static void ngx_ssl_read_async_handler(ngx_event_t * aev); ++static void ngx_ssl_write_async_handler(ngx_event_t * aev); ++static void ngx_ssl_shutdown_async_handler(ngx_event_t *aev); + + static ngx_command_t ngx_openssl_commands[] = { + +@@ -116,6 +120,15 @@ int ngx_ssl_certificate_name_index; + int ngx_ssl_stapling_index; + + ++static void ++ngx_ssl_empty_handler(ngx_event_t *aev) ++{ ++ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, aev->log, 0, "ssl empty handler"); ++ ++ return; ++} ++ ++ + ngx_int_t + ngx_ssl_init(ngx_log_t *log) + { +@@ -342,6 +355,10 @@ ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data) + SSL_CTX_set_mode(ssl->ctx, SSL_MODE_NO_AUTO_CHAIN); + #endif + ++ if(ssl->asynch) { ++ SSL_CTX_set_mode(ssl->ctx, SSL_MODE_ASYNC); ++ } ++ + SSL_CTX_set_read_ahead(ssl->ctx, 1); + + SSL_CTX_set_info_callback(ssl->ctx, ngx_ssl_info_callback); +@@ -1202,6 +1219,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags) + } + + c->ssl = sc; ++ c->asynch = ssl->asynch; + + return NGX_OK; + } +@@ -1220,6 +1238,78 @@ ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session) + return NGX_OK; + } + ++ngx_int_t ++ngx_ssl_async_process_fds(ngx_connection_t *c) ++{ ++ OSSL_ASYNC_FD *add_fds = NULL; ++ OSSL_ASYNC_FD *del_fds = NULL; ++ size_t num_add_fds = 0; ++ size_t num_del_fds = 0; ++ unsigned loop = 0; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "ngx_ssl_async_process_fds called"); ++ ++ if (!ngx_del_async_conn || !ngx_add_async_conn) { ++ ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, ++ "Async notifications not supported"); ++ return 0; ++ } ++ ++ SSL_get_changed_async_fds(c->ssl->connection, NULL, &num_add_fds, ++ NULL, &num_del_fds); ++ ++ if (num_add_fds) { ++ add_fds = ngx_alloc(num_add_fds * sizeof(OSSL_ASYNC_FD), c->log); ++ if (add_fds == NULL) { ++ ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, ++ "Memory Allocation Error"); ++ return 0; ++ } ++ } ++ ++ if (num_del_fds) { ++ del_fds = ngx_alloc(num_del_fds * sizeof(OSSL_ASYNC_FD), c->log); ++ if (del_fds == NULL) { ++ ngx_ssl_error(NGX_LOG_ALERT, c->log, 0, ++ "Memory Allocation Error"); ++ if (add_fds) ++ ngx_free(add_fds); ++ return 0; ++ } ++ } ++ ++ SSL_get_changed_async_fds(c->ssl->connection, add_fds, &num_add_fds, ++ del_fds, &num_del_fds); ++ ++ if (num_del_fds) { ++ for (loop = 0; loop < num_del_fds; loop++) { ++ c->async_fd = del_fds[loop]; ++ if (c->num_async_fds) { ++ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "%s: deleting fd = %d", __func__, c->async_fd); ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++ } ++ if (num_add_fds) { ++ for (loop = 0; loop < num_add_fds; loop++) { ++ if (c->num_async_fds == 0) { ++ c->num_async_fds++; ++ c->async_fd = add_fds[loop]; ++ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "%s: adding fd = %d", __func__, c->async_fd); ++ ngx_add_async_conn(c); ++ } ++ } ++ } ++ ++ if (add_fds) ++ ngx_free(add_fds); ++ if (del_fds) ++ ngx_free(del_fds); ++ ++ return 1; ++} + + ngx_int_t + ngx_ssl_handshake(ngx_connection_t *c) +@@ -1235,6 +1325,10 @@ ngx_ssl_handshake(ngx_connection_t *c) + + if (n == 1) { + ++ if(c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } ++ + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { + return NGX_ERROR; + } +@@ -1316,6 +1410,10 @@ ngx_ssl_handshake(ngx_connection_t *c) + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr); + + if (sslerr == SSL_ERROR_WANT_READ) { ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } ++ + c->read->ready = 0; + c->read->handler = ngx_ssl_handshake_handler; + c->write->handler = ngx_ssl_handshake_handler; +@@ -1332,6 +1430,9 @@ ngx_ssl_handshake(ngx_connection_t *c) + } + + if (sslerr == SSL_ERROR_WANT_WRITE) { ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } + c->write->ready = 0; + c->read->handler = ngx_ssl_handshake_handler; + c->write->handler = ngx_ssl_handshake_handler; +@@ -1347,6 +1448,22 @@ ngx_ssl_handshake(ngx_connection_t *c) + return NGX_AGAIN; + } + ++ if (c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) ++ { ++ c->async->handler = ngx_ssl_handshake_async_handler; ++ c->read->saved_handler = c->read->handler; ++ c->read->handler = ngx_ssl_empty_handler; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "SSL ASYNC WANT recieved: \"%s\"", __func__); ++ ++ if (ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } ++ ++ return NGX_AGAIN; ++ } ++ + #if OPENSSL_VERSION_NUMBER >= 0x10002000L + if (sslerr == SSL_ERROR_WANT_X509_LOOKUP + # ifdef SSL_ERROR_PENDING_SESSION +@@ -1390,6 +1507,28 @@ ngx_ssl_handshake(ngx_connection_t *c) + } + + ++static void ++ngx_ssl_handshake_async_handler(ngx_event_t *aev) ++{ ++ ngx_connection_t *c; ++ ++ c = aev->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "SSL handshake async handler"); ++ ++ aev->ready = 0; ++ aev->handler = ngx_ssl_empty_handler; ++ c->read->handler = c->read->saved_handler; ++ ++ if (ngx_ssl_handshake(c) == NGX_AGAIN) { ++ return; ++ } ++ ++ c->ssl->handler(c); ++} ++ ++ + static void + ngx_ssl_handshake_handler(ngx_event_t *ev) + { +@@ -1409,6 +1548,11 @@ ngx_ssl_handshake_handler(ngx_event_t *ev) + return; + } + ++ /* ++ * empty the handler of async event to avoid ++ * going back to previous ssl handshake state ++ */ ++ c->async->handler = ngx_ssl_empty_handler; + c->ssl->handler(c); + } + +@@ -1580,6 +1724,10 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n) + + if (n > 0) { + ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } ++ + if (c->ssl->saved_write_handler) { + + c->write->handler = c->ssl->saved_write_handler; +@@ -1603,6 +1751,9 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n) + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr); + + if (sslerr == SSL_ERROR_WANT_READ) { ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } + c->read->ready = 0; + return NGX_AGAIN; + } +@@ -1612,6 +1763,10 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n) + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "peer started SSL renegotiation"); + ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } ++ + c->write->ready = 0; + + if (ngx_handle_write_event(c->write, 0) != NGX_OK) { +@@ -1630,6 +1785,21 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n) + return NGX_AGAIN; + } + ++ if (c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) { ++ c->async->handler = ngx_ssl_read_async_handler; ++ c->read->saved_handler = c->read->handler; ++ c->read->handler = ngx_ssl_empty_handler; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "SSL ASYNC WANT recieved: \"%s\"", __func__); ++ ++ if (ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } ++ ++ return NGX_AGAIN; ++ } ++ + c->ssl->no_wait_shutdown = 1; + c->ssl->no_send_shutdown = 1; + +@@ -1645,6 +1815,24 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n) + } + + ++static void ++ngx_ssl_read_async_handler(ngx_event_t *aev) ++{ ++ ngx_connection_t *c; ++ ++ c = aev->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "SSL read async handler"); ++ ++ aev->ready = 0; ++ aev->handler = ngx_ssl_empty_handler; ++ c->read->handler = c->read->saved_handler; ++ ++ c->read->handler(c->read); ++} ++ ++ + static void + ngx_ssl_write_handler(ngx_event_t *wev) + { +@@ -1835,6 +2023,10 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) + + if (n > 0) { + ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } ++ + if (c->ssl->saved_read_handler) { + + c->read->handler = c->ssl->saved_read_handler; +@@ -1860,6 +2052,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr); + + if (sslerr == SSL_ERROR_WANT_WRITE) { ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } + c->write->ready = 0; + return NGX_AGAIN; + } +@@ -1869,6 +2064,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "peer started SSL renegotiation"); + ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } + c->read->ready = 0; + + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { +@@ -1888,6 +2086,21 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) + return NGX_AGAIN; + } + ++ if(c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) { ++ c->async->handler = ngx_ssl_write_async_handler; ++ c->read->saved_handler = c->read->handler; ++ c->read->handler = ngx_ssl_empty_handler; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "SSL ASYNC WANT recieved: \"%s\"", __func__); ++ ++ if (ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } ++ ++ return NGX_AGAIN; ++ } ++ + c->ssl->no_wait_shutdown = 1; + c->ssl->no_send_shutdown = 1; + c->write->error = 1; +@@ -1898,6 +2111,24 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size) + } + + ++static void ++ngx_ssl_write_async_handler(ngx_event_t *aev) ++{ ++ ngx_connection_t *c; ++ ++ c = aev->data; ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "SSL write async handler"); ++ ++ aev->ready = 0; ++ aev->handler = ngx_ssl_empty_handler; ++ c->read->handler = c->read->saved_handler; ++ ++ c->write->handler(c->write); ++} ++ ++ + static void + ngx_ssl_read_handler(ngx_event_t *rev) + { +@@ -1926,6 +2157,10 @@ ngx_ssl_shutdown(ngx_connection_t *c) + int n, sslerr, mode; + ngx_err_t err; + ++ if(!c->ssl) { ++ return NGX_OK; ++ } ++ + if (SSL_in_init(c->ssl->connection)) { + /* + * OpenSSL 1.0.2f complains if SSL_shutdown() is called during +@@ -1933,6 +2168,30 @@ ngx_ssl_shutdown(ngx_connection_t *c) + * Avoid calling SSL_shutdown() if handshake wasn't completed. + */ + ++ if(c->asynch) { ++ /* Check if there is inflight request. ++ * Wait till async job becomes finished. ++ */ ++ if (SSL_want_async(c->ssl->connection) && !c->timedout) { ++ c->async->handler = ngx_ssl_shutdown_async_handler; ++ ngx_ssl_async_process_fds(c); ++ ngx_add_timer(c->async, 300); ++ return NGX_AGAIN; ++ } ++ ++ /* Ignore errors from ngx_ssl_async_process_fds as ++ we want to carry on and close the SSL connection ++ anyway. */ ++ ngx_ssl_async_process_fds(c); ++ if (ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++ ngx_del_conn(c, NGX_DISABLE_EVENT); ++ } ++ + SSL_free(c->ssl->connection); + c->ssl = NULL; + +@@ -1972,13 +2231,34 @@ ngx_ssl_shutdown(ngx_connection_t *c) + /* before 0.9.8m SSL_shutdown() returned 0 instead of -1 on errors */ + + if (n != 1 && ERR_peek_error()) { ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } + sslerr = SSL_get_error(c->ssl->connection, n); + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, + "SSL_get_error: %d", sslerr); + } ++ else if (c->asynch && n == -1) { ++ sslerr = SSL_get_error(c->ssl->connection, n); ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "SSL_get_error async: %d", sslerr); ++ } + + if (n == 1 || sslerr == 0 || sslerr == SSL_ERROR_ZERO_RETURN) { ++ if(c->asynch) { ++ /* Ignore errors from ngx_ssl_async_process_fds as ++ we want to carry on and close the SSL connection ++ anyway. */ ++ ngx_ssl_async_process_fds(c); ++ if (ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++ ngx_del_conn(c, NGX_DISABLE_EVENT); ++ } + SSL_free(c->ssl->connection); + c->ssl = NULL; + +@@ -1986,9 +2266,16 @@ ngx_ssl_shutdown(ngx_connection_t *c) + } + + if (sslerr == SSL_ERROR_WANT_READ || sslerr == SSL_ERROR_WANT_WRITE) { ++ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) { ++ return NGX_ERROR; ++ } + c->read->handler = ngx_ssl_shutdown_handler; + c->write->handler = ngx_ssl_shutdown_handler; + ++ //Work around: Readd write event on shutdown; ++ c->write->ready = 0; ++ c->write->active = 0; ++ + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { + return NGX_ERROR; + } +@@ -2001,9 +2288,41 @@ ngx_ssl_shutdown(ngx_connection_t *c) + ngx_add_timer(c->read, 30000); + } + ++ if (sslerr == SSL_ERROR_WANT_WRITE) { ++ ngx_add_timer(c->write, 10000); ++ } ++ + return NGX_AGAIN; + } + ++ if(c->asynch) { ++ if (sslerr == SSL_ERROR_WANT_ASYNC) { ++ c->async->handler = ngx_ssl_shutdown_async_handler; ++ c->read->saved_handler = ngx_ssl_shutdown_handler; ++ c->read->handler = ngx_ssl_empty_handler; ++ c->write->handler = ngx_ssl_shutdown_handler; ++ ++ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, ++ "SSL ASYNC WANT recieved: \"%s\"", __func__); ++ ++ /* Ignore errors from ngx_ssl_async_process_fds as ++ we want to carry on anyway */ ++ ngx_ssl_async_process_fds(c); ++ return NGX_AGAIN; ++ } ++ ++ /* Ignore errors from ngx_ssl_async_process_fds as ++ we want to carry on and close the SSL connection ++ anyway. */ ++ ngx_ssl_async_process_fds(c); ++ if (ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++ ngx_del_conn(c, NGX_DISABLE_EVENT); ++ } + err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0; + + ngx_ssl_connection_error(c, sslerr, err, "SSL_shutdown() failed"); +@@ -2015,6 +2334,34 @@ ngx_ssl_shutdown(ngx_connection_t *c) + } + + ++static void ++ngx_ssl_shutdown_async_handler(ngx_event_t *aev) ++{ ++ ngx_connection_t *c; ++ ngx_connection_handler_pt handler; ++ ++ c = aev->data; ++ handler = c->ssl->handler; ++ ++ if (aev->timedout) { ++ c->timedout = 1; ++ } ++ ++ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, aev->log, 0, ++ "SSL shutdown async handler"); ++ ++ aev->ready = 0; ++ aev->handler = ngx_ssl_empty_handler; ++ c->read->handler = c->read->saved_handler; ++ ++ if (ngx_ssl_shutdown(c) == NGX_AGAIN) { ++ return; ++ } ++ ++ handler(c); ++} ++ ++ + static void + ngx_ssl_shutdown_handler(ngx_event_t *ev) + { +@@ -2034,6 +2381,11 @@ ngx_ssl_shutdown_handler(ngx_event_t *ev) + return; + } + ++ /* ++ * empty the handler of async event to avoid ++ * going back to previous ssl shutdown state ++ */ ++ c->async->handler = ngx_ssl_empty_handler; + handler(c); + } + +@@ -4227,6 +4579,10 @@ ngx_openssl_engine(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) + + oscf->engine = 1; + ++ if(cf->no_ssl_init) { ++ return NGX_CONF_OK; ++ } ++ + value = cf->args->elts; + + engine = ENGINE_by_id((char *) value[1].data); +diff --git a/src/src/event/ngx_event_openssl.h b/src/src/event/ngx_event_openssl.h +index b9a3a96..e3aecd3 100644 +--- a/src/src/event/ngx_event_openssl.h ++++ b/src/src/event/ngx_event_openssl.h +@@ -63,6 +63,7 @@ struct ngx_ssl_s { + SSL_CTX *ctx; + ngx_log_t *log; + size_t buffer_size; ++ ngx_flag_t asynch; + }; + + +@@ -193,6 +194,7 @@ ngx_int_t ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session); + + ngx_int_t ngx_ssl_check_host(ngx_connection_t *c, ngx_str_t *name); + ++#define ngx_ssl_waiting_for_async(c) SSL_waiting_for_async(c->ssl->connection) + + ngx_int_t ngx_ssl_get_protocol(ngx_connection_t *c, ngx_pool_t *pool, + ngx_str_t *s); +@@ -247,7 +249,7 @@ ngx_int_t ngx_ssl_shutdown(ngx_connection_t *c); + void ngx_cdecl ngx_ssl_error(ngx_uint_t level, ngx_log_t *log, ngx_err_t err, + char *fmt, ...); + void ngx_ssl_cleanup_ctx(void *data); +- ++ngx_int_t ngx_ssl_async_process_fds(ngx_connection_t *c) ; + + extern int ngx_ssl_connection_index; + extern int ngx_ssl_server_conf_index; +diff --git a/src/src/http/modules/ngx_http_memcached_module.c b/src/src/http/modules/ngx_http_memcached_module.c +index 69f28fa..8e466b6 100644 +--- a/src/src/http/modules/ngx_http_memcached_module.c ++++ b/src/src/http/modules/ngx_http_memcached_module.c +@@ -533,7 +533,7 @@ ngx_http_memcached_filter(void *data, ssize_t bytes) + + last += (size_t) (u->length - NGX_HTTP_MEMCACHED_END); + +- if (ngx_strncmp(last, ngx_http_memcached_end, b->last - last) != 0) { ++ if (ngx_strncmp(last, ngx_http_memcached_end,(unsigned)(b->last - last)) != 0) { + ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0, + "memcached sent invalid trailer"); + +diff --git a/src/src/http/modules/ngx_http_ssl_module.c b/src/src/http/modules/ngx_http_ssl_module.c +index 7d62176..47fdfae 100644 +--- a/src/src/http/modules/ngx_http_ssl_module.c ++++ b/src/src/http/modules/ngx_http_ssl_module.c +@@ -43,6 +43,8 @@ static char *ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, + + static char *ngx_http_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); ++static char *ngx_http_ssl_enable_asynch(ngx_conf_t *cf, ngx_command_t *cmd, ++ void *conf); + static char *ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + static char *ngx_http_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd, +@@ -80,6 +82,13 @@ static ngx_command_t ngx_http_ssl_commands[] = { + offsetof(ngx_http_ssl_srv_conf_t, enable), + NULL }, + ++ { ngx_string("ssl_asynch"), ++ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG, ++ ngx_http_ssl_enable_asynch, ++ NGX_HTTP_SRV_CONF_OFFSET, ++ offsetof(ngx_http_ssl_srv_conf_t, enable_asynch), ++ NULL }, ++ + { ngx_string("ssl_certificate"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_array_slot, +@@ -546,6 +555,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf) + */ + + sscf->enable = NGX_CONF_UNSET; ++ sscf->enable_asynch = NGX_CONF_UNSET; + sscf->prefer_server_ciphers = NGX_CONF_UNSET; + sscf->buffer_size = NGX_CONF_UNSET_SIZE; + sscf->verify = NGX_CONF_UNSET_UINT; +@@ -583,6 +593,17 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) + } + } + ++ if (conf->enable_asynch == NGX_CONF_UNSET) { ++ if (prev->enable_asynch == NGX_CONF_UNSET) { ++ conf->enable_asynch = 0; ++ ++ } else { ++ conf->enable_asynch = prev->enable_asynch; ++ conf->file = prev->file; ++ conf->line = prev->line; ++ } ++ } ++ + ngx_conf_merge_value(conf->session_timeout, + prev->session_timeout, 300); + +@@ -655,6 +676,7 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child) + return NGX_CONF_ERROR; + } + ++ conf->ssl.asynch = conf->enable_asynch; + } else { + + if (conf->certificates == NULL) { +@@ -827,6 +849,37 @@ ngx_http_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) + return NGX_CONF_OK; + } + ++static char * ++ngx_http_ssl_enable_asynch(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) ++{ ++ ngx_http_ssl_srv_conf_t *sscf = conf; ++ ++ char *rv; ++ ++ ngx_flag_t *pssl, *pssl_asynch; ++ ++ rv = ngx_conf_set_flag_slot(cf, cmd, conf); ++ ++ if (rv != NGX_CONF_OK) { ++ return rv; ++ } ++ ++ /* If ssl_asynch on is configured, then ssl on is configured by default ++ * This will align 'ssl_asynch on;' and 'listen port ssl' diretives ++ * */ ++ pssl = (ngx_flag_t *) ((char *)conf + offsetof(ngx_http_ssl_srv_conf_t, enable)); ++ pssl_asynch = (ngx_flag_t *) ((char *)conf + cmd->offset); ++ ++ if(*pssl_asynch && *pssl != 1) { ++ *pssl = *pssl_asynch; ++ } ++ ++ sscf->file = cf->conf_file->file.name.data; ++ sscf->line = cf->conf_file->line; ++ ++ return NGX_CONF_OK; ++} ++ + + static char * + ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +diff --git a/src/src/http/modules/ngx_http_ssl_module.h b/src/src/http/modules/ngx_http_ssl_module.h +index 57f5941..b97417a 100644 +--- a/src/src/http/modules/ngx_http_ssl_module.h ++++ b/src/src/http/modules/ngx_http_ssl_module.h +@@ -17,6 +17,8 @@ + typedef struct { + ngx_flag_t enable; + ++ ngx_flag_t enable_asynch; ++ + ngx_ssl_t ssl; + + ngx_flag_t prefer_server_ciphers; +diff --git a/src/src/http/ngx_http_request.c b/src/src/http/ngx_http_request.c +index 0de8ace..653990c 100644 +--- a/src/src/http/ngx_http_request.c ++++ b/src/src/http/ngx_http_request.c +@@ -451,9 +451,21 @@ ngx_http_wait_request_handler(ngx_event_t *rev) + * We are trying to not hold c->buffer's memory for an idle connection. + */ + +- if (ngx_pfree(c->pool, b->start) == NGX_OK) { +- b->start = NULL; ++ /* For the Async implementation we need the same buffer to be used ++ * again on any async calls that have not completed. ++ * As such we need to turn off this optimisation if an async request ++ * is still in progress. ++ */ ++ ++#if (NGX_HTTP_SSL) ++ if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) { ++#endif ++ if (ngx_pfree(c->pool, b->start) == NGX_OK) { ++ b->start = NULL; ++ } ++#if (NGX_HTTP_SSL) + } ++#endif + + return; + } +@@ -1414,12 +1426,21 @@ ngx_http_read_request_header(ngx_http_request_t *r) + return n; + } + +- if (rev->ready) { ++#if (NGX_HTTP_SSL) ++ if(c->asynch) + n = c->recv(c, r->header_in->last, +- r->header_in->end - r->header_in->last); +- } else { +- n = NGX_AGAIN; ++ r->header_in->end - r->header_in->last); ++ else { ++#endif ++ if (rev->ready) { ++ n = c->recv(c, r->header_in->last, ++ r->header_in->end - r->header_in->last); ++ } else { ++ n = NGX_AGAIN; ++ } ++#if (NGX_HTTP_SSL) + } ++#endif + + if (n == NGX_AGAIN) { + if (!rev->timer_set) { +@@ -1482,8 +1503,8 @@ ngx_http_alloc_large_header_buffer(ngx_http_request_t *r, + cscf = ngx_http_get_module_srv_conf(r, ngx_http_core_module); + + if (r->state != 0 +- && (size_t) (r->header_in->pos - old) +- >= cscf->large_client_header_buffers.size) ++ && (size_t) (unsigned) (r->header_in->pos - old) \ ++ >= cscf->large_client_header_buffers.size) + { + return NGX_DECLINED; + } +@@ -3014,103 +3035,122 @@ ngx_http_set_keepalive(ngx_http_request_t *r) + * c->pool and are freed too. + */ + +- b = c->buffer; +- +- if (ngx_pfree(c->pool, b->start) == NGX_OK) { ++ /* For the Async implementation we need the same buffer to be used ++ * again on any async calls that have not completed. ++ * As such we need to turn off this optimisation if an async request ++ * is still in progress. ++ */ + +- /* +- * the special note for ngx_http_keepalive_handler() that +- * c->buffer's memory was freed +- */ ++#if (NGX_HTTP_SSL) ++ if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) { ++#endif ++ b = c->buffer; + +- b->pos = NULL; ++ if (ngx_pfree(c->pool, b->start) == NGX_OK) { + +- } else { +- b->pos = b->start; +- b->last = b->start; +- } ++ /* ++ * the special note for ngx_http_keepalive_handler() that ++ * c->buffer's memory was freed ++ */ + +- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc free: %p", +- hc->free); ++ b->pos = NULL; + +- if (hc->free) { +- for (cl = hc->free; cl; /* void */) { +- ln = cl; +- cl = cl->next; +- ngx_pfree(c->pool, ln->buf->start); +- ngx_free_chain(c->pool, ln); ++ } else { ++ b->pos = b->start; ++ b->last = b->start; + } + +- hc->free = NULL; +- } ++ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc free: %p", ++ hc->free); + +- ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc busy: %p %i", +- hc->busy, hc->nbusy); ++ if (hc->free) { ++ for (cl = hc->free; cl; /* void */) { ++ ln = cl; ++ cl = cl->next; ++ ngx_pfree(c->pool, ln->buf->start); ++ ngx_free_chain(c->pool, ln); ++ } + +- if (hc->busy) { +- for (cl = hc->busy; cl; /* void */) { +- ln = cl; +- cl = cl->next; +- ngx_pfree(c->pool, ln->buf->start); +- ngx_free_chain(c->pool, ln); ++ hc->free = NULL; + } + +- hc->busy = NULL; +- hc->nbusy = 0; +- } ++ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc busy: %p %i", ++ hc->busy, hc->nbusy); ++ ++ if (hc->busy) { ++ for (cl = hc->busy; cl; /* void */) { ++ ln = cl; ++ cl = cl->next; ++ ngx_pfree(c->pool, ln->buf->start); ++ ngx_free_chain(c->pool, ln); ++ } ++ ++ hc->busy = NULL; ++ hc->nbusy = 0; ++ } + + #if (NGX_HTTP_SSL) +- if (c->ssl) { +- ngx_ssl_free_buffer(c); +- } ++ if (c->ssl) { ++ ngx_ssl_free_buffer(c); ++ } + #endif + +- rev->handler = ngx_http_keepalive_handler; ++ rev->handler = ngx_http_keepalive_handler; + +- if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) { +- if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) { +- ngx_http_close_connection(c); +- return; ++ if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) { ++#if (NGX_HTTP_SSL) ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif ++ if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) { ++ ngx_http_close_connection(c); ++ return; ++ } + } +- } + +- c->log->action = "keepalive"; ++ c->log->action = "keepalive"; + +- if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) { +- if (ngx_tcp_push(c->fd) == -1) { +- ngx_connection_error(c, ngx_socket_errno, ngx_tcp_push_n " failed"); +- ngx_http_close_connection(c); +- return; +- } ++ if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) { ++ if (ngx_tcp_push(c->fd) == -1) { ++ ngx_connection_error(c, ngx_socket_errno, ngx_tcp_push_n " failed"); ++ ngx_http_close_connection(c); ++ return; ++ } + +- c->tcp_nopush = NGX_TCP_NOPUSH_UNSET; +- tcp_nodelay = ngx_tcp_nodelay_and_tcp_nopush ? 1 : 0; ++ c->tcp_nopush = NGX_TCP_NOPUSH_UNSET; ++ tcp_nodelay = ngx_tcp_nodelay_and_tcp_nopush ? 1 : 0; + +- } else { +- tcp_nodelay = 1; +- } ++ } else { ++ tcp_nodelay = 1; ++ } + +- if (tcp_nodelay && clcf->tcp_nodelay && ngx_tcp_nodelay(c) != NGX_OK) { +- ngx_http_close_connection(c); +- return; +- } ++ if (tcp_nodelay && clcf->tcp_nodelay && ngx_tcp_nodelay(c) != NGX_OK) { ++ ngx_http_close_connection(c); ++ return; ++ } + + #if 0 +- /* if ngx_http_request_t was freed then we need some other place */ +- r->http_state = NGX_HTTP_KEEPALIVE_STATE; ++ /* if ngx_http_request_t was freed then we need some other place */ ++ r->http_state = NGX_HTTP_KEEPALIVE_STATE; + #endif + +- c->idle = 1; +- ngx_reusable_connection(c, 1); ++ c->idle = 1; ++ ngx_reusable_connection(c, 1); + +- ngx_add_timer(rev, clcf->keepalive_timeout); ++ ngx_add_timer(rev, clcf->keepalive_timeout); + +- if (rev->ready) { +- ngx_post_event(rev, &ngx_posted_events); ++ if (rev->ready) { ++ ngx_post_event(rev, &ngx_posted_events); ++ } ++#if (NGX_HTTP_SSL) + } ++#endif + } + +- + static void + ngx_http_keepalive_handler(ngx_event_t *rev) + { +@@ -3192,14 +3232,25 @@ ngx_http_keepalive_handler(ngx_event_t *rev) + * c->buffer's memory for a keepalive connection. + */ + +- if (ngx_pfree(c->pool, b->start) == NGX_OK) { ++ /* For the Asynch implementation we need the same buffer to be used ++ * on subsequent read requests. As such we need to turn off this optimisation that ++ * frees the buffer between invocations as may end up with a buffer that is at a ++ * different address */ + +- /* +- * the special note that c->buffer's memory was freed +- */ ++#if (NGX_HTTP_SSL) ++ if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) { ++#endif ++ if (ngx_pfree(c->pool, b->start) == NGX_OK) { + +- b->pos = NULL; ++ /* ++ * the special note that c->buffer's memory was freed ++ */ ++ ++ b->pos = NULL; ++ } ++#if (NGX_HTTP_SSL) + } ++#endif + + return; + } +@@ -3268,6 +3319,14 @@ ngx_http_set_lingering_close(ngx_http_request_t *r) + wev->handler = ngx_http_empty_handler; + + if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) { ++#if (NGX_HTTP_SSL) ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif + if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) { + ngx_http_close_request(r, 0); + return; +diff --git a/src/src/http/ngx_http_upstream.c b/src/src/http/ngx_http_upstream.c +index 2d14067..3746224 100644 +--- a/src/src/http/ngx_http_upstream.c ++++ b/src/src/http/ngx_http_upstream.c +@@ -938,7 +938,8 @@ ngx_http_upstream_cache(ngx_http_request_t *r, ngx_http_upstream_t *u) + + case NGX_DECLINED: + +- if ((size_t) (u->buffer.end - u->buffer.start) < u->conf->buffer_size) { ++ if ((size_t) (unsigned) (u->buffer.end - u->buffer.start) \ ++ < u->conf->buffer_size) { + u->buffer.start = NULL; + + } else { +@@ -1314,7 +1315,14 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, + if ((ngx_event_flags & NGX_USE_LEVEL_EVENT) && ev->active) { + + event = ev->write ? NGX_WRITE_EVENT : NGX_READ_EVENT; +- ++#if (NGX_HTTP_SSL) ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif + if (ngx_del_event(ev, event, 0) != NGX_OK) { + ngx_http_upstream_finalize_request(r, u, + NGX_HTTP_INTERNAL_SERVER_ERROR); +@@ -1441,7 +1449,14 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, + if ((ngx_event_flags & NGX_USE_LEVEL_EVENT) && ev->active) { + + event = ev->write ? NGX_WRITE_EVENT : NGX_READ_EVENT; +- ++#if (NGX_HTTP_SSL) ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif + if (ngx_del_event(ev, event, 0) != NGX_OK) { + ngx_http_upstream_finalize_request(r, u, + NGX_HTTP_INTERNAL_SERVER_ERROR); +@@ -4353,7 +4368,8 @@ ngx_http_upstream_finalize_request(ngx_http_request_t *r, + } + } + +- ngx_http_file_cache_free(r->cache, u->pipe->temp_file); ++ if (u->pipe) ++ ngx_http_file_cache_free(r->cache, u->pipe->temp_file); + } + + #endif +diff --git a/src/src/os/unix/ngx_daemon.c b/src/src/os/unix/ngx_daemon.c +index ab67211..1834922 100644 +--- a/src/src/os/unix/ngx_daemon.c ++++ b/src/src/os/unix/ngx_daemon.c +@@ -10,26 +10,27 @@ + + + ngx_int_t +-ngx_daemon(ngx_log_t *log) ++ngx_daemon(ngx_cycle_t *cycle) + { + int fd; + + switch (fork()) { + case -1: +- ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "fork() failed"); ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "fork() failed"); + return NGX_ERROR; + + case 0: + break; + + default: ++ ngx_destroy_pool(cycle->pool); + exit(0); + } + + ngx_pid = ngx_getpid(); + + if (setsid() == -1) { +- ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "setsid() failed"); ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "setsid() failed"); + return NGX_ERROR; + } + +@@ -37,31 +38,31 @@ ngx_daemon(ngx_log_t *log) + + fd = open("/dev/null", O_RDWR); + if (fd == -1) { +- ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "open(\"/dev/null\") failed"); + return NGX_ERROR; + } + + if (dup2(fd, STDIN_FILENO) == -1) { +- ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "dup2(STDIN) failed"); ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "dup2(STDIN) failed"); + return NGX_ERROR; + } + + if (dup2(fd, STDOUT_FILENO) == -1) { +- ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "dup2(STDOUT) failed"); ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "dup2(STDOUT) failed"); + return NGX_ERROR; + } + + #if 0 + if (dup2(fd, STDERR_FILENO) == -1) { +- ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "dup2(STDERR) failed"); ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "dup2(STDERR) failed"); + return NGX_ERROR; + } + #endif + + if (fd > STDERR_FILENO) { + if (close(fd) == -1) { +- ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "close() failed"); ++ ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "close() failed"); + return NGX_ERROR; + } + } +diff --git a/src/src/os/unix/ngx_os.h b/src/src/os/unix/ngx_os.h +index 3b32819..8249508 100644 +--- a/src/src/os/unix/ngx_os.h ++++ b/src/src/os/unix/ngx_os.h +@@ -39,7 +39,7 @@ ngx_int_t ngx_os_init(ngx_log_t *log); + void ngx_os_status(ngx_log_t *log); + ngx_int_t ngx_os_specific_init(ngx_log_t *log); + void ngx_os_specific_status(ngx_log_t *log); +-ngx_int_t ngx_daemon(ngx_log_t *log); ++ngx_int_t ngx_daemon(ngx_cycle_t *log); + ngx_int_t ngx_os_signal_process(ngx_cycle_t *cycle, char *sig, ngx_pid_t pid); + + +diff --git a/src/src/os/unix/ngx_process_cycle.c b/src/src/os/unix/ngx_process_cycle.c +index 8a4ecbf..a01c47f 100644 +--- a/src/src/os/unix/ngx_process_cycle.c ++++ b/src/src/os/unix/ngx_process_cycle.c +@@ -9,7 +9,7 @@ + #include + #include + #include +- ++#include + + static void ngx_start_worker_processes(ngx_cycle_t *cycle, ngx_int_t n, + ngx_int_t type); +@@ -684,6 +684,13 @@ ngx_reap_children(ngx_cycle_t *cycle) + && !ngx_terminate + && !ngx_quit) + { ++#if (NGX_SSL) ++ /* Delay added to give Quickassist Driver time to cleanup ++ * if worker exit with non-zero code. */ ++ if(ngx_processes[i].status != 0) { ++ usleep(2000000); ++ } ++#endif + if (ngx_spawn_process(cycle, ngx_processes[i].proc, + ngx_processes[i].data, + ngx_processes[i].name, i) +@@ -1083,6 +1090,8 @@ ngx_channel_handler(ngx_event_t *ev) + return; + } + ++ ngx_memzero(&ch, sizeof(ngx_channel_t)); ++ + c = ev->data; + + ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "channel handler"); +@@ -1096,6 +1105,14 @@ ngx_channel_handler(ngx_event_t *ev) + if (n == NGX_ERROR) { + + if (ngx_event_flags & NGX_USE_EPOLL_EVENT) { ++#if (NGX_SSL) ++ if (c->asynch && ngx_del_async_conn) { ++ if (c->num_async_fds) { ++ ngx_del_async_conn(c, NGX_DISABLE_EVENT); ++ c->num_async_fds--; ++ } ++ } ++#endif + ngx_del_conn(c, 0); + } +