From c46009d6f3610411033c71e8a4da496a22cb55d3 Mon Sep 17 00:00:00 2001 From: Thibault Charbonnier Date: Fri, 3 Mar 2017 15:43:12 -0800 Subject: [PATCH] feature: applied the resolve hosts file patch to the nginx core This patch allows the Nginx resolver to parse the system hosts file (/etc/hosts) when specifying an hosts file option to the resolver directive. --- .gitignore | 1 + patches/nginx-1.11.2-resolve-hosts-file.patch | 417 ++++++++++++++++++ t/fixtures/hosts | 13 + t/patches/01-resolve-hosts-file.t | 218 +++++++++ util/mirror-tarballs | 4 + 5 files changed, 653 insertions(+) create mode 100644 patches/nginx-1.11.2-resolve-hosts-file.patch create mode 100644 t/fixtures/hosts create mode 100644 t/patches/01-resolve-hosts-file.t diff --git a/.gitignore b/.gitignore index db4d599..837343c 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ openresty-* work/ reindex t/*.t_ +t/servroot upload upload-win32 html_out/ diff --git a/patches/nginx-1.11.2-resolve-hosts-file.patch b/patches/nginx-1.11.2-resolve-hosts-file.patch new file mode 100644 index 0000000..aec4b7e --- /dev/null +++ b/patches/nginx-1.11.2-resolve-hosts-file.patch @@ -0,0 +1,417 @@ +# HG changeset patch +# User Thibault Charbonnier +# Date 1488252201 28800 +# Mon Feb 27 19:23:21 2017 -0800 +# Node ID 558041ef1d70689ccc4c12f2487c3f75e6bbbccc +# Parent a2f5e25d6a283546f76435b9fc3e7e814b092bae +Resolver: parse hosts file entries + +The resolver directive can now take an optional 'hostsfile=' option, +such as: + + resolver 8.8.4.4 hostsfile=/etc/hosts; + +Hosts parsed from the hosts file are considered valid forever. The behavior +tries to be conservative, and only parses the hosts file when the option is +provided, to enforce backwards compatibility. + +Additionally, this patch makes the resolver able to handle a host file in the +absence of a nameserver, like so: + + resolver hostsfile=/etc/hosts; + +The 'hostsfile' option also honors the 'ipv6' flag of the 'resolver' +directive, as in: + + resolver 8.8.4.4 hostsfile=/etc/hosts ipv6=off; + +diff -r a2f5e25d6a28 -r 558041ef1d70 src/core/ngx_resolver.c +--- a/src/core/ngx_resolver.c Thu Aug 10 22:21:23 2017 +0300 ++++ b/src/core/ngx_resolver.c Mon Feb 27 19:23:21 2017 -0800 +@@ -9,11 +9,12 @@ + #include + #include + +- +-#define NGX_RESOLVER_UDP_SIZE 4096 +- +-#define NGX_RESOLVER_TCP_RSIZE (2 + 65535) +-#define NGX_RESOLVER_TCP_WSIZE 8192 ++#define NGX_RESOLVER_HOSTSFILE_BUFFER_SIZE 4096 ++ ++#define NGX_RESOLVER_UDP_SIZE 4096 ++ ++#define NGX_RESOLVER_TCP_RSIZE (2 + 65535) ++#define NGX_RESOLVER_TCP_WSIZE 8192 + + + typedef struct { +@@ -122,6 +123,8 @@ + ngx_resolver_node_t *rn); + static void ngx_resolver_srv_names_handler(ngx_resolver_ctx_t *ctx); + static ngx_int_t ngx_resolver_cmp_srvs(const void *one, const void *two); ++static ngx_int_t ngx_resolver_parse_hosts_file(ngx_conf_t *cf, ++ ngx_resolver_t *r); + + #if (NGX_HAVE_INET6) + static void ngx_resolver_rbtree_insert_addr6_value(ngx_rbtree_node_t *temp, +@@ -130,7 +133,6 @@ + struct in6_addr *addr, uint32_t hash); + #endif + +- + ngx_resolver_t * + ngx_resolver_create(ngx_conf_t *cf, ngx_str_t *names, ngx_uint_t n) + { +@@ -246,6 +248,20 @@ + } + #endif + ++ if (ngx_strncmp(names[i].data, "hostsfile=", 10) == 0) { ++ r->hosts_file.len = names[i].len - 10; ++ ++ if (r->hosts_file.len == 0) { ++ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, ++ "invalid parameter: %V", &names[i]); ++ return NULL; ++ } ++ ++ r->hosts_file.data = names[i].data + 10; ++ ++ continue; ++ } ++ + ngx_memzero(&u, sizeof(ngx_url_t)); + + u.url = names[i]; +@@ -276,6 +292,13 @@ + } + } + ++ if (r->hosts_file.len > 0 ++ && ngx_resolver_parse_hosts_file(cf, r) ++ != NGX_OK) ++ { ++ return NULL; ++ } ++ + return r; + } + +@@ -396,7 +419,7 @@ + } + } + +- if (r->connections.nelts == 0) { ++ if (r->connections.nelts == 0 && !r->hosts_file.len) { + return NGX_NO_RESOLVER; + } + +@@ -807,6 +830,13 @@ + #endif + + ngx_rbtree_insert(tree, &rn->node); ++ ++ if (r->connections.nelts == 0) { ++ ctx->quick = 1; ++ ctx->state = NGX_RESOLVE_NXDOMAIN; ++ ctx->handler(ctx); ++ return NGX_OK; ++ } + } + + if (ctx->service.len) { +@@ -4652,3 +4682,281 @@ + + return p1 - p2; + } ++ ++ ++static ngx_int_t ++ngx_resolver_parse_hosts_file(ngx_conf_t *cf, ngx_resolver_t *r) ++{ ++ off_t file_size; ++ u_char ch; ++ u_char *start; ++ size_t len; ++ ssize_t n, size; ++ ngx_int_t rc; ++ ngx_buf_t b; ++ ngx_fd_t fd; ++ ngx_str_t filename, s; ++ ngx_file_t file; ++ in_addr_t addr; ++ ngx_resolver_node_t *rn; ++ enum { ++ scan_line = 0, ++ scan_skipline, ++ scan_addr, ++ scan_hosts, ++ scan_name ++ } state; ++ ++ b.start = NULL; ++ s.data = NULL; ++ s.len = 0; ++ rn = NULL; ++ rc = NGX_OK; ++ ++ filename = r->hosts_file; ++ ++ fd = ngx_open_file(filename.data, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); ++ if (fd == NGX_INVALID_FILE) { ++ ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, ++ ngx_open_file_n " \"%s\" failed", filename.data); ++ return NGX_ERROR; ++ } ++ ++ ngx_memzero(&file, sizeof(ngx_file_t)); ++ ++ if (ngx_fd_info(fd, &file.info) == NGX_FILE_ERROR) { ++ ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, ++ ngx_fd_info_n " \"%s\" failed", filename.data); ++ goto fail; ++ } ++ ++ file.fd = fd; ++ file.log = cf->log; ++ file.name.len = filename.len; ++ file.name.data = filename.data; ++ file.offset = 0; ++ ++ b.start = ngx_alloc(NGX_RESOLVER_HOSTSFILE_BUFFER_SIZE, cf->log); ++ if (b.start == NULL) { ++ goto fail; ++ } ++ ++ b.pos = b.start; ++ b.last = b.start; ++ b.end = b.last + NGX_RESOLVER_HOSTSFILE_BUFFER_SIZE; ++ b.temporary = 1; ++ ++ start = b.pos; ++ state = scan_line; ++ file_size = ngx_file_size(&file.info); ++ ++ for ( ;; ) { ++ ++ if (b.pos >= b.last) { ++ len = b.pos - start; ++ ++ if (len) { ++ ngx_memmove(b.start, start, len); ++ } ++ ++ size = (ssize_t) (file_size - file.offset); ++ ++ if (size > b.end - (b.start + len)) { ++ size = b.end - (b.start + len); ++ ++ } else if (size == 0) { ++ goto done; ++ } ++ ++ n = ngx_read_file(&file, b.start + len, size, file.offset); ++ if (n == NGX_ERROR) { ++ ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, ++ ngx_read_file_n, " \"%s\" failed", ++ filename.data); ++ goto fail; ++ } ++ ++ if (n != size) { ++ ngx_log_error(NGX_LOG_ERR, cf->log, ngx_errno, ++ ngx_read_file_n, " returned only %z bytes " ++ "instead of %z", n, size); ++ goto fail; ++ } ++ ++ b.pos = b.start + len; ++ b.last = b.pos + n; ++ start = b.start; ++ } ++ ++ ch = *b.pos; ++ ++ switch (state) { ++ ++ case scan_line: ++ if (ch == ' ') { ++ break; ++ } ++ ++ if (ch == '#') { ++ state = scan_skipline; ++ break; ++ } ++ ++ if (ch != LF && ch != CR) { ++ start = b.pos; ++ state = scan_addr; ++ } ++ ++ break; ++ ++ case scan_skipline: ++ if (ch == LF || ch == CR) { ++ state = scan_line; ++ } ++ ++ break; ++ ++ case scan_addr: ++ if (ch == LF || ch == CR) { ++ state = scan_line; ++ break; ++ } ++ ++ if (ch == ' ' || ch == '\t') { ++ if (s.data) { ++ ngx_free(s.data); ++ } ++ ++ s.len = b.pos - start; ++ ++ s.data = ngx_alloc(s.len, cf->log); ++ if (s.data == NULL) { ++ goto fail; ++ } ++ ++ ngx_memcpy(s.data, start, s.len); ++ ++ state = scan_hosts; ++ } ++ ++ break; ++ ++ case scan_hosts: ++ if (ch == LF || ch == CR) { ++ state = scan_line; ++ break; ++ } ++ ++ if (ch == ' ' || ch == '\t') { ++ break; ++ } ++ ++ start = b.pos; ++ state = scan_name; ++ break; ++ ++ case scan_name: ++ if (ch == ' ' || ch == '\t' || ch == LF || ch == CR) { ++ rn = ngx_calloc(sizeof(ngx_resolver_node_t), cf->log); ++ if (rn == NULL) { ++ goto fail; ++ } ++ ++ rn->nlen = b.pos - start; ++ ++ rn->name = ngx_alloc(rn->nlen, cf->log); ++ if (rn->name == NULL) { ++ goto fail; ++ } ++ ++ ngx_memcpy(rn->name, start, rn->nlen); ++ ++ rn->ttl = NGX_MAX_UINT32_VALUE; ++ rn->valid = NGX_MAX_UINT32_VALUE; ++ rn->expire = NGX_MAX_UINT32_VALUE; ++ rn->node.key = ngx_crc32_short(rn->name, rn->nlen); ++ ++ if (ngx_strlchr(s.data, ++ s.data + s.len, ':') != NULL) ++ { ++ ++#if (NGX_HAVE_INET6) ++ if (!r->ipv6 ++ || ngx_inet6_addr(s.data, s.len, ++ rn->u6.addr6.s6_addr) != NGX_OK) ++ { ++#endif ++ ++ ngx_resolver_free_node(r, rn); ++ state = scan_skipline; ++ break; ++ ++#if (NGX_HAVE_INET6) ++ } ++ ++ rn->naddrs6 = 1; ++#endif ++ ++ } else { ++ addr = ngx_inet_addr(s.data, s.len); ++ if (addr == INADDR_NONE) { ++ ngx_resolver_free_node(r, rn); ++ state = scan_skipline; ++ break; ++ } ++ ++ rn->naddrs = 1; ++ rn->u.addr = addr; ++ } ++ ++ ngx_log_error(NGX_LOG_NOTICE, cf->log, 0, ++ "host \"%*s\" will resolve to \"%V\" " ++ "(hosts file at \"%V\")", ++ rn->nlen, rn->name, &s, &filename); ++ ++ ngx_rbtree_insert(&r->name_rbtree, &rn->node); ++ ++ ngx_queue_insert_head(&r->name_expire_queue, &rn->queue); ++ ++ if (ch == LF || ch == CR) { ++ state = scan_line; ++ break; ++ } ++ ++ state = scan_hosts; ++ } ++ ++ break; ++ } ++ ++ b.pos++; ++ } ++ ++fail: ++ ++ rc = NGX_ERROR; ++ ++done: ++ ++ if (s.data) { ++ ngx_free(s.data); ++ } ++ ++ if (b.start) { ++ ngx_free(b.start); ++ } ++ ++ if (ngx_close_file(fd) == NGX_FILE_ERROR) { ++ ngx_log_error(NGX_LOG_ALERT, cf->log, ngx_errno, ++ ngx_close_file_n, " \"%s\" failed", ++ filename.data); ++ rc = NGX_ERROR; ++ } ++ ++ if (rc == NGX_ERROR) { ++ return NGX_ERROR; ++ } ++ ++ return NGX_OK; ++} ++ +diff -r a2f5e25d6a28 -r 558041ef1d70 src/core/ngx_resolver.h +--- a/src/core/ngx_resolver.h Thu Aug 10 22:21:23 2017 +0300 ++++ b/src/core/ngx_resolver.h Mon Feb 27 19:23:21 2017 -0800 +@@ -146,6 +146,8 @@ + + + struct ngx_resolver_s { ++ ngx_str_t hosts_file; ++ + /* has to be pointer because of "incomplete type" */ + ngx_event_t *event; + void *dummy; diff --git a/t/fixtures/hosts b/t/fixtures/hosts new file mode 100644 index 0000000..a9749a6 --- /dev/null +++ b/t/fixtures/hosts @@ -0,0 +1,13 @@ +## +# Host Database +# +# localhost is used to configure the loopback interface +# when the system is booting. Do not change this entry. +## + +127.0.0.1 localhost localhost.dev +::1 localhost-v6 + +#172.168.0.1 ignoreme + +127.0.0.2 localhost2 diff --git a/t/patches/01-resolve-hosts-file.t b/t/patches/01-resolve-hosts-file.t new file mode 100644 index 0000000..5d17384 --- /dev/null +++ b/t/patches/01-resolve-hosts-file.t @@ -0,0 +1,218 @@ +# vim:set ft= ts=4 sw=4 et fdm=marker: +use Test::Nginx::Socket::Lua; +use Cwd qw(cwd); + +plan tests => blocks() * repeat_each() * 3 + 2; + +our $CWD = cwd(); + +$ENV{TEST_NGINX_RESOLVER} ||= '8.8.8.8'; +$ENV{TEST_NGINX_HOSTS_FILE_PATH} = "$::CWD/t/fixtures/hosts"; + +run_tests() + +__DATA__ + +=== TEST 1: does not resolve localhost without hostsfile +--- config + resolver $TEST_NGINX_RESOLVER; + + location = /echo { + echo "Hello"; + } + + location = /t { + set $upstream localhost; + proxy_pass http://$upstream:$TEST_NGINX_SERVER_PORT/echo; + } +--- request +GET /t +--- error_code: 502 +--- error_log +localhost could not be resolved (3: Host not found) + + + +=== TEST 2: exits if file does not exists +--- config + resolver $TEST_NGINX_RESOLVER hostsfile=/tmp/no_such_hostsfile; +--- must_die +--- error_log +open() "/tmp/no_such_hostsfile" failed (2: No such file or directory) +--- no_error_log +[error] + + + +=== TEST 3: hostsfile option parses hosts file (--with-ipv6) +--- config + resolver $TEST_NGINX_RESOLVER hostsfile=$TEST_NGINX_HOSTS_FILE_PATH; + + location = /t { + echo "Hello"; + } +--- request +GET /t +--- error_log eval +[ + qr/\[notice\] .*? host "localhost" will resolve to "127\.0\.0\.1"/, + qr/\[notice\] .*? host "localhost\.dev" will resolve to "127\.0\.0\.1"/, + qr/\[notice\] .*? host "localhost2" will resolve to "127\.0\.0\.2"/, + qr/\[notice\] .*? host "localhost-v6" will resolve to "::1"/, +] +--- no_error_log +[error] +host "ignoreme" will resolve to "172.168.0.1" +--- skip_eval: 3: system("ping6 -c 1 ::1 >/dev/null 2>&1") eq 0 + + + +=== TEST 4: hostsfile option parses hosts file (--without-ipv6) +--- config + resolver $TEST_NGINX_RESOLVER hostsfile=$TEST_NGINX_HOSTS_FILE_PATH; + + location = /t { + echo "Hello"; + } +--- request +GET /t +--- error_log eval +[ + qr/\[notice\] .*? host "localhost" will resolve to "127\.0\.0\.1"/, + qr/\[notice\] .*? host "localhost\.dev" will resolve to "127\.0\.0\.1"/, + qr/\[notice\] .*? host "localhost2" will resolve to "127\.0\.0\.2"/, +] +--- no_error_log +[error] +host "ignoreme" will resolve to "172.168.0.1" +host "localhost-v6" will resolve to "::1", +--- skip_eval: 3: system("ping6 -c 1 ::1 >/dev/null 2>&1") ne 0 + + + +=== TEST 5: hostsfile option caches parsed hostnames +--- config + resolver $TEST_NGINX_RESOLVER hostsfile=$TEST_NGINX_HOSTS_FILE_PATH; + + location = /echo { + echo "Hello"; + } + + location = /t { + set $upstream localhost; + proxy_pass http://$upstream:$TEST_NGINX_SERVER_PORT/echo; + } +--- request +GET /t +--- response_body +Hello +--- no_error_log +[error] + + + +=== TEST 6: hostsfile option caches parsed hostnames (bis) +--- config + resolver $TEST_NGINX_RESOLVER hostsfile=$TEST_NGINX_HOSTS_FILE_PATH; + + location = /echo { + echo "Hello"; + } + + location = /t { + set $upstream localhost.dev; + proxy_pass http://$upstream:$TEST_NGINX_SERVER_PORT/echo; + } +--- request +GET /t +--- response_body +Hello +--- no_error_log +[error] + + + +=== TEST 7: hostsfile option honors ipv6=off flag +--- config + resolver $TEST_NGINX_RESOLVER hostsfile=$TEST_NGINX_HOSTS_FILE_PATH ipv6=off; + + location = /t { + echo "Hello"; + } +--- request +GET /t +--- response_body +Hello +--- error_log eval +[ + qr/\[notice\] .*? host "localhost" will resolve to "127\.0\.0\.1"/, + qr/\[notice\] .*? host "localhost\.dev" will resolve to "127\.0\.0\.1"/, + qr/\[notice\] .*? host "localhost2" will resolve to "127\.0\.0\.2"/, +] +--- no_error_log +host "localhost-v6" will resolve to "::1" +--- skip_eval: 3: system("ping6 -c 1 ::1 >/dev/null 2>&1") eq 0 + + + +=== TEST 8: resolver without nameserver are accepted +--- config + resolver hostsfile=$TEST_NGINX_HOSTS_FILE_PATH; + + location = /t { + echo "Hello"; + } +--- request +GET /t +--- response_body +Hello +--- no_error_log +[error] + + + +=== TEST 9: resolver without nameserver and without hostsfile still errors out +--- config + resolver; +--- must_die +--- error_log +invalid number of arguments in "resolver" directive +--- no_error_log +[error] + + + +=== TEST 10: resolver without nameserver returns cached hostnames from hostsfile +--- config + resolver hostsfile=$TEST_NGINX_HOSTS_FILE_PATH; + + location = /echo { + echo "Hello"; + } + + location = /t { + set $upstream localhost; + proxy_pass http://$upstream:$TEST_NGINX_SERVER_PORT/echo; + } +--- request +GET /t +--- response_body +Hello +--- no_error_log +[error] + + + +=== TEST 11: resolver without nameserver returns proper error when no cached hostname +--- config + resolver hostsfile=$TEST_NGINX_HOSTS_FILE_PATH; + + location = /t { + set $upstream openresty.org; + proxy_pass http://$upstream/; + } +--- request +GET /t +--- error_code: 502 +--- error_log +openresty.org could not be resolved (3: Host not found) diff --git a/util/mirror-tarballs b/util/mirror-tarballs index 31527df..5d74ce3 100755 --- a/util/mirror-tarballs +++ b/util/mirror-tarballs @@ -386,6 +386,10 @@ echo "$info_txt applying the safe_resolver_ipv6_option patch for nginx" patch -p1 < $root/patches/nginx-$main_ver-safe_resolver_ipv6_option.patch || exit 1 echo +echo "$info_txt applying the resolve hosts file patch for nginx" +patch -p1 < $root/patches/nginx-$main_ver-resolve-hosts-file.patch || exit 1 +echo + cp $root/html/index.html docs/html/ || exit 1 cp $root/html/50x.html docs/html/ || exit 1