From d3735f8c33af4da2d233d08438173cd2cda817fa Mon Sep 17 00:00:00 2001 From: zhsnew Date: Mon, 17 Jul 2023 20:15:30 +0800 Subject: [PATCH 1/9] bugfix: When using thread_pool program crash --- src/event/modules/ngx_epoll_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c index be4da2d0c1..d5cafe6ae5 100644 --- a/src/event/modules/ngx_epoll_module.c +++ b/src/event/modules/ngx_epoll_module.c @@ -1009,7 +1009,7 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) #if (NGX_SSL && NGX_SSL_ASYNC) aev = c->async; - if ((revents & EPOLLIN) && aev->active && async) { + if ((revents & EPOLLIN) && aev && aev->active && async) { if (c->async_fd == -1 || aev->instance!= instance) { /* From 7d0833b820d88bfe54c2a6a15c6fec19041814fc Mon Sep 17 00:00:00 2001 From: K1 Date: Sun, 23 Jul 2023 11:47:36 +0800 Subject: [PATCH 2/9] Fix compilation failure of grpc module when add ngx_tongsuo_ntls CI test-ntls adds http v2 module, so grpc module is enabled to avoid compilation failure like this. Fix #1812 --- .github/workflows/test-ntls.yml | 1 + src/http/modules/ngx_http_grpc_module.c | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-ntls.yml b/.github/workflows/test-ntls.yml index 6abf669fca..1c0b8f28ad 100644 --- a/.github/workflows/test-ntls.yml +++ b/.github/workflows/test-ntls.yml @@ -77,6 +77,7 @@ jobs: --with-openssl=../Tongsuo \ --with-openssl-opt="--api=1.1.1 enable-ntls" \ --with-http_ssl_module \ + --with-http_v2_module \ --with-stream \ --with-stream_ssl_module \ --with-stream_sni diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index 23609f6b93..a7763ca1cf 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -5057,18 +5057,19 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ngx_http_grpc_loc_conf_t *glcf) #if (T_NGX_SSL_NTLS) glcf->upstream.tls_method = SSL_CTX_get_ssl_method(glcf->upstream.ssl->ctx); - if (glcf->enc_certificate.len) { + if (glcf->upstream.enc_certificate.len) { - if (glcf->enc_certificate_key.len == 0) { + if (glcf->upstream.enc_certificate_key.len == 0) { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, "no \"grpc_ssl_enc_certificate_key\" is defined " - "for certificate \"%V\"", &glcf->enc_certificate); + "for certificate \"%V\"", + &glcf->upstream.enc_certificate); return NGX_ERROR; } if (ngx_ssl_certificate(cf, glcf->upstream.ssl, - &glcf->upstream.enc_certificate->value, - &glcf->upstream.enc_certificate_key->value, + &glcf->upstream.enc_certificate, + &glcf->upstream.enc_certificate_key, glcf->upstream.ssl_passwords, SSL_ENC_CERT) != NGX_OK) @@ -5077,12 +5078,13 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ngx_http_grpc_loc_conf_t *glcf) } } - if (glcf->sign_certificate.len) { + if (glcf->upstream.sign_certificate.len) { - if (glcf->sign_certificate_key.len == 0) { + if (glcf->upstream.sign_certificate_key.len == 0) { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, "no \"grpc_ssl_sign_certificate_key\" is defined " - "for certificate \"%V\"", &glcf->sign_certificate); + "for certificate \"%V\"", + &glcf->upstream.sign_certificate); return NGX_ERROR; } From c8b70e47dead461346ae3cc7a831c939f5029ead Mon Sep 17 00:00:00 2001 From: drawing Date: Mon, 24 Jul 2023 21:27:13 +0800 Subject: [PATCH 3/9] README:add xquic document --- modules/ngx_http_xquic_module/README.md | 90 +++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/modules/ngx_http_xquic_module/README.md b/modules/ngx_http_xquic_module/README.md index 2b9355a183..590d0fe5a5 100644 --- a/modules/ngx_http_xquic_module/README.md +++ b/modules/ngx_http_xquic_module/README.md @@ -97,3 +97,93 @@ http { ``` 更为详细的指令可参考官网文档 [XQUIC模块](http://tengine.taobao.org/document_cn/xquic_cn.html) + +# 浏览器使用 HTTP3 + +浏览器默认不会使用 `HTTP3` 请求，需要服务端响应包头 `Alt-Svc` 进行升级说明，浏览器通过响应包头感知到服务端是支持 `HTTP3` 的，下次请求会尝试使用 `HTTP3`。 + +```nginx +worker_processes 1; + +events { + worker_connections 1024; +} + +xquic_log "pipe:rollback /usr/local/tengine/logs/tengine-xquic.log baknum=10 maxsize=1G interval=1d adjust=600" info; + +http { + xquic_ssl_certificate /usr/local/tengine/ssl/default-fake-certificate.pem; + xquic_ssl_certificate_key /usr/local/tengine/ssl/default-fake-certificate.pem; + + server { + listen 2443 xquic reuseport; + + location / { + } + } + + server { + listen 80 default_server reuseport backlog=4096; + listen 443 default_server reuseport backlog=4096 ssl http2; + listen 443 default_server reuseport backlog=4096 xquic; + + add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always; + + ssl_certificate /etc/ingress-controller/ssl/s1.crt; + ssl_certificate_key /etc/ingress-controller/ssl/s1.key; + } + + server { + listen 80; + listen 443 ssl http2; + listen 443 xquic; + + add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always; + + ssl_certificate /etc/ingress-controller/ssl/s2.crt; + ssl_certificate_key /etc/ingress-controller/ssl/s2.key; + } +} +``` + +通过以上配置，浏览器访问对应域名，第一次访问 `HTTP2`，下次访问会切换至 `HTTP3`。 + +**注意**： + +在生产环境中，处于安全性考虑，一般情况会以普通用户权限启动 `Tenigne`，而 `xquic` 功能在普通用户权限下，监听端口必须配置为 1024 以上，如监听 2443 端口，那对外的四层负载均衡需要做 443 到 2443 端口的映射，`Tenigne` `Server`段配置示例： + +```nginx + server { + listen 80 default_server reuseport backlog=4096; + listen 443 default_server reuseport backlog=4096 ssl http2; + listen 2443 default_server reuseport backlog=4096 xquic; + + add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always; + + ssl_certificate /etc/ingress-controller/ssl/s1.crt; + ssl_certificate_key /etc/ingress-controller/ssl/s1.key; + } +``` + +四层负载均衡配置示例： + +```yaml + type: LoadBalancer + ports: + - port: 80 + name: tengine-tcp-80 + protocol: TCP + targetPort: 80 + - port: 443 + name: tengine-tcp-443 + protocol: TCP + targetPort: 443 + - port: 443 + name: tengine-udp-443 + protocol: UDP + targetPort: 2443 + selector: + app: tengine +``` + +对用户来讲，还是通过 443 端口访问，通过四层负责均衡设备，转换为 `Tengine` 的 2443 端口。 From 62aa98bfba0d8f4ff699609fe238e121a910a866 Mon Sep 17 00:00:00 2001 From: drawing Date: Mon, 24 Jul 2023 21:32:10 +0800 Subject: [PATCH 4/9] README:xquic document add servername --- modules/ngx_http_xquic_module/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/ngx_http_xquic_module/README.md b/modules/ngx_http_xquic_module/README.md index 590d0fe5a5..3291fb89be 100644 --- a/modules/ngx_http_xquic_module/README.md +++ b/modules/ngx_http_xquic_module/README.md @@ -127,6 +127,8 @@ http { listen 443 default_server reuseport backlog=4096 ssl http2; listen 443 default_server reuseport backlog=4096 xquic; + server_name s1.test.com; + add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always; ssl_certificate /etc/ingress-controller/ssl/s1.crt; @@ -138,6 +140,8 @@ http { listen 443 ssl http2; listen 443 xquic; + server_name s2.test.com; + add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always; ssl_certificate /etc/ingress-controller/ssl/s2.crt; From f942f54c384a1500a46ed653717f7b59ac77f409 Mon Sep 17 00:00:00 2001 From: drawing Date: Tue, 25 Jul 2023 14:16:14 +0800 Subject: [PATCH 5/9] xquic module: fix server_tokens #1827 --- .../ngx_http_xquic_filter_module.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c b/modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c index 7e2430efe4..54a045e259 100644 --- a/modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c +++ b/modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c @@ -465,8 +465,18 @@ ngx_http_xquic_header_filter(ngx_http_request_t *r) h->hash = 1; ngx_str_set(&h->key, NGX_HTTP_XQUIC_NAME_SERVER); - if (clcf->server_tokens) { + if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +#if (T_NGX_SERVER_INFO) + ngx_str_set(&h->value, TENGINE_VER); +#else ngx_str_set(&h->value, NGINX_VER); +#endif + } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +#if (T_NGX_SERVER_INFO) + ngx_str_set(&h->value, TENGINE_VER_BUILD); +#else + ngx_str_set(&h->value, NGINX_VER_BUILD); +#endif } else { ngx_str_set(&h->value, TENGINE); } From 0bc59962b86d5f6824f212adbc7bf344bafbe55b Mon Sep 17 00:00:00 2001 From: drawing Date: Tue, 25 Jul 2023 20:57:30 +0800 Subject: [PATCH 6/9] xquic: support lua cert cb #1826 --- modules/ngx_http_lua_module/config | 2 + .../src/ngx_http_lua_ssl_certby.c | 11 +++++ modules/ngx_http_xquic_module/README.md | 2 + .../ngx_http_xquic_module/ngx_http_xquic.c | 40 +++++++++++++++++++ src/core/ngx_connection.h | 4 ++ src/event/ngx_event_openssl.c | 6 +++ 6 files changed, 65 insertions(+) diff --git a/modules/ngx_http_lua_module/config b/modules/ngx_http_lua_module/config index 0e572c8bea..22edfd7bd9 100644 --- a/modules/ngx_http_lua_module/config +++ b/modules/ngx_http_lua_module/config @@ -514,3 +514,5 @@ CORE_INCS="$CORE_INCS $ngx_addon_dir/src/api" CFLAGS="$CFLAGS -DNDK_SET_VAR" echo "/* DO NOT EDIT! This file was automatically generated by config */" > "$ngx_addon_dir/src/ngx_http_lua_autoconf.h" + +have=T_NGX_HTTP_HAVE_LUA_MODULE . auto/have diff --git a/modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c b/modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c index b8e70ddefa..2819edbb10 100644 --- a/modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c +++ b/modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c @@ -231,6 +231,13 @@ ngx_http_lua_ssl_cert_handler(ngx_ssl_conn_t *ssl_conn, void *data) hc = c->data; +#if (T_NGX_XQUIC) + if (c->xquic_conn) { + ngx_http_xquic_connection_t *qc = (ngx_http_xquic_connection_t *)c->data; + hc = qc->http_connection; + } +#endif + fc = ngx_http_lua_create_fake_connection(NULL); if (fc == NULL) { goto failed; @@ -255,6 +262,10 @@ ngx_http_lua_ssl_cert_handler(ngx_ssl_conn_t *ssl_conn, void *data) fc->log->log_level = c->log->log_level; fc->ssl = c->ssl; +#if (T_NGX_XQUIC) + fc->xquic_conn = c->xquic_conn; +#endif + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); #if (nginx_version >= 1009000) diff --git a/modules/ngx_http_xquic_module/README.md b/modules/ngx_http_xquic_module/README.md index 3291fb89be..4a1ce49ba3 100644 --- a/modules/ngx_http_xquic_module/README.md +++ b/modules/ngx_http_xquic_module/README.md @@ -100,6 +100,8 @@ http { # 浏览器使用 HTTP3 +**注意：浏览器访问需要确保证书受信。** + 浏览器默认不会使用 `HTTP3` 请求，需要服务端响应包头 `Alt-Svc` 进行升级说明，浏览器通过响应包头感知到服务端是支持 `HTTP3` 的，下次请求会尝试使用 `HTTP3`。 ```nginx diff --git a/modules/ngx_http_xquic_module/ngx_http_xquic.c b/modules/ngx_http_xquic_module/ngx_http_xquic.c index a5ea09f18b..470366454c 100644 --- a/modules/ngx_http_xquic_module/ngx_http_xquic.c +++ b/modules/ngx_http_xquic_module/ngx_http_xquic.c @@ -20,6 +20,10 @@ #endif +#ifdef T_NGX_HTTP_HAVE_LUA_MODULE +#include +extern ngx_module_t ngx_http_lua_module; +#endif ngx_int_t ngx_http_v3_conn_check_concurrent_cnt(ngx_http_xquic_main_conf_t *qmcf) @@ -187,6 +191,22 @@ ngx_http_v3_cert_cb(const char *sni, void **chain, hc = qc->http_connection; c = qc->connection; +#ifdef T_NGX_HTTP_HAVE_LUA_MODULE + ngx_http_lua_srv_conf_t *lscf = NULL; + + lscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_lua_module); + if (lscf != NULL && lscf->srv.ssl_cert_src.len) { + ngx_ssl_conn_t *ssl_conn = qc->ssl_conn; + + ngx_http_lua_ssl_cert_handler(ssl_conn, NULL); + *chain = NULL; + *cert = NULL; + *key = NULL; + + return XQC_OK; + } +#endif + /* * get the server core conf by sni, this is useful when multiple server * block listen on the same port. but useless when there is noly a single @@ -256,6 +276,8 @@ int ngx_http_v3_conn_create_notify(xqc_h3_conn_t *h3_conn, const xqc_cid_t *cid, void *user_data) { + ngx_connection_t *c; + /* we set alp user_data when accept connection */ ngx_http_xquic_connection_t *user_conn = (ngx_http_xquic_connection_t *) user_data; user_conn->ssl_conn = (ngx_ssl_conn_t *) xqc_h3_conn_get_ssl(h3_conn); @@ -265,6 +287,24 @@ ngx_http_v3_conn_create_notify(xqc_h3_conn_t *h3_conn, xqc_h3_conn_set_user_data(h3_conn, user_conn); + c = user_conn->connection; + + if (SSL_set_ex_data(user_conn->ssl_conn, ngx_ssl_connection_index, c) == 0) + { + ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "|xquic|SSL_set_ex_data() failed|"); + return XQC_ERROR; + } + + c->xquic_conn = 1; + + ngx_ssl_connection_t *p_ssl = ngx_pcalloc(c->pool, sizeof(ngx_ssl_connection_t)); + if (p_ssl == NULL) { + ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "|xquic|alloc ngx_ssl_connection_t failed|"); + return XQC_ERROR; + } + p_ssl->connection = user_conn->ssl_conn; + c->ssl = p_ssl; + return NGX_OK; } diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h index 33b4570dce..8a048b63fd 100644 --- a/src/core/ngx_connection.h +++ b/src/core/ngx_connection.h @@ -264,6 +264,10 @@ struct ngx_connection_s { #if (T_NGX_HAVE_XUDP) unsigned xudp_tx:1; #endif + +#if (T_NGX_XQUIC) + unsigned xquic_conn:1; +#endif }; diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index de1c04ac56..49f0df1cc0 100755 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3898,7 +3898,13 @@ ngx_ssl_shutdown(ngx_connection_t *c) return rc; } +#if (T_NGX_XQUIC) + if (!c->xquic_conn) { + SSL_free(c->ssl->connection); + } +#else SSL_free(c->ssl->connection); +#endif c->ssl = NULL; c->recv = ngx_recv; From ca1126b3eb3fa46fb40974304b5d0bd52c8a840f Mon Sep 17 00:00:00 2001 From: drawing Date: Wed, 26 Jul 2023 12:27:30 +0800 Subject: [PATCH 7/9] xquic: update readme --- modules/ngx_http_xquic_module/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ngx_http_xquic_module/README.md b/modules/ngx_http_xquic_module/README.md index 4a1ce49ba3..65fd238f86 100644 --- a/modules/ngx_http_xquic_module/README.md +++ b/modules/ngx_http_xquic_module/README.md @@ -192,4 +192,4 @@ http { app: tengine ``` -对用户来讲，还是通过 443 端口访问，通过四层负责均衡设备，转换为 `Tengine` 的 2443 端口。 +对用户来讲，还是通过 443 端口访问，通过四层负载均衡设备，转换为 `Tengine` 的 2443 端口。 From 9cbcecf24ab8153f22c7c407ed53146a8bc54b01 Mon Sep 17 00:00:00 2001 From: drawing Date: Fri, 28 Jul 2023 15:24:07 +0800 Subject: [PATCH 8/9] xquic document: add error_log --- modules/ngx_http_xquic_module/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/ngx_http_xquic_module/README.md b/modules/ngx_http_xquic_module/README.md index 65fd238f86..c8222eab38 100644 --- a/modules/ngx_http_xquic_module/README.md +++ b/modules/ngx_http_xquic_module/README.md @@ -65,6 +65,8 @@ make install ```nginx worker_processes 1; +error_log logs/error.log debug; + events { worker_connections 1024; } @@ -107,6 +109,8 @@ http { ```nginx worker_processes 1; +error_log logs/error.log debug; + events { worker_connections 1024; } From 6a7688e9a6469301644ef6b06d4c3292a6f61c3a Mon Sep 17 00:00:00 2001 From: drawing Date: Fri, 28 Jul 2023 15:30:11 +0800 Subject: [PATCH 9/9] xquic document: change user root --- modules/ngx_http_xquic_module/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ngx_http_xquic_module/README.md b/modules/ngx_http_xquic_module/README.md index c8222eab38..d2d48ee9ae 100644 --- a/modules/ngx_http_xquic_module/README.md +++ b/modules/ngx_http_xquic_module/README.md @@ -109,6 +109,8 @@ http { ```nginx worker_processes 1; +user root; + error_log logs/error.log debug; events {