diff --git a/.github/workflows/test-ntls.yml b/.github/workflows/test-ntls.yml index 6abf669fca..1c0b8f28ad 100644 --- a/.github/workflows/test-ntls.yml +++ b/.github/workflows/test-ntls.yml @@ -77,6 +77,7 @@ jobs: --with-openssl=../Tongsuo \ --with-openssl-opt="--api=1.1.1 enable-ntls" \ --with-http_ssl_module \ + --with-http_v2_module \ --with-stream \ --with-stream_ssl_module \ --with-stream_sni diff --git a/modules/ngx_http_lua_module/config b/modules/ngx_http_lua_module/config index 0e572c8bea..22edfd7bd9 100644 --- a/modules/ngx_http_lua_module/config +++ b/modules/ngx_http_lua_module/config @@ -514,3 +514,5 @@ CORE_INCS="$CORE_INCS $ngx_addon_dir/src/api" CFLAGS="$CFLAGS -DNDK_SET_VAR" echo "/* DO NOT EDIT! This file was automatically generated by config */" > "$ngx_addon_dir/src/ngx_http_lua_autoconf.h" + +have=T_NGX_HTTP_HAVE_LUA_MODULE . auto/have diff --git a/modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c b/modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c index b8e70ddefa..2819edbb10 100644 --- a/modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c +++ b/modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c @@ -231,6 +231,13 @@ ngx_http_lua_ssl_cert_handler(ngx_ssl_conn_t *ssl_conn, void *data) hc = c->data; +#if (T_NGX_XQUIC) + if (c->xquic_conn) { + ngx_http_xquic_connection_t *qc = (ngx_http_xquic_connection_t *)c->data; + hc = qc->http_connection; + } +#endif + fc = ngx_http_lua_create_fake_connection(NULL); if (fc == NULL) { goto failed; @@ -255,6 +262,10 @@ ngx_http_lua_ssl_cert_handler(ngx_ssl_conn_t *ssl_conn, void *data) fc->log->log_level = c->log->log_level; fc->ssl = c->ssl; +#if (T_NGX_XQUIC) + fc->xquic_conn = c->xquic_conn; +#endif + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); #if (nginx_version >= 1009000) diff --git a/modules/ngx_http_xquic_module/README.md b/modules/ngx_http_xquic_module/README.md index 2b9355a183..d2d48ee9ae 100644 --- a/modules/ngx_http_xquic_module/README.md +++ b/modules/ngx_http_xquic_module/README.md @@ -65,6 +65,8 @@ make install ```nginx worker_processes 1; +error_log logs/error.log debug; + events { worker_connections 1024; } @@ -97,3 +99,103 @@ http { ``` 更为详细的指令可参考官网文档 [XQUIC模块](http://tengine.taobao.org/document_cn/xquic_cn.html) + +# 浏览器使用 HTTP3 + +**注意：浏览器访问需要确保证书受信。** + +浏览器默认不会使用 `HTTP3` 请求，需要服务端响应包头 `Alt-Svc` 进行升级说明，浏览器通过响应包头感知到服务端是支持 `HTTP3` 的，下次请求会尝试使用 `HTTP3`。 + +```nginx +worker_processes 1; + +user root; + +error_log logs/error.log debug; + +events { + worker_connections 1024; +} + +xquic_log "pipe:rollback /usr/local/tengine/logs/tengine-xquic.log baknum=10 maxsize=1G interval=1d adjust=600" info; + +http { + xquic_ssl_certificate /usr/local/tengine/ssl/default-fake-certificate.pem; + xquic_ssl_certificate_key /usr/local/tengine/ssl/default-fake-certificate.pem; + + server { + listen 2443 xquic reuseport; + + location / { + } + } + + server { + listen 80 default_server reuseport backlog=4096; + listen 443 default_server reuseport backlog=4096 ssl http2; + listen 443 default_server reuseport backlog=4096 xquic; + + server_name s1.test.com; + + add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always; + + ssl_certificate /etc/ingress-controller/ssl/s1.crt; + ssl_certificate_key /etc/ingress-controller/ssl/s1.key; + } + + server { + listen 80; + listen 443 ssl http2; + listen 443 xquic; + + server_name s2.test.com; + + add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always; + + ssl_certificate /etc/ingress-controller/ssl/s2.crt; + ssl_certificate_key /etc/ingress-controller/ssl/s2.key; + } +} +``` + +通过以上配置，浏览器访问对应域名，第一次访问 `HTTP2`，下次访问会切换至 `HTTP3`。 + +**注意**： + +在生产环境中，处于安全性考虑，一般情况会以普通用户权限启动 `Tenigne`，而 `xquic` 功能在普通用户权限下，监听端口必须配置为 1024 以上，如监听 2443 端口，那对外的四层负载均衡需要做 443 到 2443 端口的映射，`Tenigne` `Server`段配置示例： + +```nginx + server { + listen 80 default_server reuseport backlog=4096; + listen 443 default_server reuseport backlog=4096 ssl http2; + listen 2443 default_server reuseport backlog=4096 xquic; + + add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always; + + ssl_certificate /etc/ingress-controller/ssl/s1.crt; + ssl_certificate_key /etc/ingress-controller/ssl/s1.key; + } +``` + +四层负载均衡配置示例： + +```yaml + type: LoadBalancer + ports: + - port: 80 + name: tengine-tcp-80 + protocol: TCP + targetPort: 80 + - port: 443 + name: tengine-tcp-443 + protocol: TCP + targetPort: 443 + - port: 443 + name: tengine-udp-443 + protocol: UDP + targetPort: 2443 + selector: + app: tengine +``` + +对用户来讲，还是通过 443 端口访问，通过四层负载均衡设备，转换为 `Tengine` 的 2443 端口。 diff --git a/modules/ngx_http_xquic_module/ngx_http_xquic.c b/modules/ngx_http_xquic_module/ngx_http_xquic.c index a5ea09f18b..470366454c 100644 --- a/modules/ngx_http_xquic_module/ngx_http_xquic.c +++ b/modules/ngx_http_xquic_module/ngx_http_xquic.c @@ -20,6 +20,10 @@ #endif +#ifdef T_NGX_HTTP_HAVE_LUA_MODULE +#include +extern ngx_module_t ngx_http_lua_module; +#endif ngx_int_t ngx_http_v3_conn_check_concurrent_cnt(ngx_http_xquic_main_conf_t *qmcf) @@ -187,6 +191,22 @@ ngx_http_v3_cert_cb(const char *sni, void **chain, hc = qc->http_connection; c = qc->connection; +#ifdef T_NGX_HTTP_HAVE_LUA_MODULE + ngx_http_lua_srv_conf_t *lscf = NULL; + + lscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_lua_module); + if (lscf != NULL && lscf->srv.ssl_cert_src.len) { + ngx_ssl_conn_t *ssl_conn = qc->ssl_conn; + + ngx_http_lua_ssl_cert_handler(ssl_conn, NULL); + *chain = NULL; + *cert = NULL; + *key = NULL; + + return XQC_OK; + } +#endif + /* * get the server core conf by sni, this is useful when multiple server * block listen on the same port. but useless when there is noly a single @@ -256,6 +276,8 @@ int ngx_http_v3_conn_create_notify(xqc_h3_conn_t *h3_conn, const xqc_cid_t *cid, void *user_data) { + ngx_connection_t *c; + /* we set alp user_data when accept connection */ ngx_http_xquic_connection_t *user_conn = (ngx_http_xquic_connection_t *) user_data; user_conn->ssl_conn = (ngx_ssl_conn_t *) xqc_h3_conn_get_ssl(h3_conn); @@ -265,6 +287,24 @@ ngx_http_v3_conn_create_notify(xqc_h3_conn_t *h3_conn, xqc_h3_conn_set_user_data(h3_conn, user_conn); + c = user_conn->connection; + + if (SSL_set_ex_data(user_conn->ssl_conn, ngx_ssl_connection_index, c) == 0) + { + ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "|xquic|SSL_set_ex_data() failed|"); + return XQC_ERROR; + } + + c->xquic_conn = 1; + + ngx_ssl_connection_t *p_ssl = ngx_pcalloc(c->pool, sizeof(ngx_ssl_connection_t)); + if (p_ssl == NULL) { + ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "|xquic|alloc ngx_ssl_connection_t failed|"); + return XQC_ERROR; + } + p_ssl->connection = user_conn->ssl_conn; + c->ssl = p_ssl; + return NGX_OK; } diff --git a/modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c b/modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c index 7e2430efe4..54a045e259 100644 --- a/modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c +++ b/modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c @@ -465,8 +465,18 @@ ngx_http_xquic_header_filter(ngx_http_request_t *r) h->hash = 1; ngx_str_set(&h->key, NGX_HTTP_XQUIC_NAME_SERVER); - if (clcf->server_tokens) { + if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) { +#if (T_NGX_SERVER_INFO) + ngx_str_set(&h->value, TENGINE_VER); +#else ngx_str_set(&h->value, NGINX_VER); +#endif + } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) { +#if (T_NGX_SERVER_INFO) + ngx_str_set(&h->value, TENGINE_VER_BUILD); +#else + ngx_str_set(&h->value, NGINX_VER_BUILD); +#endif } else { ngx_str_set(&h->value, TENGINE); } diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h index 33b4570dce..8a048b63fd 100644 --- a/src/core/ngx_connection.h +++ b/src/core/ngx_connection.h @@ -264,6 +264,10 @@ struct ngx_connection_s { #if (T_NGX_HAVE_XUDP) unsigned xudp_tx:1; #endif + +#if (T_NGX_XQUIC) + unsigned xquic_conn:1; +#endif }; diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c index be4da2d0c1..d5cafe6ae5 100644 --- a/src/event/modules/ngx_epoll_module.c +++ b/src/event/modules/ngx_epoll_module.c @@ -1009,7 +1009,7 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) #if (NGX_SSL && NGX_SSL_ASYNC) aev = c->async; - if ((revents & EPOLLIN) && aev->active && async) { + if ((revents & EPOLLIN) && aev && aev->active && async) { if (c->async_fd == -1 || aev->instance!= instance) { /* diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c index de1c04ac56..49f0df1cc0 100755 --- a/src/event/ngx_event_openssl.c +++ b/src/event/ngx_event_openssl.c @@ -3898,7 +3898,13 @@ ngx_ssl_shutdown(ngx_connection_t *c) return rc; } +#if (T_NGX_XQUIC) + if (!c->xquic_conn) { + SSL_free(c->ssl->connection); + } +#else SSL_free(c->ssl->connection); +#endif c->ssl = NULL; c->recv = ngx_recv; diff --git a/src/http/modules/ngx_http_grpc_module.c b/src/http/modules/ngx_http_grpc_module.c index 23609f6b93..a7763ca1cf 100644 --- a/src/http/modules/ngx_http_grpc_module.c +++ b/src/http/modules/ngx_http_grpc_module.c @@ -5057,18 +5057,19 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ngx_http_grpc_loc_conf_t *glcf) #if (T_NGX_SSL_NTLS) glcf->upstream.tls_method = SSL_CTX_get_ssl_method(glcf->upstream.ssl->ctx); - if (glcf->enc_certificate.len) { + if (glcf->upstream.enc_certificate.len) { - if (glcf->enc_certificate_key.len == 0) { + if (glcf->upstream.enc_certificate_key.len == 0) { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, "no \"grpc_ssl_enc_certificate_key\" is defined " - "for certificate \"%V\"", &glcf->enc_certificate); + "for certificate \"%V\"", + &glcf->upstream.enc_certificate); return NGX_ERROR; } if (ngx_ssl_certificate(cf, glcf->upstream.ssl, - &glcf->upstream.enc_certificate->value, - &glcf->upstream.enc_certificate_key->value, + &glcf->upstream.enc_certificate, + &glcf->upstream.enc_certificate_key, glcf->upstream.ssl_passwords, SSL_ENC_CERT) != NGX_OK) @@ -5077,12 +5078,13 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ngx_http_grpc_loc_conf_t *glcf) } } - if (glcf->sign_certificate.len) { + if (glcf->upstream.sign_certificate.len) { - if (glcf->sign_certificate_key.len == 0) { + if (glcf->upstream.sign_certificate_key.len == 0) { ngx_log_error(NGX_LOG_EMERG, cf->log, 0, "no \"grpc_ssl_sign_certificate_key\" is defined " - "for certificate \"%V\"", &glcf->sign_certificate); + "for certificate \"%V\"", + &glcf->upstream.sign_certificate); return NGX_ERROR; }