Merge remote-tracking branch 'tengine/master'

.github/workflows/test-ntls.yml

@@ -77,6 +77,7 @@ jobs:
             --with-openssl=../Tongsuo \
             --with-openssl-opt="--api=1.1.1 enable-ntls" \
             --with-http_ssl_module \
+            --with-http_v2_module \
             --with-stream \
             --with-stream_ssl_module \
             --with-stream_sni

modules/ngx_http_lua_module/config

@@ -514,3 +514,5 @@ CORE_INCS="$CORE_INCS $ngx_addon_dir/src/api"
 CFLAGS="$CFLAGS -DNDK_SET_VAR"
 
 echo "/* DO NOT EDIT! This file was automatically generated by config */" > "$ngx_addon_dir/src/ngx_http_lua_autoconf.h"
+
+have=T_NGX_HTTP_HAVE_LUA_MODULE . auto/have

modules/ngx_http_lua_module/src/ngx_http_lua_ssl_certby.c

@@ -231,6 +231,13 @@ ngx_http_lua_ssl_cert_handler(ngx_ssl_conn_t *ssl_conn, void *data)
 
     hc = c->data;
 
+#if (T_NGX_XQUIC)
+    if (c->xquic_conn) {
+        ngx_http_xquic_connection_t *qc = (ngx_http_xquic_connection_t *)c->data;
+        hc = qc->http_connection;
+    }
+#endif
+
     fc = ngx_http_lua_create_fake_connection(NULL);
     if (fc == NULL) {
         goto failed;
@@ -255,6 +262,10 @@ ngx_http_lua_ssl_cert_handler(ngx_ssl_conn_t *ssl_conn, void *data)
     fc->log->log_level = c->log->log_level;
     fc->ssl = c->ssl;
 
+#if (T_NGX_XQUIC)
+    fc->xquic_conn = c->xquic_conn;
+#endif
+
     clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
 
 #if (nginx_version >= 1009000)

modules/ngx_http_xquic_module/README.md

@@ -65,6 +65,8 @@ make install
 ```nginx
 worker_processes  1;
 
+error_log  logs/error.log debug;
+
 events {
     worker_connections  1024;
 }
@@ -97,3 +99,103 @@ http {
 ```
 
 更为详细的指令可参考官网文档 [XQUIC模块](http://tengine.taobao.org/document_cn/xquic_cn.html)
+
+# 浏览器使用 HTTP3
+
+**注意：浏览器访问需要确保证书受信。**
+
+浏览器默认不会使用 `HTTP3` 请求，需要服务端响应包头 `Alt-Svc` 进行升级说明，浏览器通过响应包头感知到服务端是支持 `HTTP3` 的，下次请求会尝试使用 `HTTP3`。
+
+```nginx
+worker_processes  1;
+
+user root;
+
+error_log  logs/error.log debug;
+
+events {
+    worker_connections  1024;
+}
+
+xquic_log   "pipe:rollback /usr/local/tengine/logs/tengine-xquic.log baknum=10 maxsize=1G interval=1d adjust=600" info;
+
+http {
+    xquic_ssl_certificate        /usr/local/tengine/ssl/default-fake-certificate.pem;
+    xquic_ssl_certificate_key    /usr/local/tengine/ssl/default-fake-certificate.pem;
+
+    server {
+        listen 2443 xquic reuseport;
+
+        location / {
+        }
+    }
+
+    server {
+        listen 80 default_server reuseport backlog=4096;
+        listen 443 default_server reuseport backlog=4096 ssl http2;
+        listen 443 default_server reuseport backlog=4096 xquic;
+
+        server_name s1.test.com;
+
+        add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always;
+
+        ssl_certificate     /etc/ingress-controller/ssl/s1.crt;
+        ssl_certificate_key /etc/ingress-controller/ssl/s1.key;
+    }
+
+    server {
+        listen 80;
+        listen 443 ssl http2;
+        listen 443 xquic;
+
+        server_name s2.test.com;
+
+        add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always;
+
+        ssl_certificate     /etc/ingress-controller/ssl/s2.crt;
+        ssl_certificate_key /etc/ingress-controller/ssl/s2.key;
+    }
+}
+```
+
+通过以上配置，浏览器访问对应域名，第一次访问 `HTTP2`，下次访问会切换至 `HTTP3`。
+
+**注意**：
+
+在生产环境中，处于安全性考虑，一般情况会以普通用户权限启动 `Tenigne`，而 `xquic` 功能在普通用户权限下，监听端口必须配置为 1024 以上，如监听 2443 端口，那对外的四层负载均衡需要做 443 到 2443 端口的映射，`Tenigne` `Server`段配置示例：
+
+```nginx
+    server {
+        listen 80 default_server reuseport backlog=4096;
+        listen 443 default_server reuseport backlog=4096 ssl http2;
+        listen 2443 default_server reuseport backlog=4096 xquic;
+
+        add_header Alt-Svc 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000' always;
+
+        ssl_certificate     /etc/ingress-controller/ssl/s1.crt;
+        ssl_certificate_key /etc/ingress-controller/ssl/s1.key;
+    }
+```
+
+四层负载均衡配置示例：
+
+```yaml
+  type: LoadBalancer
+  ports:
+  - port: 80
+    name: tengine-tcp-80
+    protocol: TCP
+    targetPort: 80
+  - port: 443
+    name: tengine-tcp-443
+    protocol: TCP
+    targetPort: 443
+  - port: 443
+    name: tengine-udp-443
+    protocol: UDP
+    targetPort: 2443
+  selector:
+    app: tengine
+```
+
+对用户来讲，还是通过 443 端口访问，通过四层负载均衡设备，转换为 `Tengine` 的 2443 端口。

modules/ngx_http_xquic_module/ngx_http_xquic.c

@@ -20,6 +20,10 @@
 #endif
 
 
+#ifdef T_NGX_HTTP_HAVE_LUA_MODULE
+#include <ngx_http_lua_ssl_certby.h>
+extern ngx_module_t ngx_http_lua_module;
+#endif
 
 ngx_int_t
 ngx_http_v3_conn_check_concurrent_cnt(ngx_http_xquic_main_conf_t *qmcf)
@@ -187,6 +191,22 @@ ngx_http_v3_cert_cb(const char *sni, void **chain,
     hc = qc->http_connection;
     c = qc->connection;
 
+#ifdef T_NGX_HTTP_HAVE_LUA_MODULE
+    ngx_http_lua_srv_conf_t *lscf = NULL;
+
+    lscf = ngx_http_get_module_srv_conf(hc->conf_ctx, ngx_http_lua_module);
+    if (lscf != NULL && lscf->srv.ssl_cert_src.len)  {
+        ngx_ssl_conn_t *ssl_conn = qc->ssl_conn;
+
+        ngx_http_lua_ssl_cert_handler(ssl_conn, NULL);
+        *chain = NULL;
+        *cert = NULL;
+        *key = NULL;
+
+        return XQC_OK;
+    }
+#endif
+
     /*
      * get the server core conf by sni, this is useful when multiple server
      * block listen on the same port. but useless when there is noly a single
@@ -256,6 +276,8 @@ int
 ngx_http_v3_conn_create_notify(xqc_h3_conn_t *h3_conn, 
     const xqc_cid_t *cid, void *user_data)
 {
+    ngx_connection_t               *c;
+
     /* we set alp user_data when accept connection */
     ngx_http_xquic_connection_t *user_conn = (ngx_http_xquic_connection_t *) user_data;
     user_conn->ssl_conn = (ngx_ssl_conn_t *) xqc_h3_conn_get_ssl(h3_conn);
@@ -265,6 +287,24 @@ ngx_http_v3_conn_create_notify(xqc_h3_conn_t *h3_conn,
 
     xqc_h3_conn_set_user_data(h3_conn, user_conn);
 
+    c = user_conn->connection;
+
+    if (SSL_set_ex_data(user_conn->ssl_conn, ngx_ssl_connection_index, c) == 0)
+    {
+        ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "|xquic|SSL_set_ex_data() failed|");
+        return XQC_ERROR;
+    }
+
+    c->xquic_conn = 1;
+
+    ngx_ssl_connection_t *p_ssl = ngx_pcalloc(c->pool, sizeof(ngx_ssl_connection_t));
+    if (p_ssl ==  NULL) {
+        ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0, "|xquic|alloc ngx_ssl_connection_t failed|");
+        return XQC_ERROR;
+    }
+    p_ssl->connection = user_conn->ssl_conn;
+    c->ssl = p_ssl;
+
     return NGX_OK;
 }
 

modules/ngx_http_xquic_module/ngx_http_xquic_filter_module.c

@@ -465,8 +465,18 @@ ngx_http_xquic_header_filter(ngx_http_request_t *r)
 
         h->hash = 1;
         ngx_str_set(&h->key, NGX_HTTP_XQUIC_NAME_SERVER);
-        if (clcf->server_tokens) {
+        if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_ON) {
+#if (T_NGX_SERVER_INFO)
+            ngx_str_set(&h->value, TENGINE_VER);
+#else
             ngx_str_set(&h->value, NGINX_VER);
+#endif
+        } else if (clcf->server_tokens == NGX_HTTP_SERVER_TOKENS_BUILD) {
+#if (T_NGX_SERVER_INFO)
+            ngx_str_set(&h->value, TENGINE_VER_BUILD);
+#else
+            ngx_str_set(&h->value, NGINX_VER_BUILD);
+#endif
         } else {
             ngx_str_set(&h->value, TENGINE);
         }

src/core/ngx_connection.h

@@ -264,6 +264,10 @@ struct ngx_connection_s {
 #if (T_NGX_HAVE_XUDP)
     unsigned            xudp_tx:1;
 #endif
+
+#if (T_NGX_XQUIC)
+    unsigned            xquic_conn:1;
+#endif
 };
 
 

src/event/modules/ngx_epoll_module.c

@@ -1009,7 +1009,7 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
 #if (NGX_SSL && NGX_SSL_ASYNC)
         aev = c->async;
 
-        if ((revents & EPOLLIN) && aev->active && async) {
+        if ((revents & EPOLLIN) && aev && aev->active && async) {
 
             if (c->async_fd == -1 || aev->instance!= instance) {
                 /*

src/event/ngx_event_openssl.c

@@ -3898,7 +3898,13 @@ ngx_ssl_shutdown(ngx_connection_t *c)
         return rc;
     }
 
+#if (T_NGX_XQUIC)
+	if (!c->xquic_conn) {
+		SSL_free(c->ssl->connection);
+	}
+#else
     SSL_free(c->ssl->connection);
+#endif
     c->ssl = NULL;
     c->recv = ngx_recv;
 

src/http/modules/ngx_http_grpc_module.c

@@ -5057,18 +5057,19 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ngx_http_grpc_loc_conf_t *glcf)
 
 #if (T_NGX_SSL_NTLS)
     glcf->upstream.tls_method = SSL_CTX_get_ssl_method(glcf->upstream.ssl->ctx);
-    if (glcf->enc_certificate.len) {
+    if (glcf->upstream.enc_certificate.len) {
 
-        if (glcf->enc_certificate_key.len == 0) {
+        if (glcf->upstream.enc_certificate_key.len == 0) {
             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
                           "no \"grpc_ssl_enc_certificate_key\" is defined "
-                          "for certificate \"%V\"", &glcf->enc_certificate);
+                          "for certificate \"%V\"",
+                          &glcf->upstream.enc_certificate);
             return NGX_ERROR;
         }
 
         if (ngx_ssl_certificate(cf, glcf->upstream.ssl,
-                                &glcf->upstream.enc_certificate->value,
-                                &glcf->upstream.enc_certificate_key->value,
+                                &glcf->upstream.enc_certificate,
+                                &glcf->upstream.enc_certificate_key,
                                 glcf->upstream.ssl_passwords,
                                 SSL_ENC_CERT)
             != NGX_OK)
@@ -5077,12 +5078,13 @@ ngx_http_grpc_set_ssl(ngx_conf_t *cf, ngx_http_grpc_loc_conf_t *glcf)
         }
     }
 
-    if (glcf->sign_certificate.len) {
+    if (glcf->upstream.sign_certificate.len) {
 
-        if (glcf->sign_certificate_key.len == 0) {
+        if (glcf->upstream.sign_certificate_key.len == 0) {
             ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
                           "no \"grpc_ssl_sign_certificate_key\" is defined "
-                          "for certificate \"%V\"", &glcf->sign_certificate);
+                          "for certificate \"%V\"",
+                          &glcf->upstream.sign_certificate);
             return NGX_ERROR;
         }