Initial support to proxy request with Transfer-Encoding: chunked

Author: tkan145

gateway/src/apicast/http_proxy.lua

@@ -7,6 +7,10 @@ local http_proxy = require 'resty.http.proxy'
 local file_reader = require("resty.file").file_reader
 local chunked_reader = require('resty.http.chunked').chunked_reader
 local chunked_writer = require('resty.http.chunked').chunked_writer
+local req_headers = ngx.req.get_headers
+local http_ver = ngx.req.http_version
+local ngx_send_headers = ngx.send_headers
+local ngx_flush = ngx.flush
 
 local _M = { }
 
@@ -84,20 +88,55 @@ local function absolute_url(uri)
 end
 
 local function forward_https_request(proxy_uri, uri, skip_https_connect)
-    local sock, err
+    local sock, ok, err
     local chunksize = 32 * 1024
     local body
+    local headers = req_headers()
+
+    local encoding = headers.transfer_encoding
+    if type(encoding) == "table" then
+        encoding = encoding[1]
+    end
+
+    if encoding and encoding:lower() == "chunked" then
+        if http_ver() ~= 1.1 then
+            ngx.log(ngx.ERR, "bad http version")
+            ngx.exit(ngx.HTTP_BAD_REQUEST)
+         end
+         local expect = headers["Expect"]
+
+        if type(expect) == "table" then
+            expect = expect[1]
+        end
+
+        if expect and expect:lower() == "100-continue" then
+            ngx.status = 100
+            ok, err = ngx_send_headers()
+
+            if not ok then
+                ngx.log(ngx.ERR, "failed to send response header: " .. (err or "unknown"))
+            end
+
+            ok, err = ngx_flush(true)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to flush response header: " .. (err or "unknown"))
+            end
+        end
 
-    if ngx.req.get_headers()["Transfer-Encoding"] == "chunked" then
         -- The default ngx reader does not support chunked request
         -- so we will need to get the raw request socket and manually
         -- decode the chunked request
         sock, err = ngx.req.socket(true)
 
         if not sock then
-            return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)
+            if err == "no body" then
+                body = nil
+            else
+                return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)
+            end
+        else
+            body = chunked_reader(sock, chunksize)
         end
-        body = chunked_reader(sock, chunksize)
     else
         -- This is needed to call ngx.req.get_body_data() below.
         ngx.req.read_body()
@@ -157,7 +196,11 @@ local function forward_https_request(proxy_uri, uri, skip_https_connect)
         httpc:set_keepalive()
     else
         ngx.log(ngx.ERR, 'failed to proxy request to: ', proxy_uri, ' err : ', err)
-        return ngx.exit(ngx.HTTP_BAD_GATEWAY)
+        if sock then
+            sock:send("HTTP/1.1 502 Bad Gateway\\r\\n")
+        else
+            return ngx.exit(ngx.HTTP_BAD_GATEWAY)
+        end
     end
 end
 

gateway/src/resty/http/chunked.lua

@@ -21,7 +21,10 @@ local function print_err(error_code, ...)
     return ngx.exit(error_code)
 end
 
--- TODO: support Trailers
+-- This is a copy of lua-resty-http _chunked_body_reader function but with
+-- extra bits to make chunked encoding work with raw socket
+-- https://github.com/ledgetech/lua-resty-http/blob/v0.16.1/lib/resty/http.lua#L418
+
 -- chunked_reader return a body reader that translates the data read from sock
 -- out of HTTP "chunked" format before returning it
 --
@@ -54,7 +57,7 @@ function _M.chunked_reader(sock, max_chunk_size)
                 --  chunk-size CRLF
                 local line, err = sock:receive()
                 if not line then
-                    co_yield(nil, "chunked_reader: failed to receive chunk size, err: " .. err)
+                    co_yield(nil, "chunked_reader: failed to receive chunk size, err: " .. (err or "unknown"))
                 end
 
                 size = tonumber(line, 16)
@@ -74,29 +77,31 @@ function _M.chunked_reader(sock, max_chunk_size)
                 -- Receive the chunk
                 local chunk, err = sock:receive(size)
                 if not chunk then
-                    co_yield(nil, "chunked_reader: failed to receive chunk of size " .. size .. " err: " .. err)
+                    co_yield(nil, "chunked_reader: failed to receive chunk of size " .. size .. " err: " .. (err or "unknown"))
                 end
 
-                -- We're at the end of a chunk, read the next  two bytes
-                -- and verify they are  "\r\n"
-                local data, err = sock:receive(2)
-                if not data then
-                    co_yield(nil, "chunked_reader: failed to receive chunk terminator, err: " .. err)
+                if remaining == 0 then
+                    -- We're at the end of a chunk, read the next  two bytes
+                    -- and verify they are  "\r\n"
+                    local data, err = sock:receive(2)
+                    if not data then
+                        co_yield(nil, "chunked_reader: failed to receive chunk terminator, err: " .. (err or "unknown"))
+                    end
                 end
 
-                chunk = string.format("%x\r\n", size) .. chunk .. data
+                chunk = string.format("%x\r\n", size) .. chunk .. cr_lf
 
                 co_yield(chunk)
             else
                 -- we're at the end of a chunk, read the next two
                 -- bytes to verify they are "\r\n".
                 local chunk, err = sock:receive(2)
                 if not chunk then
-                    co_yield(nil, "chunked_reader: failed to receive chunk terminator, err: " .. err)
+                    co_yield(nil, "chunked_reader: failed to receive chunk terminator, err: " .. (err or "unknown"))
                 end
 
                 if chunk ~= "\r\n" then
-                    co_yield(nil, "chunked_reader: bad chunk terminator: " .. chunk)
+                    co_yield(nil, "chunked_reader: bad chunk terminator")
                 end
 
                 eof = true
@@ -118,22 +123,22 @@ function _M.chunked_writer(sock, res, chunksize)
     local status  = "HTTP/1.1 " .. res.status .. " " .. res.reason .. cr_lf
     bytes, err = send(sock, status)
     if not bytes then
-        print_err(503, "chunked_writer: failed to send status line, err: ", err)
+        print_err(503, "chunked_writer: failed to send status line, err: " .. (err or "unknown"))
     end
 
     -- Rest of header
     for k, v in pairs(res.headers) do
         local header = k .. ": " .. v .. cr_lf
         bytes, err = send(sock, header)
         if not bytes then
-            print_err(503, "chunked_writer: failed to send header, err: ", err)
+            print_err(503, "chunked_writer: failed to send header, err: " .. (err or "unknown"))
         end
     end
 
     -- End-of-header
     bytes, err = send(sock, cr_lf)
     if not bytes then
-        print_err(503, "chunked_writer: failed to send end of header, err: ", err)
+        print_err(503, "chunked_writer: failed to send end of header, err: " .. (err or "unknown"))
     end
 
     -- Write body and trailer
@@ -145,13 +150,13 @@ function _M.chunked_writer(sock, res, chunksize)
 
             chunk, read_err = reader(chunksize)
             if read_err then
-                print_err(503, "chunked_writer: failed to read body, err: ", err)
+                print_err(503, "chunked_writer: failed to read body, err: " .. (err or "unknown"))
             end
 
             if chunk then
                 bytes, err = send(sock, chunk)
                 if not bytes then
-                    print_err("chunked_writer: failed to send body, err: ", err)
+                    print_err(503, "chunked_writer: failed to send body, err: " .. (err or "unknown"))
                 end
             end
         until not chunk