🔀 Merge pull request #445 from ruby/backport/v0.4-max_response_size

✨ Limit max_response_size (backport 0.4)

Author: nevans

lib/net/imap.rb

@@ -232,6 +232,10 @@ module Net
   #
   # Use paginated or limited versions of commands whenever possible.
   #
+  # Use Config#max_response_size to impose a limit on incoming server responses
+  # as they are being read.  <em>This is especially important for untrusted
+  # servers.</em>
+  #
   # Use #add_response_handler to handle responses after each one is received.
   # Use the +response_handlers+ argument to ::new to assign response handlers
   # before the receiver thread is started.  Use #extract_responses,
@@ -824,9 +828,17 @@ class << self
     # Seconds to wait until an IDLE response is received.
     # Delegates to {config.idle_response_timeout}[rdoc-ref:Config#idle_response_timeout].
 
+    ##
+    # :attr_accessor: max_response_size
+    #
+    # The maximum allowed server response size, in bytes.
+    # Delegates to {config.max_response_size}[rdoc-ref:Config#max_response_size].
+
     # :stopdoc:
     def open_timeout;           config.open_timeout            end
     def idle_response_timeout;  config.idle_response_timeout   end
+    def max_response_size;      config.max_response_size       end
+    def max_response_size=(val) config.max_response_size = val end
     # :startdoc:
 
     # The hostname this client connected to

lib/net/imap/config.rb

@@ -242,6 +242,41 @@ def self.[](config)
       #   Use +SASL-IR+ when it is supported by the server and the mechanism.
       attr_accessor :sasl_ir, type: :boolean
 
+      # The maximum allowed server response size.  When +nil+, there is no limit
+      # on response size.
+      #
+      # The default value (512 MiB, since +v0.5.7+) is <em>very high</em> and
+      # unlikely to be reached.  To use a lower limit, fetch message bodies in
+      # chunks rather than all at once.  A _much_ lower value should be used
+      # with untrusted servers (for example, when connecting to a user-provided
+      # hostname).
+      #
+      # <em>Please Note:</em> this only limits the size per response.  It does
+      # not prevent a flood of individual responses and it does not limit how
+      # many unhandled responses may be stored on the responses hash.  See
+      # Net::IMAP@Unbounded+memory+use.
+      #
+      # Socket reads are limited to the maximum remaining bytes for the current
+      # response: max_response_size minus the bytes that have already been read.
+      # When the limit is reached, or reading a +literal+ _would_ go over the
+      # limit, ResponseTooLargeError is raised and the connection is closed.
+      # See also #socket_read_limit.
+      #
+      # Note that changes will not take effect immediately, because the receiver
+      # thread may already be waiting for the next response using the previous
+      # value.  Net::IMAP#noop can force a response and enforce the new setting
+      # immediately.
+      #
+      # ==== Versioned Defaults
+      #
+      # Net::IMAP#max_response_size <em>was added in +v0.2.5+ and +v0.3.9+ as an
+      # attr_accessor, and in +v0.4.20+ and +v0.5.7+ as a delegator to this
+      # config attribute.</em>
+      #
+      # * original: +nil+ <em>(no limit)</em>
+      # * +0.5+: 512 MiB
+      attr_accessor :max_response_size, type: Integer?
+
       # Controls the behavior of Net::IMAP#responses when called without any
       # arguments (+type+ or +block+).
       #
@@ -419,6 +454,7 @@ def defaults_hash
         open_timeout: 30,
         idle_response_timeout: 5,
         sasl_ir: true,
+        max_response_size: nil,
         responses_without_block: :silence_deprecation_warning,
         parser_use_deprecated_uidplus_data: true,
         parser_max_deprecated_uidplus_data_size: 1000,
@@ -430,6 +466,7 @@ def defaults_hash
 
       version_defaults[0r] = Config[:default].dup.update(
         sasl_ir: false,
+        max_response_size: nil,
         parser_use_deprecated_uidplus_data: true,
         parser_max_deprecated_uidplus_data_size: 10_000,
       ).freeze
@@ -444,6 +481,7 @@ def defaults_hash
       ).freeze
 
       version_defaults[0.5r] = Config[0.4r].dup.update(
+        max_response_size: 512 << 20, # 512 MiB
         responses_without_block: :warn,
         parser_use_deprecated_uidplus_data: :up_to_max_size,
         parser_max_deprecated_uidplus_data_size: 100,

lib/net/imap/config/attr_type_coercion.rb

@@ -18,6 +18,8 @@ def attr_accessor(attr, type: nil)
             super(attr)
             AttrTypeCoercion.attr_accessor(attr, type: type)
           end
+
+          module_function def Integer?; NilOrInteger end
         end
         private_constant :Macros
 
@@ -46,6 +48,8 @@ def self.attr_accessor(attr, type: nil)
           define_method :"#{attr}?" do send attr end if type == Boolean
         end
 
+        NilOrInteger = safe{->val { Integer val unless val.nil? }}
+
         Enum = ->(*enum) {
           enum     = safe{enum}
           expected = -"one of #{enum.map(&:inspect).join(", ")}"

lib/net/imap/errors.rb

@@ -11,6 +11,39 @@ class Error < StandardError
     class DataFormatError < Error
     end
 
+    # Error raised when the socket cannot be read, due to a Config limit.
+    class ResponseReadError < Error
+    end
+
+    # Error raised when a response is larger than IMAP#max_response_size.
+    class ResponseTooLargeError < ResponseReadError
+      attr_reader :bytes_read, :literal_size
+      attr_reader :max_response_size
+
+      def initialize(msg = nil, *args,
+                     bytes_read:        nil,
+                     literal_size:      nil,
+                     max_response_size: nil,
+                     **kwargs)
+        @bytes_read        = bytes_read
+        @literal_size      = literal_size
+        @max_response_size = max_response_size
+        msg ||= [
+          "Response size", response_size_msg, "exceeds max_response_size",
+          max_response_size && "(#{max_response_size}B)",
+        ].compact.join(" ")
+        super(msg, *args, **kwargs)
+      end
+
+      private
+
+      def response_size_msg
+        if bytes_read && literal_size
+          "(#{bytes_read}B read + #{literal_size}B literal)"
+        end
+      end
+    end
+
     # Error raised when a response from the server is non-parsable.
     class ResponseParseError < Error
     end

lib/net/imap/response_reader.rb

@@ -28,19 +28,48 @@ def read_response_buffer
 
       attr_reader :buff, :literal_size
 
+      def bytes_read;       buff.bytesize                         end
+      def empty?;           buff.empty?                           end
+      def done?;            line_done? && !get_literal_size       end
+      def line_done?;       buff.end_with?(CRLF)                  end
       def get_literal_size; /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i end
 
       def read_line
-        buff << (@sock.gets(CRLF) or throw :eof)
+        buff << (@sock.gets(CRLF, read_limit) or throw :eof)
+        max_response_remaining! unless line_done?
       end
 
       def read_literal
+        # check before allocating memory for literal
+        max_response_remaining!
         literal = String.new(capacity: literal_size)
-        buff << (@sock.read(literal_size, literal) or throw :eof)
+        buff << (@sock.read(read_limit(literal_size), literal) or throw :eof)
       ensure
         @literal_size = nil
       end
 
+      def read_limit(limit = nil)
+        [limit, max_response_remaining!].compact.min
+      end
+
+      def max_response_size;      client.max_response_size                end
+      def max_response_remaining; max_response_size &.- bytes_read        end
+      def response_too_large?;    max_response_size &.< min_response_size end
+      def min_response_size;      bytes_read + min_response_remaining     end
+
+      def min_response_remaining
+        empty? ? 3 : done? ? 0 : (literal_size || 0) + 2
+      end
+
+      def max_response_remaining!
+        return max_response_remaining unless response_too_large?
+        raise ResponseTooLargeError.new(
+          max_response_size: max_response_size,
+          bytes_read:        bytes_read,
+          literal_size:      literal_size,
+        )
+      end
+
     end
   end
 end

test/net/imap/test_config.rb

@@ -428,4 +428,17 @@ def duck.to_r;   1/11111 end
     assert_same grandchild, greatgrandchild.parent
   end
 
+  test "#max_response_size=(Integer | nil)" do
+    config = Config.new
+
+    config.max_response_size = 10_000
+    assert_equal 10_000, config.max_response_size
+
+    config.max_response_size = nil
+    assert_nil config.max_response_size
+
+    assert_raise(ArgumentError) do config.max_response_size = "invalid" end
+    assert_raise(TypeError) do config.max_response_size = :invalid  end
+  end
+
 end

test/net/imap/test_errors.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "net/imap"
+require "test/unit"
+
+class IMAPErrorsTest < Test::Unit::TestCase
+
+  test "ResponseTooLargeError" do
+    err = Net::IMAP::ResponseTooLargeError.new
+    assert_nil err.bytes_read
+    assert_nil err.literal_size
+    assert_nil err.max_response_size
+
+    err = Net::IMAP::ResponseTooLargeError.new("manually set message")
+    assert_equal "manually set message", err.message
+    assert_nil err.bytes_read
+    assert_nil err.literal_size
+    assert_nil err.max_response_size
+
+    err = Net::IMAP::ResponseTooLargeError.new(max_response_size: 1024)
+    assert_equal "Response size exceeds max_response_size (1024B)", err.message
+    assert_nil err.bytes_read
+    assert_nil err.literal_size
+    assert_equal 1024, err.max_response_size
+
+    err = Net::IMAP::ResponseTooLargeError.new(bytes_read:        1200,
+                                               max_response_size: 1200)
+    assert_equal 1200, err.bytes_read
+    assert_equal "Response size exceeds max_response_size (1200B)", err.message
+
+    err = Net::IMAP::ResponseTooLargeError.new(bytes_read:        800,
+                                               literal_size:      1000,
+                                               max_response_size: 1200)
+    assert_equal  800, err.bytes_read
+    assert_equal 1000, err.literal_size
+    assert_equal("Response size (800B read + 1000B literal) " \
+                 "exceeds max_response_size (1200B)", err.message)
+  end
+
+end

test/net/imap/test_imap_max_response_size.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "net/imap"
+require "test/unit"
+require_relative "fake_server"
+
+class IMAPMaxResponseSizeTest < Test::Unit::TestCase
+  include Net::IMAP::FakeServer::TestHelper
+
+  def setup
+    Net::IMAP.config.reset
+    @do_not_reverse_lookup = Socket.do_not_reverse_lookup
+    Socket.do_not_reverse_lookup = true
+    @threads = []
+  end
+
+  def teardown
+    if !@threads.empty?
+      assert_join_threads(@threads)
+    end
+  ensure
+    Socket.do_not_reverse_lookup = @do_not_reverse_lookup
+  end
+
+  test "#max_response_size reading literals" do
+    with_fake_server(preauth: true) do |server, imap|
+      imap.max_response_size = 12_345 + 30
+      server.on("NOOP") do |resp|
+        resp.untagged("1 FETCH (BODY[] {12345}\r\n" + "a" * 12_345 + ")")
+        resp.done_ok
+      end
+      imap.noop
+      assert_equal "a" * 12_345, imap.responses("FETCH").first.message
+    end
+  end
+
+  test "#max_response_size closes connection for too long line" do
+    Net::IMAP.config.max_response_size = 10
+    run_fake_server_in_thread(preauth: false, ignore_io_error: true) do |server|
+      assert_raise_with_message(
+        Net::IMAP::ResponseTooLargeError, /exceeds max_response_size .*\b10B\b/
+      ) do
+        with_client("localhost", port: server.port) do
+          fail "should not get here (greeting longer than max_response_size)"
+        end
+      end
+    end
+  end
+
+  test "#max_response_size closes connection for too long literal" do
+    Net::IMAP.config.max_response_size = 1<<20
+    with_fake_server(preauth: false, ignore_io_error: true) do |server, client|
+      client.max_response_size = 50
+      server.on("NOOP") do |resp|
+        resp.untagged("1 FETCH (BODY[] {1000}\r\n" + "a" * 1000 + ")")
+      end
+      assert_raise_with_message(
+        Net::IMAP::ResponseTooLargeError,
+        /\d+B read \+ 1000B literal.* exceeds max_response_size .*\b50B\b/
+      ) do
+        client.noop
+        fail "should not get here (FETCH literal longer than max_response_size)"
+      end
+    end
+  end
+
+end

test/net/imap/test_response_reader.rb

@@ -11,6 +11,7 @@ def setup
 
   class FakeClient
     def config; @config ||= Net::IMAP.config.new end
+    def max_response_size; config.max_response_size end
   end
 
   def literal(str) "{#{str.bytesize}}\r\n#{str}" end
@@ -49,4 +50,19 @@ def literal(str) "{#{str.bytesize}}\r\n#{str}" end
     assert_equal "",           rcvr.read_response_buffer.to_str
   end
 
+  test "#read_response_buffer with max_response_size" do
+    client = FakeClient.new
+    client.config.max_response_size = 10
+    under = "+ 3456\r\n"
+    exact = "+ 345678\r\n"
+    over  = "+ 3456789\r\n"
+    io = StringIO.new([under, exact, over].join)
+    rcvr = Net::IMAP::ResponseReader.new(client, io)
+    assert_equal under, rcvr.read_response_buffer.to_str
+    assert_equal exact, rcvr.read_response_buffer.to_str
+    assert_raise Net::IMAP::ResponseTooLargeError do
+      rcvr.read_response_buffer
+    end
+  end
+
 end