support encoding introduced since Ruby 1.9.

joe1chen · Jul 5, 2010 · 61398ec · 61398ec
1 parent 2653c68
commit 61398ec
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 1 deletion.
diff --git a/lib/openid/fetchers.rb b/lib/openid/fetchers.rb
@@ -205,6 +205,7 @@ def fetch(url, body=nil, headers=nil, redirect_limit=REDIRECT_LIMIT)
             conn.request_post(url.request_uri, body, headers)
           end
         }
+        setup_encoding(response)
       rescue Timeout::Error => why
         raise FetchingError, "Error fetching #{url}: #{why}"
       rescue RuntimeError => why
@@ -234,5 +235,24 @@ def fetch(url, body=nil, headers=nil, redirect_limit=REDIRECT_LIMIT)
         return HTTPResponse._from_net_response(response, unparsed_url)
       end
     end
+
+    private
+    def setup_encoding(response)
+      return unless defined?(::Encoding::ASCII_8BIT)
+      charset = response.type_params["charset"]
+      return if charset.nil?
+      encoding = nil
+      begin
+        encoding = Encoding.find(charset)
+      rescue ArgumentError
+      end
+      encoding ||= Encoding::ASCII_8BIT
+      body = response.body
+      if body.respond_to?(:force_encoding)
+        body.force_encoding(encoding)
+      else
+        body.set_encoding(encoding)
+      end
+    end
   end
 end
diff --git a/test/test_fetchers.rb b/test/test_fetchers.rb
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 require 'test/unit'
 require 'net/http'
 require 'webrick'
@@ -112,7 +114,22 @@ def _redirect_loop
       assert_block("Fetched too many times.") { @_redirect_counter < 10 }
     }
   end
-
+
+  UTF8_PAGE_CONTENT = <<-EOHTML
+<html>
+  <head><title>UTF-8</title></head>
+  <body>こんにちは</body>
+</html>
+EOHTML
+  def _utf8_page
+    lambda { |req, resp|
+      resp['Content-Type'] = "text/html; charset=utf-8"
+      body = UTF8_PAGE_CONTENT.dup
+      body.force_encoding("ASCII-8BIT") if body.respond_to?(:force_encoding)
+      resp.body = body
+    }
+  end
+
   def setup
     @fetcher = OpenID::StandardFetcher.new
     @logfile = StringIO.new
@@ -138,6 +155,7 @@ def setup
       }
       @server.mount_proc('/post', _require_post)
       @server.mount_proc('/redirect_loop', _redirect_loop)
+      @server.mount_proc('/utf8_page', _utf8_page)
       @server.start
     }
     @uri = _uri_build
@@ -211,6 +229,15 @@ def test_redirect_limit
     }
   end
 
+  def test_utf8_page
+    uri = _uri_build('/utf8_page')
+    response = @fetcher.fetch(uri)
+    assert_equal(UTF8_PAGE_CONTENT, response.body)
+    if response.body.respond_to?(:encoding)
+      assert_equal(Encoding::UTF_8, response.body.encoding)
+    end
+  end
+
   def test_cases
     for path, expected_code, expected_url in @@cases
       uri = _uri_build(path)