Merge pull request #1167 from ruby/test_skip

Add TestSkip module to skip tests that fails in ruby CI

Author: soutaro

test/rbs/cli_test.rb

@@ -45,10 +45,6 @@ def with_cli
     @stderr = nil
   end
 
-  def bundler?
-    ENV.key?("BUNDLE_GEMFILE")
-  end
-
   def test_ast
     with_cli do |cli|
       cli.run(%w(-r set ast))
@@ -542,8 +538,6 @@ def foo: () -> void
   end
 
   def test_collection_install
-    omit unless bundler?
-
     Dir.mktmpdir do |dir|
       Dir.chdir(dir) do
         dir = Pathname(dir)
@@ -597,8 +591,6 @@ def test_collection_install
   end
 
   def test_collection_install_frozen
-    omit unless bundler?
-
     Dir.mktmpdir do |dir|
       Dir.chdir(dir) do
         dir = Pathname(dir)
@@ -630,8 +622,6 @@ def test_collection_install_frozen
   end
 
   def test_collection_update
-    omit unless bundler?
-
     Dir.mktmpdir do |dir|
       Dir.chdir(dir) do
         dir = Pathname(dir)

test/stdlib/Net_HTTP_test.rb

@@ -9,8 +9,6 @@ class NetSingletonTest < Test::Unit::TestCase
   testing "singleton(::Net::HTTP)"
 
   def test_get
-    omit if ci?
-
     $stdout = StringIO.new
     assert_send_type "(URI::Generic) -> nil",
                      Net::HTTP, :get_print, URI("https://www.ruby-lang.org")
@@ -35,17 +33,13 @@ def test_get
   end
 
   def test_post
-    omit if ci?
-
     assert_send_type "(URI, String, Hash[String, String]) -> Net::HTTPResponse",
                      Net::HTTP, :post, URI('http://www.example.com/api/search'), { "q" => "ruby", "max" => "50" }.to_json, "Content-Type" => "application/json"
     assert_send_type "(URI, Hash[String, Symbol]) -> Net::HTTPResponse",
                      Net::HTTP, :post_form, URI('http://www.example.com/api/search'), { "q" => :ruby, "max" => :max }
   end
 
   def test_new
-    omit if ci?
-
     assert_send_type "(String, Integer, nil, nil, nil, nil, nil) -> Net::HTTP",
                      Net::HTTP, :new, 'www.ruby-lang.org', 80, nil, nil, nil, nil, nil
   end
@@ -64,114 +58,92 @@ def self.new
   end
 
   def test_inspect
-    omit if ci?
-
     assert_send_type "() -> String",
                      TestNet.new, :inspect
   end
 
   def test_set_debug_output
-    omit if ci?
     assert_send_type "(IO) -> void",
                      TestNet.new, :set_debug_output, $stderr
   end
 
   def test_address
-    omit if ci?
     assert_send_type "() -> String",
                      TestNet.new, :address
   end
 
   def test_port
-    omit if ci?
     assert_send_type "() -> Integer",
                      TestNet.new, :port
   end
 
   def test_ipaddr
-    omit if ci?
     assert_send_type "() -> nil",
                      TestNet.new, :ipaddr
     assert_send_type "(String) -> void",
                      TestNet.new, :ipaddr=, ('127.0.0.1')
   end
 
   def test_open_timeout
-    omit if ci?
     assert_send_type "() -> Integer",
                      TestNet.new, :open_timeout
   end
 
   def test_read_timeout
-    omit if ci?
     assert_send_type "() -> Integer",
                      TestNet.new, :read_timeout
     assert_send_type "(Integer) -> void",
                      TestNet.new, :read_timeout=, 10
   end
 
   def test_write_timeout
-    omit if ci?
     assert_send_type "() -> Integer",
                      TestNet.new, :write_timeout
     assert_send_type "(Integer) -> void",
                      TestNet.new, :write_timeout=, 10
   end
 
   def test_continue_timeout
-    omit if ci?
     assert_send_type "() -> nil",
                      TestNet.new, :continue_timeout
     assert_send_type "(Integer) -> void",
                      TestNet.new, :continue_timeout=, 10
   end
 
   def test_max_retries
-    omit if ci?
-
     assert_send_type "() -> Integer",
                      TestNet.new, :max_retries
     assert_send_type "(Integer) -> void",
                      TestNet.new, :max_retries=, 10
   end
 
   def test_keep_alive_timeout
-    omit if ci?
-
     assert_send_type "() -> Integer",
                      TestNet.new, :keep_alive_timeout
   end
 
   def test_started_?
-    omit if ci?
-
     assert_send_type "() -> bool",
                      TestNet.new, :started?
     assert_send_type "() -> bool",
                      TestNet.new, :active?
   end
 
   def test_use_ssl
-    omit if ci?
-
     assert_send_type "() -> bool",
                      TestNet.new, :use_ssl?
     assert_send_type "(bool) -> void",
                      TestNet.new, :use_ssl=, true
   end
 
   def test_start
-    omit if ci?
-
     assert_send_type "() { (Net::HTTP) -> untyped } -> untyped",
                      TestNet.new, :start do |net_http| net_http.class end
     assert_send_type "() -> Net::HTTP",
                      TestNet.new, :start
   end
 
   def test_proxy
-    omit if ci?
-
     assert_send_type "() -> bool",
                      TestNet.new, :proxy?
     assert_send_type "() -> bool",
@@ -193,8 +165,6 @@ def test_proxy
   end
 
   def test_http_verbs
-    omit if ci?
-
     assert_send_type "(String) -> Net::HTTPResponse",
                      Net::HTTP.start('www.ruby-lang.org', 443, use_ssl: true), :get, '/en'
     assert_send_type "(String, Hash[String, String]) -> Net::HTTPResponse",
@@ -326,8 +296,6 @@ def test_http_verbs
   end
 
   def test_request
-    omit if ci?
-
     assert_send_type "(String, String) -> Net::HTTPResponse",
                      Net::HTTP.start('reqres.in', 443, use_ssl: true), :send_request, 'GET', 'api/users'
     assert_send_type "(String, String, String, Hash[String, String]) -> Net::HTTPResponse",
@@ -346,15 +314,11 @@ class TestHTTPRequest < Test::Unit::TestCase
   testing "::Net::HTTPRequest"
 
   def test_inspect
-    omit if ci?
-
     assert_send_type "() -> String",
                      Net::HTTP::Get.new(URI('https://www.ruby-lang.org')), :inspect
   end
 
   def test_attr_readers
-    omit if ci?
-
     assert_send_type "() -> String",
                      Net::HTTP::Get.new(URI('https://www.ruby-lang.org')), :method
     assert_send_type "() -> String",
@@ -366,8 +330,6 @@ def test_attr_readers
   end
 
   def test_body
-    omit if ci?
-
     assert_send_type "() -> bool",
                      Net::HTTP::Get.new(URI('https://www.ruby-lang.org')), :request_body_permitted?
     assert_send_type "() -> bool",
@@ -385,8 +347,6 @@ def test_body
   end
 
   def test_manipulation_of_headers
-    omit if ci?
-
     assert_send_type "(String) -> nil",
                      Net::HTTP::Get.new(URI('https://www.ruby-lang.org')), :[], "Content-Type"
     assert_send_type "(String, untyped) -> void",
@@ -460,8 +420,6 @@ def test_manipulation_of_headers
   end
 
   def test_iteration_on_headers
-    omit if ci?
-
     assert_send_type "() { (String, String) -> untyped } -> Hash[String, Array[String]]",
                      Net::HTTP::Get.new(URI('https://www.ruby-lang.org')), :each_header do |str, array| "#{str} #{array}" end
     assert_send_type "() -> Enumerator[[String, String], Hash[String, Array[String]]]",
@@ -520,8 +478,6 @@ def self.success
   end
 
   def test_attr_readers
-    omit if ci?
-
     assert_send_type "() -> String",
                      Foo.success, :http_version
     assert_send_type "() -> String",
@@ -537,8 +493,6 @@ def test_attr_readers
   end
 
   def test_manipulation_function
-    omit if ci?
-
     assert_send_type "() -> String",
                      Foo.success, :inspect
     assert_send_type "() -> untyped",

test/stdlib/test_helper.rb

@@ -4,6 +4,11 @@
 require "tmpdir"
 require "stringio"
 require "tempfile"
+require_relative "../test_skip"
+
+class Test::Unit::TestCase
+  prepend TestSkip
+end
 
 module Spy
   def self.wrap(object, method_name)
@@ -375,10 +380,6 @@ def method_types(method)
     end
   end
 
-  def ci?
-    ENV["CI"] == "true"
-  end
-
   def allows_error(*errors)
     yield
   rescue *errors => exn

test/test_helper.rb

@@ -4,6 +4,7 @@
 require "tmpdir"
 require "stringio"
 require "open3"
+require "test_skip"
 
 unless ENV["XDG_CACHE_HOME"]
   tmpdir = Dir.mktmpdir("rbs-test-")
@@ -22,6 +23,9 @@
 rescue LoadError
 end
 
+class Test::Unit::TestCase
+  prepend TestSkip
+end
 
 module TestHelper
   def has_gem?(*gems)

test/test_skip.rb

@@ -0,0 +1,52 @@
+# Including this module allows `omit` test cases based on an external file, not in the source code
+#
+# The file contains a list of the names of the test cases to skip:
+#
+# ```
+# test_foo(RBS::UnitTest)            # Test case name
+# RBS::CLICtest                      # Test class name
+# test_collection_install(CLITest) requires bundler setup   # Can have comments
+# ```
+#
+# And start tests with `$RBS_SKIP_TESTS` env var:
+#
+# ```
+# $ RBS_SKIP_TESTS=../rbs_skip_tests rake test
+# ```
+#
+module TestSkip
+  env = ENV["RBS_SKIP_TESTS"]
+  SKIP_TESTS_FILE =
+    if env
+      Pathname(env)
+    end
+
+  SKIP_TESTS =
+    if SKIP_TESTS_FILE
+      SKIP_TESTS_FILE.each_line.with_object({}) do |line, hash|
+        line.chomp!
+        line.gsub!(/#.*/, "")
+        line.strip!
+
+        next if line.empty?
+
+        name, message = line.split(/\s+/, 2)
+
+        hash[name] = message
+      end
+    end
+
+  if SKIP_TESTS
+    def setup
+      super
+
+      if SKIP_TESTS.key?(name) || SKIP_TESTS.key?(self.class.name)
+        if message = SKIP_TESTS[name] || SKIP_TESTS[self.class.name]
+          omit "Skip test by RBS_SKIP_TESTS(#{SKIP_TESTS_FILE}): #{message}"
+        else
+          omit "Skip test by RBS_SKIP_TESTS(#{SKIP_TESTS_FILE})"
+        end
+      end
+    end
+  end
+end