Ensure match prevents integer overflow

Integers can either be Fixnum or Bignum internally (and the exact
integer at which the types change varies by platform) so support both by
using `RB_INTEGER_TYPE_P` to test values and `NUM2INT` to convert them
to C `int` (which also handles raising `RangeError` for invalid values)
instead of always assuming Fixnum integers.

Dynamically calculate the C's `INT_MAX` using Ruby's
`RbConfig::SIZEOF` hash of C type byte sizes, see
https://docs.ruby-lang.org/en/master/RbConfig.html#SIZEOF rather than
hard-coding the value to 2147483647.

Author: mudge

ext/re2/re2.cc

@@ -417,8 +417,8 @@ static re2::StringPiece *re2_matchdata_find_match(VALUE idx, const VALUE self) {
 
   int id;
 
-  if (FIXNUM_P(idx)) {
-    id = FIX2INT(idx);
+  if (RB_INTEGER_TYPE_P(idx)) {
+    id = NUM2INT(idx);
   } else if (SYMBOL_P(idx)) {
     const std::map<std::string, int>& groups = p->pattern->NamedCapturingGroups();
     std::map<std::string, int>::const_iterator search = groups.find(rb_id2name(SYM2ID(idx)));
@@ -686,10 +686,10 @@ static VALUE re2_matchdata_aref(int argc, VALUE *argv, const VALUE self) {
         std::string(RSTRING_PTR(idx), RSTRING_LEN(idx)), self);
   } else if (SYMBOL_P(idx)) {
     return re2_matchdata_named_match(rb_id2name(SYM2ID(idx)), self);
-  } else if (!NIL_P(rest) || !FIXNUM_P(idx) || FIX2INT(idx) < 0) {
+  } else if (!NIL_P(rest) || !RB_INTEGER_TYPE_P(idx) || NUM2INT(idx) < 0) {
     return rb_ary_aref(argc, argv, re2_matchdata_to_a(self));
   } else {
-    return re2_matchdata_nth_match(FIX2INT(idx), self);
+    return re2_matchdata_nth_match(NUM2INT(idx), self);
   }
 }
 
@@ -1426,7 +1426,7 @@ static VALUE re2_regexp_match(int argc, VALUE *argv, const VALUE self) {
   RE2::Anchor anchor = RE2::UNANCHORED;
 
   if (RTEST(options)) {
-    if (FIXNUM_P(options)) {
+    if (RB_INTEGER_TYPE_P(options)) {
       n = NUM2INT(options);
 
       if (n < 0) {
@@ -1440,8 +1440,6 @@ static VALUE re2_regexp_match(int argc, VALUE *argv, const VALUE self) {
       VALUE endpos_option = rb_hash_aref(options, ID2SYM(id_endpos));
       if (!NIL_P(endpos_option)) {
 #ifdef HAVE_ENDPOS_ARGUMENT
-        Check_Type(endpos_option, T_FIXNUM);
-
         endpos = NUM2INT(endpos_option);
 
         if (endpos < 0) {
@@ -1470,8 +1468,6 @@ static VALUE re2_regexp_match(int argc, VALUE *argv, const VALUE self) {
 
       VALUE submatches_option = rb_hash_aref(options, ID2SYM(id_submatches));
       if (!NIL_P(submatches_option)) {
-        Check_Type(submatches_option, T_FIXNUM);
-
         n = NUM2INT(submatches_option);
 
         if (n < 0) {
@@ -1487,8 +1483,6 @@ static VALUE re2_regexp_match(int argc, VALUE *argv, const VALUE self) {
 
       VALUE startpos_option = rb_hash_aref(options, ID2SYM(id_startpos));
       if (!NIL_P(startpos_option)) {
-        Check_Type(startpos_option, T_FIXNUM);
-
         startpos = NUM2INT(startpos_option);
 
         if (startpos < 0) {
@@ -1520,6 +1514,10 @@ static VALUE re2_regexp_match(int argc, VALUE *argv, const VALUE self) {
 #endif
     return BOOL2RUBY(matched);
   } else {
+    if (n == INT_MAX) {
+      rb_raise(rb_eRangeError, "number of matches should be < %d", INT_MAX);
+    }
+
     /* Because match returns the whole match as well. */
     n += 1;
 

spec/re2/regexp_spec.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
+require "rbconfig/sizeof"
+
 RSpec.describe RE2::Regexp do
+  INT_MAX = 2**(RbConfig::SIZEOF.fetch("int") * 8 - 1) - 1
+
   describe "#initialize" do
     it "returns an instance given only a pattern" do
       re = RE2::Regexp.new('woo')
@@ -566,6 +570,12 @@
       expect { re.match("one two three", submatches: :invalid) }.to raise_error(TypeError)
     end
 
+    it "raises an exception when given too large a number of submatches" do
+      re = RE2::Regexp.new('(\w+) (\w+) (\w+)')
+
+      expect { re.match("one two three", submatches: INT_MAX) }.to raise_error(RangeError, "number of matches should be < #{INT_MAX}")
+    end
+
     it "defaults to extracting all submatches when given nil", :aggregate_failures do
       re = RE2::Regexp.new('(\w+) (\w+) (\w+)')
       md = re.match("one two three", submatches: nil)
@@ -584,6 +594,13 @@
       expect(md[3]).to be_nil
     end
 
+    it "raises an exception if given too large a number of submatches instead of options" do
+      re = RE2::Regexp.new('(\w+) (\w+) (\w+)')
+      md = re.match("one two three", 2)
+
+      expect { re.match("one two three", INT_MAX) }.to raise_error(RangeError, "number of matches should be < #{INT_MAX}")
+    end
+
     it "raises an exception when given invalid options" do
       re = RE2::Regexp.new('(\w+) (\w+) (\w+)')