Avoid excessive backtracking in regex engine during fuzzing

The regex engine is prone to excessive backtracking, leading to
timeouts, especially while fuzzing.
This commit introduces a backtracking counter and a limit of 1000
backtracking steps. When this limit is exceeded during fuzzing, the
regex engine aborts to prevent excessive backtracking. For this, the
FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION macro is used, as per
suggested by the documentation of libFuzzer.

Makefile

@@ -154,6 +154,7 @@ CFLAGS_DEBUG=$(CFLAGS) -O0
 CFLAGS_SMALL=$(CFLAGS) -Os
 CFLAGS_OPT=$(CFLAGS) -O2
 CFLAGS_NOLTO:=$(CFLAGS_OPT)
+CFLAGS_FUZZ=$(CFLAGS_OPT) -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1
 ifdef CONFIG_COSMO
 LDFLAGS+=-s # better to strip by default
 else
@@ -255,13 +256,13 @@ qjsc$(EXE): $(OBJDIR)/qjsc.o $(QJS_LIB_OBJS)
 	$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
 
 fuzz_eval: $(OBJDIR)/fuzz_eval.o $(OBJDIR)/fuzz_common.o libquickjs.fuzz.a
-	$(CC) $(CFLAGS_OPT) $^ -o fuzz_eval $(LIB_FUZZING_ENGINE)
+	$(CC) $(CFLAGS_FUZZ) $^ -o fuzz_eval $(LIB_FUZZING_ENGINE)
 
 fuzz_compile: $(OBJDIR)/fuzz_compile.o $(OBJDIR)/fuzz_common.o libquickjs.fuzz.a
-	$(CC) $(CFLAGS_OPT) $^ -o fuzz_compile $(LIB_FUZZING_ENGINE)
+	$(CC) $(CFLAGS_FUZZ) $^ -o fuzz_compile $(LIB_FUZZING_ENGINE)
 
 fuzz_regexp: $(OBJDIR)/fuzz_regexp.o $(OBJDIR)/libregexp.fuzz.o $(OBJDIR)/cutils.fuzz.o $(OBJDIR)/libunicode.fuzz.o
-	$(CC) $(CFLAGS_OPT) $^ -o fuzz_regexp $(LIB_FUZZING_ENGINE)
+	$(CC) $(CFLAGS_FUZZ) $^ -o fuzz_regexp $(LIB_FUZZING_ENGINE)
 
 libfuzzer: fuzz_eval fuzz_compile fuzz_regexp
 
@@ -338,7 +339,7 @@ $(OBJDIR)/%.o: %.c | $(OBJDIR)
 	$(CC) $(CFLAGS_OPT) -c -o $@ $<
 
 $(OBJDIR)/fuzz_%.o: fuzz/fuzz_%.c | $(OBJDIR)
-	$(CC) $(CFLAGS_OPT) -c -I. -o $@ $<
+	$(CC) $(CFLAGS_FUZZ) -c -I. -o $@ $<
 
 $(OBJDIR)/%.host.o: %.c | $(OBJDIR)
 	$(HOST_CC) $(CFLAGS_OPT) -c -o $@ $<
@@ -359,7 +360,7 @@ $(OBJDIR)/%.debug.o: %.c | $(OBJDIR)
 	$(CC) $(CFLAGS_DEBUG) -c -o $@ $<
 
 $(OBJDIR)/%.fuzz.o: %.c | $(OBJDIR)
-	$(CC) $(CFLAGS_OPT) -fsanitize=fuzzer-no-link -c -o $@ $<
+	$(CC) $(CFLAGS_FUZZ) -fsanitize=fuzzer-no-link -c -o $@ $<
 
 $(OBJDIR)/%.check.o: %.c | $(OBJDIR)
 	$(CC) $(CFLAGS) -DCONFIG_CHECK_JSVALUE -c -o $@ $<

libregexp.c

@@ -1927,6 +1927,9 @@ typedef struct {
     /* 0 = 8 bit chars, 1 = 16 bit chars, 2 = 16 bit chars, UTF-16 */
     int cbuf_type;
     int capture_count;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    int backtrack_count;
+#endif
     int stack_size_max;
     BOOL multi_line;
     BOOL ignore_case;
@@ -1993,6 +1996,12 @@ static intptr_t lre_exec_backtrack(REExecContext *s, uint8_t **capture,
 
     for(;;) {
         //        printf("top=%p: pc=%d\n", th_list.top, (int)(pc - (bc_buf + RE_HEADER_LEN)));
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+        if (++s->backtrack_count > 1000) {
+            return -1; // backtracking limit exceeded
+        }
+#endif
+
         opcode = *pc++;
         switch(opcode) {
         case REOP_match:
@@ -2399,6 +2408,9 @@ int lre_exec(uint8_t **capture,
     s->ignore_case = (re_flags & LRE_FLAG_IGNORECASE) != 0;
     s->is_unicode = (re_flags & LRE_FLAG_UNICODE) != 0;
     s->capture_count = bc_buf[RE_HEADER_CAPTURE_COUNT];
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    s->backtrack_count = 0;
+#endif
     s->stack_size_max = bc_buf[RE_HEADER_STACK_SIZE];
     s->cbuf = cbuf;
     s->cbuf_end = cbuf + (clen << cbuf_type);