detect: guard rate filter callback registration, return bool on failure

examples/lib/custom/main.c

@@ -253,7 +253,13 @@ int main(int argc, char **argv)
      * ThreadVars will be ready. */
     SuricataInit();
 
-    SCDetectEngineRegisterRateFilterCallback(RateFilterCallback, NULL);
+    if (DetectEngineEnabled()) {
+        if (!SCDetectEngineRegisterRateFilterCallback(RateFilterCallback, NULL)) {
+            SCLogWarning("rate filter callback registration failed");
+        }
+    } else {
+        SCLogWarning("detection engine not enabled, rate filter callback not registered");
+    }
 
     /* Spawn our worker threads. */
     pthread_t worker;

examples/lib/live/main.c

@@ -306,7 +306,13 @@ int main(int argc, char **argv)
      * ThreadVars will be ready. */
     SuricataInit();
 
-    SCDetectEngineRegisterRateFilterCallback(RateFilterCallback, NULL);
+    if (DetectEngineEnabled()) {
+        if (!SCDetectEngineRegisterRateFilterCallback(RateFilterCallback, NULL)) {
+            SCLogWarning("rate filter callback registration failed");
+        }
+    } else {
+        SCLogWarning("detection engine not enabled, rate filter callback not registered");
+    }
 
     /* Spawn our worker threads, one for each interface. */
     pthread_t workers[MAX_INTERFACES];

src/detect-engine.c

@@ -5203,12 +5203,17 @@ void DetectLowerSetupCallback(
     }
 }
 
-void SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc fn, void *arg)
+bool SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc fn, void *arg)
 {
     DetectEngineCtx *de_ctx = DetectEngineGetCurrent();
+    if (de_ctx == NULL) {
+        SCLogError("no detection engine available for rate filter callback registration");
+        return false;
+    }
     de_ctx->RateFilterCallback = fn;
     de_ctx->rate_filter_callback_arg = arg;
     DetectEngineDeReference(&de_ctx);
+    return true;
 }
 
 int DetectEngineThreadCtxGetJsonContext(DetectEngineThreadCtx *det_ctx)

src/detect.h

@@ -1214,7 +1214,7 @@ typedef struct DetectEngineCtx_ {
  * This callback is added to the current detection engine and will be
  * copied to all future detection engines over rule reloads.
  */
-void SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc cb, void *arg);
+bool SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc cb, void *arg);
 
 /* Engine groups profiles (low, medium, high, custom) */
 enum {