detect: guard rate filter callback registration, return bool on failure

SCDetectEngineRegisterRateFilterCallback() dereferences the result of
DetectEngineGetCurrent() unconditionally. Add a NULL guard with
SCLogError and return false so callers can detect registration failure.

Flagged by Svace and confirmed by gcc -fanalyzer.

Ticket: 8560

src/detect-engine.c

@@ -5203,12 +5203,17 @@ void DetectLowerSetupCallback(
     }
 }
 
-void SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc fn, void *arg)
+bool SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc fn, void *arg)
 {
     DetectEngineCtx *de_ctx = DetectEngineGetCurrent();
+    if (de_ctx == NULL) {
+        SCLogError("no detection engine available for rate filter callback registration");
+        return false;
+    }
     de_ctx->RateFilterCallback = fn;
     de_ctx->rate_filter_callback_arg = arg;
     DetectEngineDeReference(&de_ctx);
+    return true;
 }
 
 int DetectEngineThreadCtxGetJsonContext(DetectEngineThreadCtx *det_ctx)

src/detect.h

@@ -1214,7 +1214,7 @@ typedef struct DetectEngineCtx_ {
  * This callback is added to the current detection engine and will be
  * copied to all future detection engines over rule reloads.
  */
-void SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc cb, void *arg);
+bool SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc cb, void *arg);
 
 /* Engine groups profiles (low, medium, high, custom) */
 enum {