Improved handling of function parameter validation

SDL supports the following use cases:
 * Normal operation with fast parameter checks (default):
    SDL_SetHint(SDL_HINT_INVALID_PARAM_CHECKS, "1");
 * Object parameters are checked for use-after-free issues:
    SDL_SetHint(SDL_HINT_INVALID_PARAM_CHECKS, "2");
 * Enable full validation, plus assert on invalid parameters:
    #define SDL_ASSERT_INVALID_PARAMS
 * Disable all parameter validation:
    #define SDL_DISABLE_INVALID_PARAMS

Author: slouken

include/SDL3/SDL_hints.h

@@ -685,6 +685,20 @@ extern "C" {
  */
 #define SDL_HINT_DISPLAY_USABLE_BOUNDS "SDL_DISPLAY_USABLE_BOUNDS"
 
+/**
+ * Set the level of checking for invalid parameters passed to SDL functions.
+ *
+ * The variable can be set to the following values:
+ *
+ * - "1": Enable fast parameter error checking, e.g. quick NULL checks, etc. (default)
+ * - "2": Enable full parameter error checking, e.g. validating objects are the correct type, etc.
+ *
+ * This hint can be set anytime.
+ *
+ * \since This hint is available since SDL 3.4.0.
+ */
+#define SDL_HINT_INVALID_PARAM_CHECKS "SDL_INVALID_PARAM_CHECKS"
+
 /**
  * Disable giving back control to the browser automatically when running with
  * asyncify.

src/SDL_internal.h

@@ -272,6 +272,14 @@ extern "C" {
 #define POP_SDL_ERROR() \
     SDL_SetError("%s", _error); SDL_free(_error); }
 
+#if defined(SDL_DISABLE_INVALID_PARAMS)
+#define CHECK_PARAM(invalid) if (false)
+#elif defined(SDL_ASSERT_INVALID_PARAMS)
+#define CHECK_PARAM(invalid) SDL_assert_always(!(invalid)); if (invalid)
+#else
+#define CHECK_PARAM(invalid) if (invalid)
+#endif
+
 // Do any initialization that needs to happen before threads are started
 extern void SDL_InitMainThread(void);
 

src/SDL_utils.c

@@ -137,6 +137,32 @@ Uint32 SDL_GetNextObjectID(void)
 
 static SDL_InitState SDL_objects_init;
 static SDL_HashTable *SDL_objects;
+static bool SDL_object_validation;
+
+static void SDLCALL SDL_InvalidParamChecksChanged(void *userdata, const char *name, const char *oldValue, const char *hint)
+{
+    bool validation_enabled = false;
+
+#ifdef SDL_ASSERT_INVALID_PARAMS
+    // Full validation is enabled by default
+    validation_enabled = true;
+#endif
+
+    if (hint) {
+        switch (*hint) {
+        case '0':
+        case '1':
+            validation_enabled = false;
+            break;
+        case '2':
+            validation_enabled = true;
+            break;
+        default:
+            break;
+        }
+    }
+    SDL_object_validation = validation_enabled;
+}
 
 static Uint32 SDLCALL SDL_HashObject(void *unused, const void *key)
 {
@@ -159,6 +185,7 @@ void SDL_SetObjectValid(void *object, SDL_ObjectType type, bool valid)
         if (!initialized) {
             return;
         }
+        SDL_AddHintCallback(SDL_HINT_INVALID_PARAM_CHECKS, SDL_InvalidParamChecksChanged, NULL);
     }
 
     if (valid) {
@@ -174,6 +201,10 @@ bool SDL_ObjectValid(void *object, SDL_ObjectType type)
         return false;
     }
 
+    if (!SDL_object_validation) {
+        return true;
+    }
+
     const void *object_type;
     if (!SDL_FindInHashTable(SDL_objects, object, &object_type)) {
         return false;
@@ -242,6 +273,7 @@ void SDL_SetObjectsInvalid(void)
         SDL_DestroyHashTable(SDL_objects);
         SDL_objects = NULL;
         SDL_SetInitialized(&SDL_objects_init, false);
+        SDL_RemoveHintCallback(SDL_HINT_INVALID_PARAM_CHECKS, SDL_InvalidParamChecksChanged, NULL);
     }
 }