diff --git a/CHANGELOG.md b/CHANGELOG.md
index dd3b658..3b5aaa2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 ## master
 
+- Do not preload Rails base classes, use load hooks everywhere. ([@palkan][])
+
 ## 0.6.7 (2023-09-13)
 
 - Fix loading Rails extensions during eager load. ([@palkan][])
diff --git a/lib/action_policy/rails/scope_matchers/action_controller_params.rb b/lib/action_policy/rails/scope_matchers/action_controller_params.rb
index 5ee73c8..f533428 100644
--- a/lib/action_policy/rails/scope_matchers/action_controller_params.rb
+++ b/lib/action_policy/rails/scope_matchers/action_controller_params.rb
@@ -12,8 +12,10 @@ def params_filter(...)
   end
 end
 
-# Register params scope matcher
-ActionPolicy::Base.scope_matcher :action_controller_params, ActionController::Parameters
-
 # Add alias to base policy
 ActionPolicy::Base.extend ActionPolicy::ScopeMatchers::ActionControllerParams
+
+ActiveSupport.on_load(:action_controller) do
+  # Register params scope matcher
+  ActionPolicy::Base.scope_matcher :action_controller_params, ActionController::Parameters
+end
diff --git a/lib/action_policy/rails/scope_matchers/active_record.rb b/lib/action_policy/rails/scope_matchers/active_record.rb
index 1505b29..a6466ce 100644
--- a/lib/action_policy/rails/scope_matchers/active_record.rb
+++ b/lib/action_policy/rails/scope_matchers/active_record.rb
@@ -12,18 +12,20 @@ def relation_scope(...)
   end
 end
 
-# Register relation scope matcher
-ActionPolicy::Base.scope_matcher :active_record_relation, ActiveRecord::Relation
-
 # Add alias to base policy
 ActionPolicy::Base.extend ActionPolicy::ScopeMatchers::ActiveRecord
 
-ActiveRecord::Relation.include(Module.new do
-  def policy_name
-    if model.respond_to?(:policy_name)
-      model.policy_name.to_s
-    else
-      "#{model}Policy"
+ActiveSupport.on_load(:active_record) do
+  # Register relation scope matcher
+  ActionPolicy::Base.scope_matcher :active_record_relation, ActiveRecord::Relation
+
+  ActiveRecord::Relation.include(Module.new do
+    def policy_name
+      if model.respond_to?(:policy_name)
+        model.policy_name.to_s
+      else
+        "#{model}Policy"
+      end
     end
-  end
-end)
+  end)
+end
diff --git a/lib/action_policy/railtie.rb b/lib/action_policy/railtie.rb
index 5a7f3cb..4dc1586 100644
--- a/lib/action_policy/railtie.rb
+++ b/lib/action_policy/railtie.rb
@@ -74,8 +74,6 @@ def cache_store=(store)
         app.config.action_policy.namespace_cache_enabled
 
       ActiveSupport.on_load(:action_controller) do
-        require "action_policy/rails/scope_matchers/action_controller_params"
-
         next unless app.config.action_policy.auto_inject_into_controller
 
         ActionController::Base.include ActionPolicy::Controller
@@ -95,21 +93,12 @@ def cache_store=(store)
         ActionCable::Channel::Base.authorize :user, through: :current_user
       end
 
+      # Scope matchers
+      require "action_policy/rails/scope_matchers/action_controller_params"
+      require "action_policy/rails/scope_matchers/active_record"
+
       ActiveSupport.on_load(:active_record) do
         require "action_policy/rails/ext/active_record"
-        require "action_policy/rails/scope_matchers/active_record"
-      end
-
-      # Trigger load hooks of the components that extend ActionPolicy itself
-      # (e.g., scope matchers)
-      begin
-        ::ActionController::Base
-      rescue NameError
-      end
-
-      begin
-        ::ActiveRecord::Base
-      rescue NameError
       end
     end
   end