diff --git a/lib/factory_bot_rails/railtie.rb b/lib/factory_bot_rails/railtie.rb
index 5512f23..de539cf 100644
--- a/lib/factory_bot_rails/railtie.rb
+++ b/lib/factory_bot_rails/railtie.rb
@@ -21,13 +21,15 @@ class Railtie < Rails::Railtie
       FactoryBot.definition_file_paths = definition_file_paths
     end
 
-    ActiveSupport.on_load :active_record do
-      config = Rails.configuration.factory_bot
+    initializer "factory_bot.reject_primary_key_attributes" do
+      ActiveSupport.on_load :active_record do
+        config = Rails.configuration.factory_bot
 
-      if config.reject_primary_key_attributes
-        require "factory_bot_rails/factory_validator/active_record_validator"
+        if config.reject_primary_key_attributes
+          require "factory_bot_rails/factory_validator/active_record_validator"
 
-        config.validator.add_validator FactoryValidator::ActiveRecordValidator.new
+          config.validator.add_validator FactoryValidator::ActiveRecordValidator.new
+        end
       end
     end