diff --git a/README.md b/README.md index 1790057..a56b39d 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,18 @@ class User end ``` +To improve performance, you can save many entities at once. (Batch operation) + +```ruby +group = Group.find(params[:group_id]) +users = %w{ alice bob charlie }.map{ |name| User.new(name: name) } +if ActiveModel::Datastore.save_all(users, parent: group) + format.html { redirect_to action: :index, notice: "#{users.count} users were successfully created.' } +else + format.html { render :new } +end +``` + ## Controller Example Now on to the controller! A scaffold generated controller works out of the box: diff --git a/lib/active_model/datastore.rb b/lib/active_model/datastore.rb index 540c1a3..f54ac0c 100644 --- a/lib/active_model/datastore.rb +++ b/lib/active_model/datastore.rb @@ -193,6 +193,12 @@ def destroy end end + def fill_id_from_entity(entity) + self.id = entity.key.id + self.parent_key_id = entity.key.parent.id if entity.key.parent.present? + entity + end + private def save_entity(parent = nil) @@ -200,9 +206,7 @@ def save_entity(parent = nil) run_callbacks :save do entity = build_entity(parent) success = self.class.retry_on_exception? { CloudDatastore.dataset.save entity } - self.id = entity.key.id if success - self.parent_key_id = entity.key.parent.id if entity.key.parent.present? - success + success ? fill_id_from_entity(entity) : false end end diff --git a/lib/active_model/datastore/batch_operation.rb b/lib/active_model/datastore/batch_operation.rb new file mode 100644 index 0000000..e254475 --- /dev/null +++ b/lib/active_model/datastore/batch_operation.rb @@ -0,0 +1,39 @@ +## +# Batch operations +# +# +# Such batch calls are faster than making separate calls for each individual entity because they +# incur the overhead for only one service call. +# +# reference: https://cloud.google.com/datastore/docs/concepts/entities#batch_operations +# +# group = Group.find(params[:group_id]) +# users = %w{ alice bob charlie }.map{ |name| User.new(name: name) } +# saved_users = ActiveModel::Datastore.save_all(users, parent: group) +# +module ActiveModel::Datastore + def self.save_all(entries, parent: nil) + return if entries.reject(&:valid?).present? + entries.each_slice(500).map do |sliced_entries| + entities = [] + results = nil + fn = lambda do |n| + entry = sliced_entries[n] + entry.run_callbacks(:save) do + entities << entry.build_entity(parent) + if n + 1 < sliced_entries.count + # recursive call + fn.call(n + 1) + else + # batch insert + results = entry.class.retry_on_exception? { CloudDatastore.dataset.save entities } + end + sliced_entries[n].fill_id_from_entity(results&.fetch(n)) + results.present? + end + end + fn.call(0) + sliced_entries + end.flatten + end +end diff --git a/lib/activemodel/datastore.rb b/lib/activemodel/datastore.rb index a426a4a..03820b4 100644 --- a/lib/activemodel/datastore.rb +++ b/lib/activemodel/datastore.rb @@ -11,4 +11,5 @@ require 'active_model/datastore/nested_attr' require 'active_model/datastore/property_values' require 'active_model/datastore/track_changes' +require 'active_model/datastore/batch_operation' require 'active_model/datastore' diff --git a/test/cases/batch_operation_test.rb b/test/cases/batch_operation_test.rb new file mode 100644 index 0000000..bf2bea4 --- /dev/null +++ b/test/cases/batch_operation_test.rb @@ -0,0 +1,78 @@ +require 'test_helper' + +class ActiveModel::BatchOperationTest < ActiveSupport::TestCase + def setup + super + @mock_models = %w[alice bob charlie].map do |name| + MockModel.new(name: name, parent_key_id: MOCK_PARENT_ID) + end + end + + test 'batch save' do + count = MockModel.count_test_entities + assert ActiveModel::Datastore.save_all(@mock_models) + assert_equal count + @mock_models.count, MockModel.count_test_entities + @mock_models.each do |mock_model| + assert_not_nil mock_model.id + key = CloudDatastore.dataset.key 'MockModel', mock_model.id + key.parent = CloudDatastore.dataset.key('ParentMockModel', MOCK_PARENT_ID) + entity = CloudDatastore.dataset.find key + assert_equal mock_model.id, entity.key.id + assert_equal 'MockModel', entity.key.kind + assert_equal 'ParentMockModel', entity.key.parent.kind + assert_equal MOCK_PARENT_ID, entity.key.parent.id + end + end + + test 'before validation callback on batch save' do + @mock_models.each do |mock_model| + class << mock_model + before_validation { self.name = nil } + end + end + refute ActiveModel::Datastore.save_all(@mock_models) + @mock_models.each do |mock_model| + assert_nil mock_model.name + end + assert_equal 0, MockModel.count_test_entities + end + + test 'after validation callback on batch save' do + @mock_models.each do |mock_model| + class << mock_model + after_validation { self.name = nil } + end + end + assert ActiveModel::Datastore.save_all(@mock_models) + @mock_models.each do |mock_model| + assert_nil mock_model.name + end + assert_equal @mock_models.count, MockModel.count_test_entities + end + + test 'before save callback on batch save' do + @mock_models.each do |mock_model| + class << mock_model + before_save { self.name = name.upcase } + end + end + assert ActiveModel::Datastore.save_all(@mock_models) + @mock_models.each_with_index do |mock_model, index| + assert_equal mock_model.name, %w[alice bob charlie][index].upcase + assert_equal MockModel.all[index].name, %w[alice bob charlie][index].upcase + end + end + + test 'after save callback on batch save' do + @mock_models.each do |mock_model| + class << mock_model + after_save { self.name = name.upcase } + end + end + assert ActiveModel::Datastore.save_all(@mock_models) + @mock_models.each_with_index do |mock_model, index| + assert_equal mock_model.name, %w[alice bob charlie][index].upcase + assert_equal MockModel.all[index].name, %w[alice bob charlie][index] + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index d7b2de8..ea06b81 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -15,6 +15,7 @@ require 'active_model/datastore/nested_attr' require 'active_model/datastore/property_values' require 'active_model/datastore/track_changes' +require 'active_model/datastore/batch_operation' require 'active_model/datastore' require 'action_controller/metal/strong_parameters'