From c5a557f5d97ba983f8117ff826998c251330e02c Mon Sep 17 00:00:00 2001 From: hschne Date: Wed, 28 May 2025 21:23:16 +0200 Subject: [PATCH 1/2] fix: split async and sync commands --- .standard.yml | 2 +- Gemfile.lock | 169 +++++++++--------- README.md | 2 +- .../litestream/restorations_controller.rb | 2 +- lib/litestream/commands.rb | 83 ++++++--- lib/tasks/litestream_tasks.rake | 74 +++----- litestream.gemspec | 6 +- test/litestream/test_commands.rb | 103 +++++++---- test/tasks/test_litestream_tasks.rb | 44 ++--- 9 files changed, 267 insertions(+), 218 deletions(-) diff --git a/.standard.yml b/.standard.yml index 3a0787a..e9efa16 100644 --- a/.standard.yml +++ b/.standard.yml @@ -1,3 +1,3 @@ # For available configuration options, see: # https://github.com/testdouble/standard -ruby_version: 2.6 +ruby_version: 3.0 diff --git a/Gemfile.lock b/Gemfile.lock index 347015d..bdce92d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,95 +6,92 @@ PATH actionview (>= 7.0) activejob (>= 7.0) activesupport (>= 7.0) - logfmt (>= 0.0.10) railties (>= 7.0) sqlite3 GEM remote: https://rubygems.org/ specs: - actioncable (7.1.3.2) - actionpack (= 7.1.3.2) - activesupport (= 7.1.3.2) + actioncable (8.0.2) + actionpack (= 8.0.2) + activesupport (= 8.0.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.3.2) - actionpack (= 7.1.3.2) - activejob (= 7.1.3.2) - activerecord (= 7.1.3.2) - activestorage (= 7.1.3.2) - activesupport (= 7.1.3.2) - mail (>= 2.7.1) - net-imap - net-pop - net-smtp - actionmailer (7.1.3.2) - actionpack (= 7.1.3.2) - actionview (= 7.1.3.2) - activejob (= 7.1.3.2) - activesupport (= 7.1.3.2) - mail (~> 2.5, >= 2.5.4) - net-imap - net-pop - net-smtp + actionmailbox (8.0.2) + actionpack (= 8.0.2) + activejob (= 8.0.2) + activerecord (= 8.0.2) + activestorage (= 8.0.2) + activesupport (= 8.0.2) + mail (>= 2.8.0) + actionmailer (8.0.2) + actionpack (= 8.0.2) + actionview (= 8.0.2) + activejob (= 8.0.2) + activesupport (= 8.0.2) + mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.1.3.2) - actionview (= 7.1.3.2) - activesupport (= 7.1.3.2) + actionpack (8.0.2) + actionview (= 8.0.2) + activesupport (= 8.0.2) nokogiri (>= 1.8.5) - racc rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.3.2) - actionpack (= 7.1.3.2) - activerecord (= 7.1.3.2) - activestorage (= 7.1.3.2) - activesupport (= 7.1.3.2) + useragent (~> 0.16) + actiontext (8.0.2) + actionpack (= 8.0.2) + activerecord (= 8.0.2) + activestorage (= 8.0.2) + activesupport (= 8.0.2) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.3.2) - activesupport (= 7.1.3.2) + actionview (8.0.2) + activesupport (= 8.0.2) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.3.2) - activesupport (= 7.1.3.2) + activejob (8.0.2) + activesupport (= 8.0.2) globalid (>= 0.3.6) - activemodel (7.1.3.2) - activesupport (= 7.1.3.2) - activerecord (7.1.3.2) - activemodel (= 7.1.3.2) - activesupport (= 7.1.3.2) + activemodel (8.0.2) + activesupport (= 8.0.2) + activerecord (8.0.2) + activemodel (= 8.0.2) + activesupport (= 8.0.2) timeout (>= 0.4.0) - activestorage (7.1.3.2) - actionpack (= 7.1.3.2) - activejob (= 7.1.3.2) - activerecord (= 7.1.3.2) - activesupport (= 7.1.3.2) + activestorage (8.0.2) + actionpack (= 8.0.2) + activejob (= 8.0.2) + activerecord (= 8.0.2) + activesupport (= 8.0.2) marcel (~> 1.0) - activesupport (7.1.3.2) + activesupport (8.0.2) base64 + benchmark (>= 0.3) bigdecimal - concurrent-ruby (~> 1.0, >= 1.0.2) + concurrent-ruby (~> 1.0, >= 1.3.1) connection_pool (>= 2.2.5) drb i18n (>= 1.6, < 2) + logger (>= 1.4.2) minitest (>= 5.1) - mutex_m - tzinfo (~> 2.0) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + uri (>= 0.13.1) ast (2.4.2) base64 (0.2.0) + benchmark (0.4.0) bigdecimal (3.1.7) builder (3.2.4) - concurrent-ruby (1.2.3) + concurrent-ruby (1.3.5) connection_pool (2.4.1) crass (1.0.6) - date (3.3.4) + date (3.4.1) drb (2.2.1) erubi (1.12.0) globalid (1.2.1) @@ -102,13 +99,14 @@ GEM i18n (1.14.4) concurrent-ruby (~> 1.0) io-console (0.7.2) - irb (1.12.0) - rdoc + irb (1.15.2) + pp (>= 0.6.0) + rdoc (>= 4.0.0) reline (>= 0.4.2) json (2.7.2) language_server-protocol (3.17.0.3) lint_roller (1.1.0) - logfmt (0.0.10) + logger (1.7.0) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -120,25 +118,27 @@ GEM marcel (1.0.4) mini_mime (1.1.5) minitest (5.22.3) - mutex_m (0.2.0) - net-imap (0.4.10) + net-imap (0.5.8) date net-protocol net-pop (0.1.2) net-protocol net-protocol (0.2.2) timeout - net-smtp (0.5.0) + net-smtp (0.5.1) net-protocol - nio4r (2.7.1) - nokogiri (1.16.4-arm64-darwin) + nio4r (2.7.4) + nokogiri (1.18.8-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.4-x86_64-linux) + nokogiri (1.18.8-x86_64-linux-gnu) racc (~> 1.4) parallel (1.24.0) parser (3.3.0.5) ast (~> 2.4.1) racc + pp (0.6.2) + prettyprint + prettyprint (0.2.0) psych (5.1.2) stringio racc (1.7.3) @@ -150,20 +150,20 @@ GEM rackup (2.1.0) rack (>= 3) webrick (~> 1.8) - rails (7.1.3.2) - actioncable (= 7.1.3.2) - actionmailbox (= 7.1.3.2) - actionmailer (= 7.1.3.2) - actionpack (= 7.1.3.2) - actiontext (= 7.1.3.2) - actionview (= 7.1.3.2) - activejob (= 7.1.3.2) - activemodel (= 7.1.3.2) - activerecord (= 7.1.3.2) - activestorage (= 7.1.3.2) - activesupport (= 7.1.3.2) + rails (8.0.2) + actioncable (= 8.0.2) + actionmailbox (= 8.0.2) + actionmailer (= 8.0.2) + actionpack (= 8.0.2) + actiontext (= 8.0.2) + actionview (= 8.0.2) + activejob (= 8.0.2) + activemodel (= 8.0.2) + activerecord (= 8.0.2) + activestorage (= 8.0.2) + activesupport (= 8.0.2) bundler (>= 1.15.0) - railties (= 7.1.3.2) + railties (= 8.0.2) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest @@ -171,10 +171,10 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.1.3.2) - actionpack (= 7.1.3.2) - activesupport (= 7.1.3.2) - irb + railties (8.0.2) + actionpack (= 8.0.2) + activesupport (= 8.0.2) + irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) @@ -205,8 +205,9 @@ GEM rubocop-ast (>= 1.30.0, < 2.0) ruby-progressbar (1.13.0) rubyzip (2.3.2) - sqlite3 (1.7.3-arm64-darwin) - sqlite3 (1.7.3-x86_64-linux) + securerandom (0.4.1) + sqlite3 (2.6.0-arm64-darwin) + sqlite3 (2.6.0-x86_64-linux-gnu) standard (1.35.1) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.0) @@ -221,12 +222,15 @@ GEM rubocop-performance (~> 1.20.2) stringio (3.1.0) thor (1.3.1) - timeout (0.4.1) + timeout (0.4.3) tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.5.0) + uri (1.0.3) + useragent (0.16.11) webrick (1.8.1) - websocket-driver (0.7.6) + websocket-driver (0.8.0) + base64 websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) zeitwerk (2.6.13) @@ -241,7 +245,6 @@ DEPENDENCIES rails rake (~> 13.0) rubyzip - sqlite3 standard (~> 1.3) BUNDLED WITH diff --git a/README.md b/README.md index dda83b4..91c89a5 100644 --- a/README.md +++ b/README.md @@ -500,7 +500,7 @@ time=YYYY-MM-DDTHH:MM:SS level=INFO msg="replicating to" name=s3 type=s3 sync-in After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. -To install this gem onto your local machine, run `bundle exec rake install`. +To install this gem onto your local machine, run `bundle exec rake install`. To download the Litestream binaries run `bundle exec rake download`. For maintainers, to release a new version, run `bin/release $VERSION`, which will create a git tag for the version, push git commits and tags, and push all of the platform-specific `.gem` files to [rubygems.org](https://rubygems.org). diff --git a/app/controllers/litestream/restorations_controller.rb b/app/controllers/litestream/restorations_controller.rb index d22fc20..360e96f 100644 --- a/app/controllers/litestream/restorations_controller.rb +++ b/app/controllers/litestream/restorations_controller.rb @@ -9,7 +9,7 @@ def create now = Time.now.utc.strftime("%Y%m%d%H%M%S") backup = File.join(dir, "#{base}-#{now}#{ext}") - Litestream::Commands.restore(database, async: false, **{"-o" => backup}) + Litestream::Commands.restore(database, **{"-o" => backup}) redirect_to root_path, notice: "Restored to #{backup}." end diff --git a/lib/litestream/commands.rb b/lib/litestream/commands.rb index 8eed563..f38b64b 100644 --- a/lib/litestream/commands.rb +++ b/lib/litestream/commands.rb @@ -1,5 +1,4 @@ require_relative "upstream" -require "logfmt" module Litestream module Commands @@ -21,6 +20,24 @@ module Commands # raised when a litestream command fails CommandFailedException = Class.new(StandardError) + module Output + class << self + def format(data) + return "" if data.nil? || data.empty? + + headers = data.first.keys.map(&:to_s) + widths = headers.map.with_index { |h, i| + [h.length, data.map { |r| r[data.first.keys[i]].to_s.length }.max].max + } + + format_str = widths.map { |w| "%-#{w}s" }.join(" ") + ([headers] + data.map(&:values)).map { |row| + sprintf(format_str, *row.map(&:to_s)) + }.join("\n") + end + end + end + class << self def platform [:cpu, :os].map { |m| Gem::Platform.local.send(m) }.join("-") @@ -78,43 +95,45 @@ def executable(exe_path: DEFAULT_DIR) end def replicate(async: false, **argv) - execute("replicate", argv, async: async, tabled_output: false) + cmd = prepare("replicate", argv) + run_async(cmd, async: async) + rescue + raise CommandFailedException, "Failed to execute `#{cmd.join(" ")}`" end - def restore(database, async: false, **argv) + def restore(database, **argv) raise DatabaseRequiredException, "database argument is required for restore command, e.g. litestream:restore -- --database=path/to/database.sqlite" if database.nil? - argv.stringify_keys! - execute("restore", argv, database, async: async, tabled_output: false) + execute("restore", argv, database, tabled_output: false) end - def databases(async: false, **argv) - execute("databases", argv, async: async, tabled_output: true) + def databases(**argv) + execute("databases", argv) end - def generations(database, async: false, **argv) + def generations(database, **argv) raise DatabaseRequiredException, "database argument is required for generations command, e.g. litestream:generations -- --database=path/to/database.sqlite" if database.nil? - execute("generations", argv, database, async: async, tabled_output: true) + execute("generations", argv, database) end - def snapshots(database, async: false, **argv) + def snapshots(database, **argv) raise DatabaseRequiredException, "database argument is required for snapshots command, e.g. litestream:snapshots -- --database=path/to/database.sqlite" if database.nil? - execute("snapshots", argv, database, async: async, tabled_output: true) + execute("snapshots", argv, database) end - def wal(database, async: false, **argv) + def wal(database, **argv) raise DatabaseRequiredException, "database argument is required for wal command, e.g. litestream:wal -- --database=path/to/database.sqlite" if database.nil? - execute("wal", argv, database, async: async, tabled_output: true) + execute("wal", argv, database) end private - def execute(command, argv = {}, database = nil, async: false, tabled_output: false) + def execute(command, argv = {}, database = nil, tabled_output: true) cmd = prepare(command, argv, database) - results = run(cmd, async: async, tabled_output: tabled_output) + results = run(cmd, tabled_output: tabled_output) if Array === results && results.one? && results[0]["level"] == "ERROR" raise CommandFailedException, "Failed to execute `#{cmd.join(" ")}`; Reason: #{results[0]["error"]}" @@ -137,20 +156,38 @@ def prepare(command, argv = {}, database = nil) cmd end - def run(cmd, async: false, tabled_output: false) + def run(cmd, tabled_output:) + stdout = `#{cmd.join(" ")}`.chomp + return stdout unless tabled_output + + keys, *rows = stdout.split("\n").map { _1.split(/\s+/) } + rows.map { keys.zip(_1).to_h } + end + + def run_async(cmd, async:) if async - # To release the resources of the Ruby process, just fork and exit. - # The forked process executes litestream and replaces itself. exec(*cmd) if fork.nil? else - stdout = `#{cmd.join(" ")}`.chomp - tabled_output ? text_table_to_hashes(stdout) : stdout.split("\n").map { Logfmt.parse(_1) } + IO.popen(cmd, err: [:child, :out]) do |io| + io.each_line { |line| puts line } + end end end - def text_table_to_hashes(string) - keys, *rows = string.split("\n").map { _1.split(/\s+/) } - rows.map { keys.zip(_1).to_h } + module Output + class << self + def format(data) + headers = data.first.keys.map(&:to_s) + widths = headers.map.with_index { |h, i| + [h.length, data.map { |r| r[data.first.keys[i]].to_s.length }.max].max + } + + format_str = widths.map { |w| "%-#{w}s" }.join(" ") + ([headers] + data.map(&:values)).map { |row| + sprintf(format_str, *row.map(&:to_s)) + }.join("\n") + end + end end end end diff --git a/lib/tasks/litestream_tasks.rake b/lib/tasks/litestream_tasks.rake index 4456cd9..3270aa1 100644 --- a/lib/tasks/litestream_tasks.rake +++ b/lib/tasks/litestream_tasks.rake @@ -10,83 +10,61 @@ namespace :litestream do desc 'Monitor and continuously replicate SQLite databases defined in your config file, for example `rake litestream:replicate -- -exec "foreman start"`' task replicate: :environment do - options = {} - if (separator_index = ARGV.index("--")) - ARGV.slice(separator_index + 1, ARGV.length) - .map { |pair| pair.split("=") } - .each { |opt| options[opt[0]] = opt[1] || nil } - end - options.symbolize_keys! + options = parse_argv_options - Litestream::Commands.replicate(async: true, **options) + Litestream::Commands.replicate(**options) end desc "Restore a SQLite database from a Litestream replica, for example `rake litestream:restore -- -database=storage/production.sqlite3`" task restore: :environment do - options = {} - if (separator_index = ARGV.index("--")) - ARGV.slice(separator_index + 1, ARGV.length) - .map { |pair| pair.split("=") } - .each { |opt| options[opt[0]] = opt[1] || nil } - end - database = options.delete("--database") || options.delete("-database") - options.symbolize_keys! + options = parse_argv_options + database = options.delete(:"--database") || options.delete(:"-database") - Litestream::Commands.restore(database, async: true, **options) + puts Litestream::Commands.restore(database, **options) end desc "List all databases and associated replicas in the config file, for example `rake litestream:databases -- -no-expand-env`" task databases: :environment do - options = {} - if (separator_index = ARGV.index("--")) - ARGV.slice(separator_index + 1, ARGV.length) - .map { |pair| pair.split("=") } - .each { |opt| options[opt[0]] = opt[1] || nil } - end - options.symbolize_keys! + options = parse_argv_options - Litestream::Commands.databases(async: true, **options) + puts Litestream::Commands::Output.format(Litestream::Commands.databases(**options)) end desc "List all generations for a database or replica, for example `rake litestream:generations -- -database=storage/production.sqlite3`" task generations: :environment do - options = {} - if (separator_index = ARGV.index("--")) - ARGV.slice(separator_index + 1, ARGV.length) - .map { |pair| pair.split("=") } - .each { |opt| options[opt[0]] = opt[1] || nil } - end - database = options.delete("--database") || options.delete("-database") - options.symbolize_keys! + options = parse_argv_options + database = options.delete(:"--database") || options.delete(:"-database") - Litestream::Commands.generations(database, async: true, **options) + puts Litestream::Commands::Output.format(Litestream::Commands.generations(database, **options)) end desc "List all snapshots for a database or replica, for example `rake litestream:snapshots -- -database=storage/production.sqlite3`" task snapshots: :environment do - options = {} - if (separator_index = ARGV.index("--")) - ARGV.slice(separator_index + 1, ARGV.length) - .map { |pair| pair.split("=") } - .each { |opt| options[opt[0]] = opt[1] || nil } - end - database = options.delete("--database") || options.delete("-database") - options.symbolize_keys! + options = parse_argv_options + database = options.delete(:"--database") || options.delete(:"-database") - Litestream::Commands.snapshots(database, async: true, **options) + puts Litestream::Commands::Output.format(Litestream::Commands.snapshots(database, **options)) end desc "List all wal files for a database or replica, for example `rake litestream:wal -- -database=storage/production.sqlite3`" task wal: :environment do + options = parse_argv_options + database = options.delete(:"--database") || options.delete(:"-database") + + puts Litestream::Commands::Output.format( + Litestream::Commands.wal(database, **options) + ) + end + + private + + def parse_argv_options options = {} if (separator_index = ARGV.index("--")) ARGV.slice(separator_index + 1, ARGV.length) .map { |pair| pair.split("=") } - .each { |opt| options[opt[0]] = opt[1] || nil } + .each { |opt| options[opt[0].to_sym] = opt[1] || nil } end - database = options.delete("--database") || options.delete("-database") - options.symbolize_keys! - - Litestream::Commands.wal(database, async: true, **options) + options.symbolize_keys end end diff --git a/litestream.gemspec b/litestream.gemspec index 2a6e0cd..e004350 100644 --- a/litestream.gemspec +++ b/litestream.gemspec @@ -25,18 +25,16 @@ Gem::Specification.new do |spec| spec.executables << "litestream" # Uncomment to register a new dependency of your gem - spec.add_dependency "logfmt", ">= 0.0.10" spec.add_dependency "sqlite3" ">= 7.0".tap do |rails_version| spec.add_dependency "actionpack", rails_version spec.add_dependency "actionview", rails_version - spec.add_dependency "activesupport", rails_version spec.add_dependency "activejob", rails_version + spec.add_dependency "activesupport", rails_version spec.add_dependency "railties", rails_version end - spec.add_development_dependency "rubyzip" spec.add_development_dependency "rails" - spec.add_development_dependency "sqlite3" + spec.add_development_dependency "rubyzip" # For more information and examples about making a new gem, check out our # guide at: https://bundler.io/guides/creating_gem.html diff --git a/test/litestream/test_commands.rb b/test/litestream/test_commands.rb index bafaaf5..0e39277 100644 --- a/test/litestream/test_commands.rb +++ b/test/litestream/test_commands.rb @@ -19,7 +19,7 @@ def teardown class TestReplicateCommand < TestCommands def test_replicate_with_no_options - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "replicate", command @@ -27,13 +27,13 @@ def test_replicate_with_no_options assert_equal "--config", argv[0] assert_match Regexp.new("dummy/config/litestream.yml"), argv[1] end - Litestream::Commands.stub :run, stub do + Litestream::Commands.stub :run_async, stub do Litestream::Commands.replicate end end def test_replicate_with_boolean_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "replicate", command @@ -42,13 +42,13 @@ def test_replicate_with_boolean_option assert_match Regexp.new("dummy/config/litestream.yml"), argv[1] assert_equal "--no-expand-env", argv[2] end - Litestream::Commands.stub :run, stub do + Litestream::Commands.stub :run_async, stub do Litestream::Commands.replicate("--no-expand-env" => nil) end end def test_replicate_with_string_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "replicate", command @@ -58,13 +58,13 @@ def test_replicate_with_string_option assert_equal "--exec", argv[2] assert_equal "command", argv[3] end - Litestream::Commands.stub :run, stub do + Litestream::Commands.stub :run_async, stub do Litestream::Commands.replicate("--exec" => "command") end end def test_replicate_with_symbol_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "replicate", command @@ -74,13 +74,13 @@ def test_replicate_with_symbol_option assert_equal "--exec", argv[2] assert_equal "command", argv[3] end - Litestream::Commands.stub :run, stub do + Litestream::Commands.stub :run_async, stub do Litestream::Commands.replicate("--exec": "command") end end def test_replicate_with_config_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "replicate", command @@ -88,7 +88,7 @@ def test_replicate_with_config_option assert_equal "--config", argv[0] assert_equal "CONFIG", argv[1] end - Litestream::Commands.stub :run, stub do + Litestream::Commands.stub :run_async, stub do Litestream::Commands.replicate("--config" => "CONFIG") end end @@ -96,7 +96,7 @@ def test_replicate_with_config_option def test_replicate_sets_replica_bucket_env_var_from_config_when_env_var_not_set Litestream.replica_bucket = "mybkt" - Litestream::Commands.stub :run, nil do + Litestream::Commands.stub :run_async, nil do Litestream::Commands.replicate end @@ -108,7 +108,7 @@ def test_replicate_sets_replica_bucket_env_var_from_config_when_env_var_not_set def test_replicate_sets_replica_key_id_env_var_from_config_when_env_var_not_set Litestream.replica_key_id = "mykey" - Litestream::Commands.stub :run, nil do + Litestream::Commands.stub :run_async, nil do Litestream::Commands.replicate end @@ -120,7 +120,7 @@ def test_replicate_sets_replica_key_id_env_var_from_config_when_env_var_not_set def test_replicate_sets_replica_access_key_env_var_from_config_when_env_var_not_set Litestream.replica_access_key = "access" - Litestream::Commands.stub :run, nil do + Litestream::Commands.stub :run_async, nil do Litestream::Commands.replicate end @@ -134,7 +134,7 @@ def test_replicate_sets_all_env_vars_from_config_when_env_vars_not_set Litestream.replica_key_id = "mykey" Litestream.replica_access_key = "access" - Litestream::Commands.stub :run, nil do + Litestream::Commands.stub :run_async, nil do Litestream::Commands.replicate end @@ -152,7 +152,7 @@ def test_replicate_does_not_set_env_var_from_config_when_env_vars_already_set Litestream.replica_key_id = "mykey" Litestream.replica_access_key = "access" - Litestream::Commands.stub :run, nil do + Litestream::Commands.stub :run_async, nil do Litestream::Commands.replicate end @@ -164,7 +164,7 @@ def test_replicate_does_not_set_env_var_from_config_when_env_vars_already_set class TestRestoreCommand < TestCommands def test_restore_with_no_options - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "restore", command @@ -179,7 +179,7 @@ def test_restore_with_no_options end def test_restore_with_boolean_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "restore", command @@ -195,7 +195,7 @@ def test_restore_with_boolean_option end def test_restore_with_string_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "restore", command @@ -212,7 +212,7 @@ def test_restore_with_string_option end def test_restore_with_config_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "restore", command @@ -297,7 +297,7 @@ def test_restore_does_not_set_env_var_from_config_when_env_vars_already_set class TestDatabasesCommand < TestCommands def test_databases_with_no_options - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "databases", command @@ -311,7 +311,7 @@ def test_databases_with_no_options end def test_databases_with_boolean_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "databases", command @@ -326,7 +326,7 @@ def test_databases_with_boolean_option end def test_databases_with_string_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "databases", command @@ -342,7 +342,7 @@ def test_databases_with_string_option end def test_databases_with_config_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "databases", command @@ -426,7 +426,7 @@ def test_databases_does_not_set_env_var_from_config_when_env_vars_already_set class TestGenerationsCommand < TestCommands def test_generations_with_no_options - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "generations", command @@ -441,7 +441,7 @@ def test_generations_with_no_options end def test_generations_with_boolean_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "generations", command @@ -457,7 +457,7 @@ def test_generations_with_boolean_option end def test_generations_with_string_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "generations", command @@ -474,7 +474,7 @@ def test_generations_with_string_option end def test_generations_with_config_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "generations", command @@ -559,7 +559,7 @@ def test_generations_does_not_set_env_var_from_config_when_env_vars_already_set class TestSnapshotsCommand < TestCommands def test_snapshots_with_no_options - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "snapshots", command @@ -574,7 +574,7 @@ def test_snapshots_with_no_options end def test_snapshots_with_boolean_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "snapshots", command @@ -590,7 +590,7 @@ def test_snapshots_with_boolean_option end def test_snapshots_with_string_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "snapshots", command @@ -607,7 +607,7 @@ def test_snapshots_with_string_option end def test_snapshots_with_config_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "snapshots", command @@ -692,7 +692,7 @@ def test_snapshots_does_not_set_env_var_from_config_when_env_vars_already_set class TestWalCommand < TestCommands def test_wal_with_no_options - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "wal", command @@ -707,7 +707,7 @@ def test_wal_with_no_options end def test_wal_with_boolean_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "wal", command @@ -723,7 +723,7 @@ def test_wal_with_boolean_option end def test_wal_with_string_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "wal", command @@ -740,7 +740,7 @@ def test_wal_with_string_option end def test_wal_with_config_option - stub = proc do |cmd, async| + stub = proc do |cmd| executable, command, *argv = cmd assert_match Regexp.new("exe/test/litestream"), executable assert_equal "wal", command @@ -822,4 +822,37 @@ def test_wal_does_not_set_env_var_from_config_when_env_vars_already_set assert_equal "original_access", ENV["LITESTREAM_SECRET_ACCESS_KEY"] end end + + class TestOutput < ActiveSupport::TestCase + def test_output_formatting_generates_table_with_data + data = [ + {path: "/storage/database.db", replicas: "s3"}, + {path: "/storage/another-database.db", replicas: "s3"} + ] + + result = Litestream::Commands::Output.format(data) + lines = result.split("\n") + + assert_equal 3, lines.length + + assert_includes lines[0], "path" + assert_includes lines[0], "replicas" + assert_includes lines[1], "/storage/database.db" + assert_includes lines[2], "/storage/another-database.db" + end + + def test_output_formatting_generates_formatted_table + data = [ + {path: "/storage/database.db", replicas: "s3"}, + {path: "/storage/another-database.db", replicas: "s3"} + ] + + result = Litestream::Commands::Output.format(data) + lines = result.split("\n") + + replicas_pos = lines[0].index("replicas") + assert_equal replicas_pos, lines[1].index("s3") + assert_equal replicas_pos, lines[2].index("s3") + end + end end diff --git a/test/tasks/test_litestream_tasks.rb b/test/tasks/test_litestream_tasks.rb index 8a757a9..9d68852 100644 --- a/test/tasks/test_litestream_tasks.rb +++ b/test/tasks/test_litestream_tasks.rb @@ -35,7 +35,7 @@ def test_env_task_when_nothing_configured_prints class TestReplicateTask < TestLitestreamTasks def test_replicate_task_with_no_arguments fake = Minitest::Mock.new - fake.expect :call, nil, [], async: true + fake.expect :call, nil, [] Litestream::Commands.stub :replicate, fake do Rake.application.invoke_task "litestream:replicate" end @@ -45,7 +45,7 @@ def test_replicate_task_with_no_arguments def test_replicate_task_with_arguments ARGV.replace ["--", "--no-expand-env"] fake = Minitest::Mock.new - fake.expect :call, nil, [], async: true, "--no-expand-env": nil + fake.expect :call, nil, [], "--no-expand-env": nil Litestream::Commands.stub :replicate, fake do Rake.application.invoke_task "litestream:replicate" end @@ -55,7 +55,7 @@ def test_replicate_task_with_arguments def test_replicate_task_with_arguments_without_separator ARGV.replace ["--no-expand-env"] fake = Minitest::Mock.new - fake.expect :call, nil, [], async: true + fake.expect :call, nil, [] Litestream::Commands.stub :replicate, fake do Rake.application.invoke_task "litestream:replicate" end @@ -67,7 +67,7 @@ class TestRestoreTask < TestLitestreamTasks def test_restore_task_with_only_database_using_single_dash ARGV.replace ["--", "-database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true + fake.expect :call, [], ["db/test.sqlite3"] Litestream::Commands.stub :restore, fake do Rake.application.invoke_task "litestream:restore" end @@ -77,7 +77,7 @@ def test_restore_task_with_only_database_using_single_dash def test_restore_task_with_only_database_using_double_dash ARGV.replace ["--", "--database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true + fake.expect :call, nil, ["db/test.sqlite3"] Litestream::Commands.stub :restore, fake do Rake.application.invoke_task "litestream:restore" end @@ -87,7 +87,7 @@ def test_restore_task_with_only_database_using_double_dash def test_restore_task_with_arguments ARGV.replace ["--", "-database=db/test.sqlite3", "--if-db-not-exists"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true, "--if-db-not-exists": nil + fake.expect :call, nil, ["db/test.sqlite3"], "--if-db-not-exists": nil Litestream::Commands.stub :restore, fake do Rake.application.invoke_task "litestream:restore" end @@ -97,7 +97,7 @@ def test_restore_task_with_arguments def test_restore_task_with_arguments_without_separator ARGV.replace ["-database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, [nil], async: true + fake.expect :call, nil, [nil] Litestream::Commands.stub :restore, fake do Rake.application.invoke_task "litestream:restore" end @@ -108,7 +108,7 @@ def test_restore_task_with_arguments_without_separator class TestDatabasesTask < TestLitestreamTasks def test_databases_task_with_no_arguments fake = Minitest::Mock.new - fake.expect :call, nil, [], async: true + fake.expect :call, nil, [] Litestream::Commands.stub :databases, fake do Rake.application.invoke_task "litestream:databases" end @@ -118,7 +118,7 @@ def test_databases_task_with_no_arguments def test_databases_task_with_arguments ARGV.replace ["--", "--no-expand-env"] fake = Minitest::Mock.new - fake.expect :call, nil, [], async: true, "--no-expand-env": nil + fake.expect :call, nil, [], "--no-expand-env": nil Litestream::Commands.stub :databases, fake do Rake.application.invoke_task "litestream:databases" end @@ -128,7 +128,7 @@ def test_databases_task_with_arguments def test_databases_task_with_arguments_without_separator ARGV.replace ["--no-expand-env"] fake = Minitest::Mock.new - fake.expect :call, nil, [], async: true + fake.expect :call, nil, [] Litestream::Commands.stub :databases, fake do Rake.application.invoke_task "litestream:databases" end @@ -140,7 +140,7 @@ class TestGenerationsTask < TestLitestreamTasks def test_generations_task_with_only_database_using_single_dash ARGV.replace ["--", "-database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true + fake.expect :call, nil, ["db/test.sqlite3"] Litestream::Commands.stub :generations, fake do Rake.application.invoke_task "litestream:generations" end @@ -150,7 +150,7 @@ def test_generations_task_with_only_database_using_single_dash def test_generations_task_with_only_database_using_double_dash ARGV.replace ["--", "--database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true + fake.expect :call, nil, ["db/test.sqlite3"] Litestream::Commands.stub :generations, fake do Rake.application.invoke_task "litestream:generations" end @@ -160,7 +160,7 @@ def test_generations_task_with_only_database_using_double_dash def test_generations_task_with_arguments ARGV.replace ["--", "-database=db/test.sqlite3", "--if-db-not-exists"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true, "--if-db-not-exists": nil + fake.expect :call, nil, ["db/test.sqlite3"], "--if-db-not-exists": nil Litestream::Commands.stub :generations, fake do Rake.application.invoke_task "litestream:generations" end @@ -170,7 +170,7 @@ def test_generations_task_with_arguments def test_generations_task_with_arguments_without_separator ARGV.replace ["-database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, [nil], async: true + fake.expect :call, nil, [nil] Litestream::Commands.stub :generations, fake do Rake.application.invoke_task "litestream:generations" end @@ -182,7 +182,7 @@ class TestSnapshotsTask < TestLitestreamTasks def test_snapshots_task_with_only_database_using_single_dash ARGV.replace ["--", "-database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true + fake.expect :call, nil, ["db/test.sqlite3"] Litestream::Commands.stub :snapshots, fake do Rake.application.invoke_task "litestream:snapshots" end @@ -192,7 +192,7 @@ def test_snapshots_task_with_only_database_using_single_dash def test_snapshots_task_with_only_database_using_double_dash ARGV.replace ["--", "--database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true + fake.expect :call, nil, ["db/test.sqlite3"] Litestream::Commands.stub :snapshots, fake do Rake.application.invoke_task "litestream:snapshots" end @@ -202,7 +202,7 @@ def test_snapshots_task_with_only_database_using_double_dash def test_snapshots_task_with_arguments ARGV.replace ["--", "-database=db/test.sqlite3", "--if-db-not-exists"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true, "--if-db-not-exists": nil + fake.expect :call, nil, ["db/test.sqlite3"], "--if-db-not-exists": nil Litestream::Commands.stub :snapshots, fake do Rake.application.invoke_task "litestream:snapshots" end @@ -212,7 +212,7 @@ def test_snapshots_task_with_arguments def test_snapshots_task_with_arguments_without_separator ARGV.replace ["-database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, [nil], async: true + fake.expect :call, nil, [nil] Litestream::Commands.stub :snapshots, fake do Rake.application.invoke_task "litestream:snapshots" end @@ -224,7 +224,7 @@ class TestWalTask < TestLitestreamTasks def test_wal_task_with_only_database_using_single_dash ARGV.replace ["--", "-database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true + fake.expect :call, nil, ["db/test.sqlite3"] Litestream::Commands.stub :wal, fake do Rake.application.invoke_task "litestream:wal" end @@ -234,7 +234,7 @@ def test_wal_task_with_only_database_using_single_dash def test_wal_task_with_only_database_using_double_dash ARGV.replace ["--", "--database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true + fake.expect :call, nil, ["db/test.sqlite3"] Litestream::Commands.stub :wal, fake do Rake.application.invoke_task "litestream:wal" end @@ -244,7 +244,7 @@ def test_wal_task_with_only_database_using_double_dash def test_wal_task_with_arguments ARGV.replace ["--", "-database=db/test.sqlite3", "--if-db-not-exists"] fake = Minitest::Mock.new - fake.expect :call, nil, ["db/test.sqlite3"], async: true, "--if-db-not-exists": nil + fake.expect :call, nil, ["db/test.sqlite3"], "--if-db-not-exists": nil Litestream::Commands.stub :wal, fake do Rake.application.invoke_task "litestream:wal" end @@ -254,7 +254,7 @@ def test_wal_task_with_arguments def test_wal_task_with_arguments_without_separator ARGV.replace ["-database=db/test.sqlite3"] fake = Minitest::Mock.new - fake.expect :call, nil, [nil], async: true + fake.expect :call, nil, [nil] Litestream::Commands.stub :wal, fake do Rake.application.invoke_task "litestream:wal" end From 4a18ca87c717e20983a26a5ec1a890bcf7638ace Mon Sep 17 00:00:00 2001 From: hschne Date: Wed, 11 Jun 2025 04:40:38 +0200 Subject: [PATCH 2/2] Implement PR feedback --- lib/litestream/commands.rb | 24 ++++++------------------ lib/tasks/litestream_tasks.rake | 4 ++-- test/litestream/test_commands.rb | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 30 deletions(-) diff --git a/lib/litestream/commands.rb b/lib/litestream/commands.rb index f38b64b..2db7361 100644 --- a/lib/litestream/commands.rb +++ b/lib/litestream/commands.rb @@ -94,9 +94,12 @@ def executable(exe_path: DEFAULT_DIR) exe_file end + # Replicate can be run either as a fork or in the same process, depending on the context. + # Puma will start replication as a forked process, while running replication from a rake + # tasks won't. def replicate(async: false, **argv) cmd = prepare("replicate", argv) - run_async(cmd, async: async) + run_replicate(cmd, async: async) rescue raise CommandFailedException, "Failed to execute `#{cmd.join(" ")}`" end @@ -164,31 +167,16 @@ def run(cmd, tabled_output:) rows.map { keys.zip(_1).to_h } end - def run_async(cmd, async:) + def run_replicate(cmd, async:) if async exec(*cmd) if fork.nil? else + # When running in-process, we capture output continuously and write to stdout. IO.popen(cmd, err: [:child, :out]) do |io| io.each_line { |line| puts line } end end end - - module Output - class << self - def format(data) - headers = data.first.keys.map(&:to_s) - widths = headers.map.with_index { |h, i| - [h.length, data.map { |r| r[data.first.keys[i]].to_s.length }.max].max - } - - format_str = widths.map { |w| "%-#{w}s" }.join(" ") - ([headers] + data.map(&:values)).map { |row| - sprintf(format_str, *row.map(&:to_s)) - }.join("\n") - end - end - end end end end diff --git a/lib/tasks/litestream_tasks.rake b/lib/tasks/litestream_tasks.rake index 3270aa1..39aeb5b 100644 --- a/lib/tasks/litestream_tasks.rake +++ b/lib/tasks/litestream_tasks.rake @@ -63,8 +63,8 @@ namespace :litestream do if (separator_index = ARGV.index("--")) ARGV.slice(separator_index + 1, ARGV.length) .map { |pair| pair.split("=") } - .each { |opt| options[opt[0].to_sym] = opt[1] || nil } + .each { |opt| options[opt[0]] = opt[1] || nil } end - options.symbolize_keys + options.symbolize_keys! end end diff --git a/test/litestream/test_commands.rb b/test/litestream/test_commands.rb index 0e39277..e425b46 100644 --- a/test/litestream/test_commands.rb +++ b/test/litestream/test_commands.rb @@ -27,7 +27,7 @@ def test_replicate_with_no_options assert_equal "--config", argv[0] assert_match Regexp.new("dummy/config/litestream.yml"), argv[1] end - Litestream::Commands.stub :run_async, stub do + Litestream::Commands.stub :run_replicate, stub do Litestream::Commands.replicate end end @@ -42,7 +42,7 @@ def test_replicate_with_boolean_option assert_match Regexp.new("dummy/config/litestream.yml"), argv[1] assert_equal "--no-expand-env", argv[2] end - Litestream::Commands.stub :run_async, stub do + Litestream::Commands.stub :run_replicate, stub do Litestream::Commands.replicate("--no-expand-env" => nil) end end @@ -58,7 +58,7 @@ def test_replicate_with_string_option assert_equal "--exec", argv[2] assert_equal "command", argv[3] end - Litestream::Commands.stub :run_async, stub do + Litestream::Commands.stub :run_replicate, stub do Litestream::Commands.replicate("--exec" => "command") end end @@ -74,7 +74,7 @@ def test_replicate_with_symbol_option assert_equal "--exec", argv[2] assert_equal "command", argv[3] end - Litestream::Commands.stub :run_async, stub do + Litestream::Commands.stub :run_replicate, stub do Litestream::Commands.replicate("--exec": "command") end end @@ -88,7 +88,7 @@ def test_replicate_with_config_option assert_equal "--config", argv[0] assert_equal "CONFIG", argv[1] end - Litestream::Commands.stub :run_async, stub do + Litestream::Commands.stub :run_replicate, stub do Litestream::Commands.replicate("--config" => "CONFIG") end end @@ -96,7 +96,7 @@ def test_replicate_with_config_option def test_replicate_sets_replica_bucket_env_var_from_config_when_env_var_not_set Litestream.replica_bucket = "mybkt" - Litestream::Commands.stub :run_async, nil do + Litestream::Commands.stub :run_replicate, nil do Litestream::Commands.replicate end @@ -108,7 +108,7 @@ def test_replicate_sets_replica_bucket_env_var_from_config_when_env_var_not_set def test_replicate_sets_replica_key_id_env_var_from_config_when_env_var_not_set Litestream.replica_key_id = "mykey" - Litestream::Commands.stub :run_async, nil do + Litestream::Commands.stub :run_replicate, nil do Litestream::Commands.replicate end @@ -120,7 +120,7 @@ def test_replicate_sets_replica_key_id_env_var_from_config_when_env_var_not_set def test_replicate_sets_replica_access_key_env_var_from_config_when_env_var_not_set Litestream.replica_access_key = "access" - Litestream::Commands.stub :run_async, nil do + Litestream::Commands.stub :run_replicate, nil do Litestream::Commands.replicate end @@ -134,7 +134,7 @@ def test_replicate_sets_all_env_vars_from_config_when_env_vars_not_set Litestream.replica_key_id = "mykey" Litestream.replica_access_key = "access" - Litestream::Commands.stub :run_async, nil do + Litestream::Commands.stub :run_replicate, nil do Litestream::Commands.replicate end @@ -152,7 +152,7 @@ def test_replicate_does_not_set_env_var_from_config_when_env_vars_already_set Litestream.replica_key_id = "mykey" Litestream.replica_access_key = "access" - Litestream::Commands.stub :run_async, nil do + Litestream::Commands.stub :run_replicate, nil do Litestream::Commands.replicate end