diff --git a/config/deploy.rb b/config/deploy.rb index f6f10bb7..ad8870a3 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -147,8 +147,9 @@ # https://www.phusionpassenger.com/library/config/standalone/reference/#--max-requests-max_requests execute "cd #{deploy_to}/current; bundle exec passenger start -d --environment #{fetch(:rails_env)} "\ "--pid-file #{fetch(:passenger_pid)} -p #{fetch(:passenger_port)} "\ - "--log-file #{fetch(:passenger_log)} --pool-idle-time 86400 --max-pool-size=#{fetch(:passenger_pool)} "\ - "--max-requests=500" + "--log-file #{fetch(:passenger_log)} --pool-idle-time 86400 --max-pool-size=#{fetch(:passenger_pool)} " + # "--max-requests=500" -- can't do this since it kills submissions, but we should manually kill if a process gets too + # big and seems to have a memory leak, unfortunately, this feature in passenger is a "pro" feature and isn't free. end end end diff --git a/lib/tasks/dev_ops.rake b/lib/tasks/dev_ops.rake index 5aafc276..6b4c0875 100644 --- a/lib/tasks/dev_ops.rake +++ b/lib/tasks/dev_ops.rake @@ -1,4 +1,5 @@ require 'yaml' +require_relative 'dev_ops/passenger' # rubocop:disable Metrics/BlockLength namespace :dev_ops do @@ -90,5 +91,17 @@ namespace :dev_ops do exec command end + desc 'Kill large memory usage passenger processes' + task kill_bloated_passengers: :environment do + passenger = DevOps::Passenger.new + + passenger.kill_bloated_pids! unless passenger.items_submitting? + + # puts "passenger.status: #{passenger.status}" + # puts "out: \n #{passenger.stdout}" + # puts passenger.bloated_pids + # puts passenger.items_submitting? + end + end # rubocop:enable Metrics/BlockLength diff --git a/lib/tasks/dev_ops/passenger.rb b/lib/tasks/dev_ops/passenger.rb new file mode 100644 index 00000000..e7501f41 --- /dev/null +++ b/lib/tasks/dev_ops/passenger.rb @@ -0,0 +1,52 @@ +require 'open3' +require 'byebug' + +module DevOps + class Passenger + + attr_reader :status, :stdout + + BLOATED_MB = 600 + + def initialize + @stdout, @stderr, @status = Open3.capture3('passenger-status') + @bloated_pids = [] + end + + def not_running? + @stderr.include?("Phusion Passenger doesn't seem to be running") + end + + # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def bloated_pids + # return (empty) array if not running or return array it is already cached with bloated pids + return @bloated_pids if not_running? || @bloated_pids.length.positive? + + process_sections = @stdout.split('* PID: ') + process_sections = process_sections[1..-1] # the second to the last of the split sections + + process_sections.each do |section| + matches = section.match(/^(\d+).+Memory *: *(\S+)/m) + pid = matches[1] + memory = 0 + memory = matches[2].to_i if matches[2].end_with?('M') + memory = matches[2].to_i * 1000 if matches[2].end_with?('G') + @bloated_pids.push(pid) if memory > BLOATED_MB + end + @bloated_pids + end + # rubocop:enable Metrics/AbcSize, Metrics/MethodLength + + def kill_bloated_pids! + bloated_pids.each do |my_pid| + `kill #{my_pid}` + end + end + + def items_submitting? + StashEngine::RepoQueueState.latest_per_resource.where(state: 'processing') + .where(hostname: StashEngine.repository.class.hostname).count.positive? + end + + end +end