Merge pull request #171 from CDL-Dryad/passenger-killer

Take out the passenger startup to kill after request and script to kill bloat

config/deploy.rb

@@ -147,8 +147,9 @@
           # https://www.phusionpassenger.com/library/config/standalone/reference/#--max-requests-max_requests
           execute "cd #{deploy_to}/current; bundle exec passenger start -d --environment #{fetch(:rails_env)} "\
               "--pid-file #{fetch(:passenger_pid)} -p #{fetch(:passenger_port)} "\
-              "--log-file #{fetch(:passenger_log)} --pool-idle-time 86400 --max-pool-size=#{fetch(:passenger_pool)} "\
-              "--max-requests=500"
+              "--log-file #{fetch(:passenger_log)} --pool-idle-time 86400 --max-pool-size=#{fetch(:passenger_pool)} "
+              # "--max-requests=500" -- can't do this since it kills submissions, but we should manually kill if a process gets too
+              # big and seems to have a memory leak, unfortunately, this feature in passenger is a "pro" feature and isn't free.
         end
       end
     end

lib/tasks/dev_ops.rake

@@ -1,4 +1,5 @@
 require 'yaml'
+require_relative 'dev_ops/passenger'
 
 # rubocop:disable Metrics/BlockLength
 namespace :dev_ops do
@@ -90,5 +91,17 @@ namespace :dev_ops do
     exec command
   end
 
+  desc 'Kill large memory usage passenger processes'
+  task kill_bloated_passengers: :environment do
+    passenger = DevOps::Passenger.new
+
+    passenger.kill_bloated_pids! unless passenger.items_submitting?
+
+    # puts "passenger.status: #{passenger.status}"
+    # puts "out: \n #{passenger.stdout}"
+    # puts passenger.bloated_pids
+    # puts passenger.items_submitting?
+  end
+
 end
 # rubocop:enable Metrics/BlockLength

lib/tasks/dev_ops/passenger.rb

@@ -0,0 +1,52 @@
+require 'open3'
+require 'byebug'
+
+module DevOps
+  class Passenger
+
+    attr_reader :status, :stdout
+
+    BLOATED_MB = 600
+
+    def initialize
+      @stdout, @stderr, @status = Open3.capture3('passenger-status')
+      @bloated_pids = []
+    end
+
+    def not_running?
+      @stderr.include?("Phusion Passenger doesn't seem to be running")
+    end
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def bloated_pids
+      # return (empty) array if not running or return array it is already cached with bloated pids
+      return @bloated_pids if not_running? || @bloated_pids.length.positive?
+
+      process_sections = @stdout.split('* PID: ')
+      process_sections = process_sections[1..-1] # the second to the last of the split sections
+
+      process_sections.each do |section|
+        matches = section.match(/^(\d+).+Memory *: *(\S+)/m)
+        pid = matches[1]
+        memory = 0
+        memory = matches[2].to_i if matches[2].end_with?('M')
+        memory = matches[2].to_i * 1000 if matches[2].end_with?('G')
+        @bloated_pids.push(pid) if memory > BLOATED_MB
+      end
+      @bloated_pids
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    def kill_bloated_pids!
+      bloated_pids.each do |my_pid|
+        `kill #{my_pid}`
+      end
+    end
+
+    def items_submitting?
+      StashEngine::RepoQueueState.latest_per_resource.where(state: 'processing')
+        .where(hostname: StashEngine.repository.class.hostname).count.positive?
+    end
+
+  end
+end