diff --git a/Gemfile b/Gemfile
index 0277a6f8b..b763bd9f4 100644
--- a/Gemfile
+++ b/Gemfile
@@ -52,6 +52,8 @@ gem "resque", "~> 2.0"
gem "resque-pool"
gem "resque-heroku-signals" # gah, weirdly needed for graceful shutdown on heroku. https://github.com/resque/resque#heroku
+gem "good_job", "~> 3.0"
+
gem 'honeybadger', '~> 4.0'
# Until we get things working under sprockets 4, lock to sprockets 3
diff --git a/Gemfile.lock b/Gemfile.lock
index 4a688eb70..95369a76b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -85,14 +85,14 @@ GEM
autoprefixer-rails (10.4.2.0)
execjs (~> 2)
aws-eventstream (1.2.0)
- aws-partitions (1.590.0)
+ aws-partitions (1.604.0)
aws-sdk-cloudwatchevents (1.57.0)
aws-sdk-core (~> 3, >= 3.127.0)
aws-sigv4 (~> 1.1)
aws-sdk-cloudwatchlogs (1.53.0)
aws-sdk-core (~> 3, >= 3.127.0)
aws-sigv4 (~> 1.1)
- aws-sdk-core (3.131.1)
+ aws-sdk-core (3.131.2)
aws-eventstream (~> 1, >= 1.0.2)
aws-partitions (~> 1, >= 1.525.0)
aws-sigv4 (~> 1.1)
@@ -100,7 +100,7 @@ GEM
aws-sdk-kms (1.57.0)
aws-sdk-core (~> 3, >= 3.127.0)
aws-sigv4 (~> 1.1)
- aws-sdk-mediaconvert (1.89.0)
+ aws-sdk-mediaconvert (1.92.0)
aws-sdk-core (~> 3, >= 3.127.0)
aws-sigv4 (~> 1.1)
aws-sdk-s3 (1.114.0)
@@ -109,10 +109,10 @@ GEM
aws-sigv4 (~> 1.4)
aws-sigv4 (1.5.0)
aws-eventstream (~> 1, >= 1.0.2)
- axe-core-api (4.4.0)
+ axe-core-api (4.4.1)
dumb_delegator
virtus
- axe-core-rspec (4.4.0)
+ axe-core-rspec (4.4.1)
axe-core-api
dumb_delegator
virtus
@@ -131,9 +131,9 @@ GEM
ostruct (>= 0.3.2)
rails (>= 5.1, < 7.1)
view_component (~> 2.43)
- blacklight_range_limit (8.1.0)
- blacklight (~> 7.0)
- bootsnap (1.11.1)
+ blacklight_range_limit (8.2.2)
+ blacklight (>= 7.22.2, < 9)
+ bootsnap (1.12.0)
msgpack (~> 1.2)
bootstrap (4.6.1)
autoprefixer-rails (>= 9.1.0)
@@ -142,14 +142,13 @@ GEM
bootstrap4-kaminari-views (1.0.1)
kaminari (>= 0.13)
rails (>= 3.1)
- browse-everything (1.1.2)
+ browse-everything (1.2.0)
addressable (~> 2.5)
aws-sdk-s3
- dropbox_api (>= 0.1.10)
- google-api-client (~> 0.23)
- google_drive (>= 2.1, < 4)
- googleauth (>= 0.6.6, < 1.0)
- rails (>= 4.2, < 7.0)
+ dropbox_api (>= 0.1.20)
+ google-apis-drive_v3
+ googleauth (>= 0.6.6, < 2.0)
+ rails (>= 4.2, < 7.1)
ruby-box
signet (~> 0.8)
typhoeus
@@ -224,6 +223,8 @@ GEM
oauth2 (~> 1.1)
dumb_delegator (1.0.0)
erubi (1.10.0)
+ et-orbi (1.2.7)
+ tzinfo
ethon (0.15.0)
ffi (>= 1.15.0)
execjs (2.8.1)
@@ -250,8 +251,8 @@ GEM
faraday-follow_redirects (0.3.0)
faraday (>= 1, < 3)
faraday-httpclient (1.0.1)
- faraday-multipart (1.0.3)
- multipart-post (>= 1.2, < 3)
+ faraday-multipart (1.0.4)
+ multipart-post (~> 2)
faraday-net_http (1.0.1)
faraday-net_http_persistent (1.2.0)
faraday-patron (1.0.0)
@@ -265,17 +266,25 @@ GEM
rake
font-awesome-rails (4.7.0.8)
railties (>= 3.2, < 8.0)
+ fugit (1.5.3)
+ et-orbi (~> 1, >= 1.2.7)
+ raabro (~> 1.4)
fx (0.7.0)
activerecord (>= 4.0.0)
railties (>= 4.0.0)
- gems (1.2.0)
geocoder (1.8.0)
globalid (1.0.0)
activesupport (>= 5.0)
- google-api-client (0.53.0)
- google-apis-core (~> 0.1)
- google-apis-generator (~> 0.1)
- google-apis-core (0.5.0)
+ good_job (3.1.0)
+ activejob (>= 6.0.0)
+ activerecord (>= 6.0.0)
+ concurrent-ruby (>= 1.0.2)
+ fugit (>= 1.1)
+ railties (>= 6.0.0)
+ thor (>= 0.14.1)
+ webrick (>= 1.3)
+ zeitwerk (>= 2.0)
+ google-apis-core (0.7.0)
addressable (~> 2.5, >= 2.5.1)
googleauth (>= 0.16.2, < 2.a)
httpclient (>= 2.8.1, < 3.a)
@@ -284,30 +293,15 @@ GEM
retriable (>= 2.0, < 4.a)
rexml
webrick
- google-apis-discovery_v1 (0.8.0)
- google-apis-core (>= 0.4, < 2.a)
- google-apis-drive_v3 (0.22.0)
- google-apis-core (>= 0.4, < 2.a)
- google-apis-generator (0.4.1)
- activesupport (>= 5.0)
- gems (~> 1.2)
- google-apis-core (>= 0.4, < 2.a)
- google-apis-discovery_v1 (~> 0.5)
- thor (>= 0.20, < 2.a)
- google-apis-sheets_v4 (0.13.0)
- google-apis-core (>= 0.4, < 2.a)
- google_drive (3.0.7)
- google-apis-drive_v3 (>= 0.5.0, < 1.0.0)
- google-apis-sheets_v4 (>= 0.4.0, < 1.0.0)
- googleauth (>= 0.5.0, < 1.0.0)
- nokogiri (>= 1.5.3, < 2.0.0)
- googleauth (0.17.1)
- faraday (>= 0.17.3, < 2.0)
+ google-apis-drive_v3 (0.25.0)
+ google-apis-core (>= 0.7, < 2.a)
+ googleauth (1.2.0)
+ faraday (>= 0.17.3, < 3.a)
jwt (>= 1.4, < 3.0)
memoist (~> 0.16)
multi_json (~> 1.11)
os (>= 0.9, < 2.0)
- signet (~> 0.15)
+ signet (>= 0.16, < 2.a)
hashdiff (1.0.1)
hashery (2.1.2)
hashie (5.0.0)
@@ -315,16 +309,16 @@ GEM
honeybadger (4.12.1)
html_aware_truncation (1.0.0)
nokogiri (~> 1.0)
- http (5.0.4)
+ http (5.1.0)
addressable (~> 2.8)
http-cookie (~> 1.0)
http-form_data (~> 2.2)
llhttp-ffi (~> 0.4.0)
- http-cookie (1.0.4)
+ http-cookie (1.0.5)
domain_name (~> 0.5)
http-form_data (2.3.0)
httpclient (2.8.3)
- i18n (1.10.0)
+ i18n (1.11.0)
concurrent-ruby (~> 1.0)
ice_nine (0.11.2)
io-console (0.5.11)
@@ -339,7 +333,7 @@ GEM
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
json (2.6.2)
- jwt (2.3.0)
+ jwt (2.4.1)
kaminari (1.2.2)
activesupport (>= 4.1.0)
kaminari-actionview (= 1.2.2)
@@ -369,10 +363,11 @@ GEM
tty-command (>= 0.8.2, < 2)
launchy (2.5.0)
addressable (~> 2.7)
- ldpath (1.1.0)
+ ldpath (1.2.0)
nokogiri (~> 1.8)
parslet
rdf (~> 3.0)
+ rdf-vocab (~> 3.0)
link_header (0.0.8)
listen (3.7.1)
rb-fsevent (~> 0.10, >= 0.10.3)
@@ -380,7 +375,7 @@ GEM
llhttp-ffi (0.4.0)
ffi-compiler (~> 1.0)
rake (~> 13.0)
- lockbox (0.6.8)
+ lockbox (1.0.0)
lograge (0.12.0)
actionpack (>= 4)
activesupport (>= 4)
@@ -406,12 +401,12 @@ GEM
method_source (1.0.0)
mini_mime (1.1.2)
mini_portile2 (2.8.0)
- minitest (5.15.0)
+ minitest (5.16.2)
mono_logger (1.1.1)
- msgpack (1.5.1)
+ msgpack (1.5.3)
multi_json (1.15.0)
multi_xml (0.6.0)
- multipart-post (2.1.1)
+ multipart-post (2.2.3)
mustermann (1.1.1)
ruby2_keywords (~> 0.0.1)
namae (1.1.1)
@@ -430,14 +425,14 @@ GEM
net-protocol
timeout
nio4r (2.5.8)
- nokogiri (1.13.6)
+ nokogiri (1.13.7)
mini_portile2 (~> 2.8.0)
racc (~> 1.4)
oai (1.2.1)
builder (>= 3.1.0)
faraday (< 3)
faraday-follow_redirects (>= 0.3.0, < 2)
- oauth2 (1.4.9)
+ oauth2 (1.4.10)
faraday (>= 0.17.3, < 3.0)
jwt (>= 1.0, < 3.0)
multi_json (~> 1.3)
@@ -461,12 +456,12 @@ GEM
hashery (~> 2.0)
ruby-rc4
ttfunk
- pg (1.3.5)
+ pg (1.4.1)
popper_js (1.16.0)
prawn (2.4.0)
pdf-core (~> 0.9.0)
ttfunk (~> 1.7)
- prawn-html (0.6.4)
+ prawn-html (0.7.0)
oga (~> 3.3)
prawn (~> 2.4)
prawn-svg (0.32.0)
@@ -491,14 +486,15 @@ GEM
nokogiri (~> 1.6)
rails (>= 5.0, < 6.2)
rdf
+ raabro (1.4.0)
racc (1.6.0)
- rack (2.2.3.1)
+ rack (2.2.4)
rack-protection (2.2.0)
rack
rack-proxy (0.7.2)
rack
- rack-test (1.1.0)
- rack (>= 1.0, < 3)
+ rack-test (2.0.2)
+ rack (>= 1.3)
rails (6.1.6)
actioncable (= 6.1.6)
actionmailbox (= 6.1.6)
@@ -521,7 +517,7 @@ GEM
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
- rails-html-sanitizer (1.4.2)
+ rails-html-sanitizer (1.4.3)
loofah (~> 2.3)
railties (6.1.6)
actionpack (= 6.1.6)
@@ -530,19 +526,21 @@ GEM
rake (>= 12.2)
thor (~> 1.0)
rake (13.0.6)
- ransack (3.2.0)
+ ransack (3.2.1)
activerecord (>= 6.1.5)
activesupport (>= 6.1.5)
i18n
rb-fsevent (0.11.1)
rb-inotify (0.10.1)
ffi (~> 1.0)
- rdf (3.2.7)
+ rdf (3.2.8)
link_header (~> 0.0, >= 0.0.8)
- redis (4.6.0)
+ rdf-vocab (3.2.1)
+ rdf (~> 3.2, >= 3.2.4)
+ redis (4.7.1)
redis-namespace (1.8.2)
redis (>= 3.0.4)
- regexp_parser (2.4.0)
+ regexp_parser (2.5.0)
reline (0.3.1)
io-console (~> 0.5)
representable (3.2.0)
@@ -567,7 +565,7 @@ GEM
retriable (3.1.2)
rexml (3.2.5)
rinku (2.0.6)
- roda (3.56.0)
+ roda (3.57.0)
rack
rsolr (2.5.0)
builder (>= 2.1.2)
@@ -614,13 +612,14 @@ GEM
sprockets (> 3.0)
sprockets-rails
tilt
- scout_apm (5.1.1)
+ scout_apm (5.2.0)
parser
scrub_rb (1.0.1)
- selenium-webdriver (4.1.0)
+ selenium-webdriver (4.3.0)
childprocess (>= 0.5, < 5.0)
rexml (~> 3.2, >= 3.2.5)
- rubyzip (>= 1.2.2)
+ rubyzip (>= 1.2.2, < 3.0)
+ websocket (~> 1.0)
semantic_range (3.0.0)
shrine (3.4.0)
content_disposition (~> 1.0)
@@ -629,9 +628,9 @@ GEM
down (~> 5.0)
http (>= 3.2, < 6)
shrine (>= 3.0.0.rc, < 4)
- signet (0.16.1)
+ signet (0.17.0)
addressable (~> 2.8)
- faraday (>= 0.17.5, < 3.0)
+ faraday (>= 0.17.5, < 3.a)
jwt (>= 1.5, < 3.0)
multi_json (~> 1.10)
simple_form (5.1.0)
@@ -650,7 +649,7 @@ GEM
retriable
ruby-progressbar
rubyzip
- sprockets (4.0.3)
+ sprockets (4.1.1)
concurrent-ruby (~> 1.0)
rack (> 1, < 3)
sprockets-rails (3.4.2)
@@ -661,7 +660,7 @@ GEM
thor (1.2.1)
thread_safe (0.3.6)
tilt (2.0.10)
- timeout (0.2.0)
+ timeout (0.3.0)
trailblazer-option (0.1.2)
traject (3.7.0)
concurrent-ruby (>= 0.8.0)
@@ -687,12 +686,12 @@ GEM
execjs (>= 0.3.0, < 3)
unf (0.1.4)
unf_ext
- unf_ext (0.0.8.1)
+ unf_ext (0.0.8.2)
uppy-s3_multipart (1.2.0)
aws-sdk-s3 (~> 1.0)
content_disposition (~> 1.0)
roda (>= 2.27, < 4)
- view_component (2.55.0)
+ view_component (2.57.1)
activesupport (>= 5.0.0, < 8.0)
method_source (~> 1.0)
virtus (2.0.0)
@@ -721,13 +720,14 @@ GEM
railties (>= 5.2)
semantic_range (>= 2.3.0)
webrick (1.7.0)
+ websocket (1.2.9)
websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
xpath (3.2.0)
nokogiri (~> 1.8)
yell (2.2.2)
- zeitwerk (2.5.4)
+ zeitwerk (2.6.0)
PLATFORMS
ruby
@@ -763,6 +763,7 @@ DEPENDENCIES
factory_bot_rails
faster_s3_url (< 2)
font-awesome-rails (~> 4.7)
+ good_job (~> 3.0)
hirefire-resource
honeybadger (~> 4.0)
html_aware_truncation (~> 1.0)
diff --git a/app/views/layouts/admin.html.erb b/app/views/layouts/admin.html.erb
index e212789d5..c096a9048 100644
--- a/app/views/layouts/admin.html.erb
+++ b/app/views/layouts/admin.html.erb
@@ -77,7 +77,8 @@
<%= link_to "Interviewee Biographies", admin_interviewee_biographies_path %>
- <%= link_to "Job Queues", admin_resque_server_path %>
+ <%= link_to "Job Queues (resque)", admin_resque_server_path %>
+ <%= link_to "Job Queues (good_job)", admin_good_job_path %>
<%= link_to "Fixity Report", admin_fixity_report_path %>
<%= link_to "Storage Report", admin_storage_report_path %>
<%= link_to "Orphan Report", admin_orphan_report_path %>
diff --git a/config/application.rb b/config/application.rb
index a49091dab..56154c503 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -13,6 +13,8 @@
require "sprockets/railtie"
require "rails/test_unit/railtie"
+# Enable good_job dashboard
+require 'good_job/engine'
# Require the gems listed in Gemfile, including any gems
# you've limited to :test, :development, or :production.
diff --git a/config/environments/development.rb b/config/environments/development.rb
index e86b3c85c..ef1b68be5 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -17,6 +17,10 @@
# sassc-rails
config.sass.inline_source_maps = true
+ # For some consistency, use good_job in development too, although by
+ # default in development it'll be running in an in-process execution mode
+ config.active_job.queue_adapter = :good_job
+
# Enable/disable caching. By default caching is disabled.
# Run rails dev:cache to toggle caching.
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 13b3fadcf..4e047d16f 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -77,7 +77,7 @@
# config.cache_store = :mem_cache_store
# Use a real queuing backend for Active Job (and separate queues per environment)
- config.active_job.queue_adapter = :resque
+ config.active_job.queue_adapter = :good_job
# We are not sharing a redis among multiple apps, seems no need to queue_name_prefix,
# and it makes it confusing when trying to set resque workers to work specific
diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb
new file mode 100644
index 000000000..98ec34387
--- /dev/null
+++ b/config/initializers/good_job.rb
@@ -0,0 +1,17 @@
+Rails.application.configure do
+ # Do NOT retry errors that make it to good_job.
+ #
+ # This kind of ALWAYS needs to be false to avoid infinite loop on erroring job,
+ # why doesn't it default to false??
+ #
+ # https://github.com/bensheldon/good_job#retries
+ config.good_job.retry_on_unhandled_error = false
+
+ # And when we do manually clear old job completion records, we do NOT
+ # want to clear any currently listed as "discarded" (ie failed)
+ config.good_job.cleanup_discarded_jobs = false
+
+ # let's keep job records for two weeks, our small volume of jobs
+ # should easily accomodate that. default was one day.
+ config.cleanup_preserved_jobs_before_seconds_ago = 2.weeks.to_i
+end
diff --git a/config/routes.rb b/config/routes.rb
index 2525415db..9f25b3309 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -337,6 +337,8 @@ def self.matches?(request)
if Shrine.storages[:cache].kind_of?(Shrine::Storage::S3)
mount Shrine.uppy_s3_multipart(:cache) => "/s3"
end
+
+ mount GoodJob::Engine => 'good_job'
end
end
diff --git a/db/migrate/20220622141336_create_good_jobs.rb b/db/migrate/20220622141336_create_good_jobs.rb
new file mode 100644
index 000000000..b45567bfa
--- /dev/null
+++ b/db/migrate/20220622141336_create_good_jobs.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+class CreateGoodJobs < ActiveRecord::Migration[6.1]
+ def change
+ enable_extension 'pgcrypto'
+
+ create_table :good_jobs, id: :uuid do |t|
+ t.text :queue_name
+ t.integer :priority
+ t.jsonb :serialized_params
+ t.timestamp :scheduled_at
+ t.timestamp :performed_at
+ t.timestamp :finished_at
+ t.text :error
+
+ t.timestamps
+
+ t.uuid :active_job_id
+ t.text :concurrency_key
+ t.text :cron_key
+ t.uuid :retried_good_job_id
+ t.timestamp :cron_at
+ end
+
+ create_table :good_job_processes, id: :uuid do |t|
+ t.timestamps
+ t.jsonb :state
+ end
+
+ add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: "index_good_jobs_on_scheduled_at"
+ add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at
+ add_index :good_jobs, [:active_job_id, :created_at], name: :index_good_jobs_on_active_job_id_and_created_at
+ add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", name: :index_good_jobs_on_concurrency_key_when_unfinished
+ add_index :good_jobs, [:cron_key, :created_at], name: :index_good_jobs_on_cron_key_and_created_at
+ add_index :good_jobs, [:cron_key, :cron_at], name: :index_good_jobs_on_cron_key_and_cron_at, unique: true
+ add_index :good_jobs, [:active_job_id], name: :index_good_jobs_on_active_job_id
+ add_index :good_jobs, [:finished_at], where: "retried_good_job_id IS NULL AND finished_at IS NOT NULL", name: :index_good_jobs_jobs_on_finished_at
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 3fbc058d7..cbe110478 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,8 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2022_05_26_194233) do
+ActiveRecord::Schema.define(version: 2022_06_22_141336) do
+
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@@ -128,6 +129,37 @@
t.index ["checked_uri"], name: "index_fixity_checks_on_checked_uri"
end
+ create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+ t.datetime "created_at", precision: 6, null: false
+ t.datetime "updated_at", precision: 6, null: false
+ t.jsonb "state"
+ end
+
+ create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+ t.text "queue_name"
+ t.integer "priority"
+ t.jsonb "serialized_params"
+ t.datetime "scheduled_at"
+ t.datetime "performed_at"
+ t.datetime "finished_at"
+ t.text "error"
+ t.datetime "created_at", precision: 6, null: false
+ t.datetime "updated_at", precision: 6, null: false
+ t.uuid "active_job_id"
+ t.text "concurrency_key"
+ t.text "cron_key"
+ t.uuid "retried_good_job_id"
+ t.datetime "cron_at"
+ t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at"
+ t.index ["active_job_id"], name: "index_good_jobs_on_active_job_id"
+ t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)"
+ t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at"
+ t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at", unique: true
+ t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))"
+ t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)"
+ t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)"
+ end
+
create_table "interviewee_biographies", force: :cascade do |t|
t.string "name", null: false
t.jsonb "json_attributes"