- Background processes
-
- <%= Seek::Workers.active_queues.count %> expected
-
+ Background job workers
+ <% if flash[:job_workers_restarting] %>
+
+ The background job workers are restarting - this can take a few moments.
+ <%= link_to 'Refresh', admin_path %> to check their status.
+
+ <% end %>
+
<% begin %>
- <% pids = Seek::Util.delayed_job_pids %>
- <% if pids.any? %>
-
- <% pids.each do |pid| %>
- -
- <% if pid.running? %>
- Running
- (Process ID: <%= pid.pid -%>)
- <% else %>
- Not running
- <% end %>
-
+ <% pid = Seek::Util.solid_queue_supervisor_pid %>
+ <% if pid %>
+ Running
+ (Process ID: <%= pid -%>)
+
+ <% alive_since = SolidQueue.process_alive_threshold.ago %>
+ <% active_queues = SolidQueue::Process.where(kind: 'Worker').where('last_heartbeat_at > ?', alive_since)
+ .flat_map { |p| p.metadata['queues'].to_s.split(',') }
+ .map(&:strip).reject(&:blank?).sort %>
+
+ Active queues (<%= active_queues.count -%>):
+
+
+ <% active_queues.each do |queue_name| %>
+ - <%= queue_name -%>
<% end %>
-
+
<% else %>
-
No background processes running
+
No background job workers running
<% end %>
<% rescue StandardError => e %>
Unable to determine current status - <%= e.message %>
diff --git a/app/views/admin/index.html.erb b/app/views/admin/index.html.erb
index 611c689514..2efc7903d9 100644
--- a/app/views/admin/index.html.erb
+++ b/app/views/admin/index.html.erb
@@ -16,4 +16,6 @@
<%= render partial: 'admin/restart_buttons' %>
+
<%= link_to 'Job queue dashboard', mission_control_jobs_path %> – inspect and manage background jobs (Solid Queue).
+
<%= git_link_tag %>
diff --git a/app/views/admin/stats/_job_queue.html.erb b/app/views/admin/stats/_job_queue.html.erb
index 9c5ad64cc4..56831e5407 100644
--- a/app/views/admin/stats/_job_queue.html.erb
+++ b/app/views/admin/stats/_job_queue.html.erb
@@ -1,14 +1,16 @@
<%
queue = AuthLookupUpdateQueue.all.to_a | ReindexingQueue.all.to_a | RdfGenerationQueue.all.to_a
queue = queue.sort_by(&:created_at)
- delayed_jobs = Delayed::Job.order(locked_at: :desc, created_at: :asc)
+ solid_queue_jobs = SolidQueue::Job.where(finished_at: nil).includes(:failed_execution).order(created_at: :asc)
%>
- Total delayed jobs waiting = <%= delayed_jobs.count -%>
+ Total jobs waiting = <%= solid_queue_jobs.count -%>
-<% if Delayed::Job.where('failed_at IS NOT NULL').count > 0 %>
+
<%= link_to 'Open the Job queue dashboard', mission_control_jobs_path %> for a fuller view of queues, failed and scheduled jobs.
+
+<% if SolidQueue::Job.failed.count > 0 %>
<%= button_link_to "Clear failed jobs", 'destroy', '#', id: 'clear-failed-btn', class: 'btn-primary' %>
<% end %>
@@ -19,34 +21,35 @@
Time created |
Priority |
Queue |
-
Attempts |
-
Run at |
-
Locked at |
-
Failed at |
-
Handler |
+
Class |
+
Scheduled at |
+
Failed |
- <% delayed_jobs.each do |item| %>
+ <% solid_queue_jobs.each do |item| %>
+ <% failure = item.failed_execution %>
| <%= date_as_string(item.created_at,true) -%> |
<%= item.priority -%> |
- <%= item.queue -%> |
- <%= item.attempts -%> |
- <%= date_as_string(item.run_at,true) -%> |
- <%= date_as_string(item.locked_at,true) -%> |
+ <%= item.queue_name -%> |
+ <%= item.class_name -%> |
+ <%= date_as_string(item.scheduled_at,true) -%> |
- <%= item.failed_at.nil? ? date_as_string(item.failed_at,true) : link_to(date_as_string(item.failed_at,true), '#', onclick: "$j('#last_error_#{item.id}').fadeIn(); return false;") -%>
+ <% if failure %>
+ <%= link_to 'Yes', '#', onclick: "$j('#last_error_#{item.id}').fadeIn(); return false;" %>
+ <% end %>
|
- <%= item.handler -%> |
- <% unless item.last_error.nil? -%>
+ <% if failure -%>
-
- <%= item.last_error -%>
+ |
+ <%# failed_execution.error is a JSON hash (exception_class/message/backtrace),
+ so format it into a readable string rather than dumping the raw hash %>
+ <%= [ "#{failure.exception_class}: #{failure.message}", *failure.backtrace ].join("\n") -%>
|
- <% end %>
+ <% end -%>
<% end -%>
@@ -102,4 +105,4 @@
}
});
-
\ No newline at end of file
+
diff --git a/app/views/statistics/application_status.text.erb b/app/views/statistics/application_status.text.erb
index d3c98d4302..df629c8d04 100644
--- a/app/views/statistics/application_status.text.erb
+++ b/app/views/statistics/application_status.text.erb
@@ -5,4 +5,4 @@
end
search = status.search_enabled ? "enabled" : "disabled"
%>
-<%= Seek::Config.instance_name %> is running | search is <%= search %> | <%= status.running_jobs %> delayed jobs running
\ No newline at end of file
+<%= Seek::Config.instance_name %> is running | search is <%= search %> | <%= status.running_jobs %> background job worker processes running
\ No newline at end of file
diff --git a/bin/jobs b/bin/jobs
new file mode 100755
index 0000000000..dcf59f309a
--- /dev/null
+++ b/bin/jobs
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/environment"
+require "solid_queue/cli"
+
+SolidQueue::Cli.start(ARGV)
diff --git a/config/application.rb b/config/application.rb
index 72a7a13617..cfd0e6e67f 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -75,7 +75,7 @@ class Application < Rails::Application
config.action_mailer.deliver_later_queue_name = 'mailers'
- config.active_job.queue_adapter = :delayed_job
+ config.active_job.queue_adapter = :solid_queue
# Revert Rails 7 change that auto loads nested locale files
initializer :avoid_nested_locale_directories, before: :add_locales do
diff --git a/config/initializers/mission_control.rb b/config/initializers/mission_control.rb
new file mode 100644
index 0000000000..e709b1c81a
--- /dev/null
+++ b/config/initializers/mission_control.rb
@@ -0,0 +1,13 @@
+# Mission Control - Jobs: a web dashboard for inspecting and managing Solid Queue jobs, mounted at
+# /jobs in config/routes.rb.
+#
+# Gate it behind SEEK's own admin authentication (see MissionControlJobsController) and turn off the
+# gem's default HTTP Basic auth, so access is controlled the same way as the rest of the admin area.
+#
+# These are set as module attributes rather than via `config.mission_control.jobs.*`: the engine
+# copies that config hash into these same attributes in a `before_initialize` hook, which runs
+# *before* config/initializers/*, so assigning the config object here would be too late to take
+# effect. Assigning the module attributes directly (initializers run after before_initialize) is
+# unambiguous.
+MissionControl::Jobs.base_controller_class = 'MissionControlJobsController'
+MissionControl::Jobs.http_basic_auth_enabled = false
diff --git a/config/initializers/solid_queue.rb b/config/initializers/solid_queue.rb
new file mode 100644
index 0000000000..181536ccc4
--- /dev/null
+++ b/config/initializers/solid_queue.rb
@@ -0,0 +1,10 @@
+# Used by the admin "background job workers" status panel/restart button
+# (app/views/admin/_restart_buttons.html.erb, AdminController#restart_job_workers)
+# to find and signal the running supervisor process.
+SolidQueue.supervisor_pidfile = Rails.root.join('tmp', 'pids', 'solid_queue_supervisor.pid')
+
+# Solid Queue preserves finished jobs (preserve_finished_jobs defaults to true) so they remain
+# visible in the admin/Mission Control dashboards. The clear_finished_jobs recurring task
+# (config/recurring.yml) prunes anything finished longer ago than this - keep 14 days of history
+# rather than the gem default of 1 day.
+SolidQueue.clear_finished_jobs_after = 14.days
diff --git a/config/queue.yml b/config/queue.yml
new file mode 100644
index 0000000000..122f5b502a
--- /dev/null
+++ b/config/queue.yml
@@ -0,0 +1,79 @@
+# Worker topology mirrors QueueNames: one worker per queue, gated by the same Seek::Config
+# feature flags, so that a queue only gets a worker if the corresponding feature is enabled.
+#
+# Threads are deliberately set to 1 per queue for now, since some process-global mutable state
+# (User.current_user, $authorization_checks_disabled) is only safe today because jobs run one
+# at a time per process. Do not raise these thread counts until those globals are made properly
+# thread-local.
+#
+# polling_interval defaults to 1s per worker (Solid Queue's own gem default is 0.1s, but with
+# one worker process per queue - rather than Solid Queue's stock single wildcard-queue worker -
+# that multiplies into a lot of continuous, mostly-empty poll queries). AUTH_LOOKUP is polled
+# faster (0.5s) since it's the highest frequency, most latency-sensitive queue (see
+# AuthLookupUpdateJob's follow_on_job? re-queuing).
+default: &default
+ dispatchers:
+ - polling_interval: 1
+ batch_size: 500
+ workers:
+ - queues: <%= QueueNames::DEFAULT %>
+ threads: 1
+ processes: 1
+ polling_interval: 1
+ - queues: <%= QueueNames::MAILERS %>
+ threads: 1
+ processes: 1
+ polling_interval: 1
+ # config/recurring.yml entries with a `command:` and no `class:` (e.g. clear_finished_jobs,
+ # application_status_refresh) are run by SolidQueue::RecurringJob, whose own queue_as is
+ # `solid_queue_recurring` - none of our named QueueNames queues. Without a worker for it here,
+ # those tasks would enqueue but never be picked up.
+ - queues: solid_queue_recurring
+ threads: 1
+ processes: 1
+ polling_interval: 1
+ <% if Seek::Config.auth_lookup_enabled %>
+ - queues: <%= QueueNames::AUTH_LOOKUP %>
+ threads: 1
+ processes: 1
+ polling_interval: 0.5
+ <% end %>
+ <% if Seek::Config.cache_remote_files %>
+ - queues: <%= QueueNames::REMOTE_CONTENT %>
+ threads: 1
+ processes: 1
+ polling_interval: 1
+ <% end %>
+ <% if Seek::Config.samples_enabled %>
+ - queues: <%= QueueNames::SAMPLES %>
+ threads: 1
+ processes: 1
+ polling_interval: 1
+ <% end %>
+ <% if Seek::Config.solr_enabled %>
+ - queues: <%= QueueNames::INDEXING %>
+ threads: 1
+ processes: 1
+ polling_interval: 1
+ <% end %>
+ <% if Seek::Config.isa_json_compliance_enabled %>
+ - queues: <%= QueueNames::TEMPLATES %>
+ threads: 1
+ processes: 1
+ polling_interval: 1
+ <% end %>
+ <% if Seek::Config.data_files_enabled %>
+ - queues: <%= QueueNames::DATAFILES %>
+ threads: 1
+ processes: 1
+ polling_interval: 1
+ <% end %>
+
+development:
+ <<: *default
+
+test:
+ <<: *default
+
+production:
+ <<: *default
diff --git a/config/recurring.yml b/config/recurring.yml
new file mode 100644
index 0000000000..ee7d974687
--- /dev/null
+++ b/config/recurring.yml
@@ -0,0 +1,77 @@
+# Schedules use standard 5-field cron syntax, in UTC:
+#
+# ┌───────────── minute (0-59)
+# │ ┌───────────── hour (0-23)
+# │ │ ┌───────────── day of month (1-31)
+# │ │ │ ┌───────────── month (1-12)
+# │ │ │ │ ┌───────────── day of week (0-6, Sunday = 0)
+# │ │ │ │ │
+# * * * * *
+#
+# `*` means every value, `*/n` every nth value, and `a,b,c` a specific list. So "0 0 * * 0" is
+# midnight every Sunday, and "0 */4 * * *" is on the hour every 4 hours. (Fugit parses these, so a
+# few natural-language forms like "every hour at minute 12" also work.)
+#
+# Recurring entries for scheduled jobs and tasks that map onto Solid Queue, via either a
+# `class:` entry (an ActiveJob class, enqueued with `perform_later`) or a `command:` entry
+# (arbitrary Ruby, run via SolidQueue::RecurringJob#perform's `eval` - not limited to ActiveJob
+# calls, e.g. clear_finished_jobs/application_status_refresh below are plain class method
+# calls). `command:`-only entries (no `class:`) enqueue onto the `solid_queue_recurring` queue
+# (SolidQueue::RecurringJob's own queue_as), which is why config/queue.yml has a dedicated
+# worker for it. All periodic application jobs live here; the only periodic work handled by cron
+# (docker/seek.crontab) is OS-level process reaping (kill-long-running-soffice.sh) that isn't an
+# ActiveJob or Ruby call.
+#
+# The heavier daily jobs are staggered across the early-morning hours (00:00-04:00) so they don't
+# all run at once.
+production:
+ periodic_subscription_email_daily:
+ class: PeriodicSubscriptionEmailJob
+ args: [ daily ]
+ schedule: "0 0 * * *"
+ periodic_subscription_email_weekly:
+ class: PeriodicSubscriptionEmailJob
+ args: [ weekly ]
+ schedule: "0 0 * * 0"
+ periodic_subscription_email_monthly:
+ class: PeriodicSubscriptionEmailJob
+ args: [ monthly ]
+ schedule: "0 0 1 * *"
+ regular_maintenance:
+ class: RegularMaintenanceJob
+ schedule: "0 */4 * * *"
+ auth_lookup_maintenance:
+ class: AuthLookupMaintenanceJob
+ schedule: "0 */8 * * *"
+ cache_overflow_cleanup:
+ class: CacheOverflowCleanupJob
+ schedule: "0 4 * * *"
+ life_monitor_status:
+ class: LifeMonitorStatusJob
+ schedule: "0 2 * * *"
+ news_feed_refresh:
+ class: NewsFeedRefreshJob
+ priority: 3
+ schedule: "<%= NewsFeedRefreshJob.cron_schedule %>"
+ queue_timed_jobs:
+ command: "ApplicationJob.queue_timed_jobs"
+ schedule: "*/10 * * * *"
+ clear_finished_jobs:
+ command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
+ schedule: every hour at minute 12
+ application_status_refresh:
+ command: "ApplicationStatus.instance.refresh"
+ schedule: "* * * * *"
+ galaxy_tool_map_refresh:
+ command: "Galaxy::ToolMap.instance.refresh"
+ schedule: "0 3 * * *"
+ bioschema_data_dump_generate:
+ command: "Seek::BioSchema::DataDump.generate_dumps"
+ schedule: "10 0 * * *"
+ sitemap_refresh:
+ class: SitemapRefreshJob
+ schedule: "45 0 * * *"
+
+# No recurring entries for development/test - matches solid_queue's own default
+# template convention and avoids periodic jobs (subscription emails, etc.) firing
+# unexpectedly on a local machine when `bin/jobs` is run for testing.
diff --git a/config/routes.rb b/config/routes.rb
index 7d67ce69f9..2c127d3133 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -8,6 +8,9 @@
end
mount MagicLamp::Genie, at: (SEEK::Application.config.relative_url_root || '/') + 'magic_lamp' if defined?(MagicLamp)
# mount Teaspoon::Engine, :at => (SEEK::Application.config.relative_url_root || "/") + "teaspoon" if defined?(Teaspoon)
+ # Solid Queue job dashboard, gated behind admin auth via
+ # MissionControlJobsController / config/initializers/mission_control.rb
+ mount MissionControl::Jobs::Engine, at: (SEEK::Application.config.relative_url_root || '/') + 'jobs' if defined?(MissionControl::Jobs::Engine)
# TRS
namespace :ga4gh do
@@ -170,7 +173,7 @@
post :update_home_settings
post :delete_carousel_form
post :restart_server
- post :restart_delayed_job
+ post :restart_job_workers
post :update_admins
post :update_rebrand
post :test_email_configuration
diff --git a/config/schedule.rb b/config/schedule.rb
index 0eef0a743e..e69de29bb2 100644
--- a/config/schedule.rb
+++ b/config/schedule.rb
@@ -1,92 +0,0 @@
-# Use this file to easily define all of your cron jobs.
-#
-# It's helpful, but not entirely necessary to understand cron before proceeding.
-# http://en.wikipedia.org/wiki/Cron
-
-# Example:
-#
-# set :output, "/path/to/my/cron_log.log"
-#
-# every 2.hours do
-# command "/usr/bin/some_great_command"
-# runner "MyModel.some_method"
-# rake "some:great:rake:task"
-# end
-#
-# every 4.days do
-# runner "AnotherModel.prune_old_records"
-# end
-
-# Learn more: http://github.com/javan/whenever
-#
-
-# Set environment variables
-ENV.each_key do |key|
- env key.to_sym, ENV[key]
-end
-
-require File.expand_path(File.dirname(__FILE__) + "/../config/environment") unless defined? SEEK
-
-set :output, "#{path}/log/schedule.log"
-
-MIDNIGHT = Time.now.midnight
-# Apply a static offset, plus an optional configured offset, to run times of periodic jobs.
-# This is to avoid them all occurring at the same time and overloading the server.
-def offset(off_hours)
- off_minutes = Seek::Config.regular_job_offset || 0
- (MIDNIGHT + off_hours.hours + off_minutes.minutes).strftime("%-l:%M%P")
-end
-
-PeriodicSubscriptionEmailJob::DELAYS.each do |frequency, period|
- every period, at: offset(0) do
- runner "PeriodicSubscriptionEmailJob.new('#{frequency}').queue_job"
- end
-end
-
-every RegularMaintenanceJob::RUN_PERIOD, at: offset(1) do
- runner "RegularMaintenanceJob.perform_later"
-end
-
-every AuthLookupMaintenanceJob::RUN_PERIOD, at: offset(1) do
- runner "AuthLookupMaintenanceJob.perform_later"
-end
-
-every CacheOverflowCleanupJob::RUN_PERIOD, at: offset(4) do
- runner "CacheOverflowCleanupJob.perform_later"
-end
-
-every LifeMonitorStatusJob::PERIOD, at: offset(2) do
- runner "LifeMonitorStatusJob.perform_later"
-end
-
-every Seek::Config.home_feeds_cache_timeout.minutes do # Crontab will need to be regenerated if this changes...
- runner "NewsFeedRefreshJob.set(priority: 3).perform_later"
-end
-
-every 10.minutes do
- runner "ApplicationJob.queue_timed_jobs"
-end
-
-every 1.minute do
- runner 'ApplicationStatus.instance.refresh'
-end
-
-every 1.day, at: offset(3) do
- runner 'Galaxy::ToolMap.instance.refresh'
-end
-
-every 1.day, at: '12:10 am' do
- runner "Seek::BioSchema::DataDump.generate_dumps"
-end
-
-# Generate a new sitemap...
-every 1.day, at: '12:45 am' do
- rake "-s sitemap:refresh"
-end
-
-# not safe to automatically add in a non containerised environment
-if Seek::Docker.using_docker?
- every 10.minutes do
- command "sh /seek/script/kill-long-running-soffice.sh"
- end
-end
diff --git a/db/migrate/20260715145804_create_solid_queue_tables.rb b/db/migrate/20260715145804_create_solid_queue_tables.rb
new file mode 100644
index 0000000000..5e3386f5e0
--- /dev/null
+++ b/db/migrate/20260715145804_create_solid_queue_tables.rb
@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+# Adds Solid Queue's tables to the primary database (shared, not a separate queue database).
+# Table definitions mirror solid_queue 1.4.0's bundled db/queue_schema.rb, translated into a
+# regular migration.
+class CreateSolidQueueTables < ActiveRecord::Migration[7.2]
+ def change
+ create_table :solid_queue_jobs do |t|
+ t.string :queue_name, null: false
+ t.string :class_name, null: false
+ t.text :arguments
+ t.integer :priority, default: 0, null: false
+ t.string :active_job_id
+ t.datetime :scheduled_at
+ t.datetime :finished_at
+ t.string :concurrency_key
+ t.timestamps
+
+ t.index :active_job_id
+ t.index :class_name
+ t.index :finished_at
+ t.index %i[queue_name finished_at], name: 'index_solid_queue_jobs_for_filtering'
+ t.index %i[scheduled_at finished_at], name: 'index_solid_queue_jobs_for_alerting'
+ end
+
+ create_table :solid_queue_scheduled_executions do |t|
+ t.references :job, null: false, index: { unique: true }
+ t.string :queue_name, null: false
+ t.integer :priority, default: 0, null: false
+ t.datetime :scheduled_at, null: false
+ t.datetime :created_at, null: false
+
+ t.index %i[scheduled_at priority job_id], name: 'index_solid_queue_dispatch_all'
+ end
+
+ create_table :solid_queue_ready_executions do |t|
+ t.references :job, null: false, index: { unique: true }
+ t.string :queue_name, null: false
+ t.integer :priority, default: 0, null: false
+ t.datetime :created_at, null: false
+
+ t.index %i[priority job_id], name: 'index_solid_queue_poll_all'
+ t.index %i[queue_name priority job_id], name: 'index_solid_queue_poll_by_queue'
+ end
+
+ create_table :solid_queue_claimed_executions do |t|
+ t.references :job, null: false, index: { unique: true }
+ t.bigint :process_id
+ t.datetime :created_at, null: false
+
+ t.index %i[process_id job_id]
+ end
+
+ create_table :solid_queue_blocked_executions do |t|
+ t.references :job, null: false, index: { unique: true }
+ t.string :queue_name, null: false
+ t.integer :priority, default: 0, null: false
+ t.string :concurrency_key, null: false
+ t.datetime :expires_at, null: false
+ t.datetime :created_at, null: false
+
+ t.index %i[expires_at concurrency_key], name: 'index_solid_queue_blocked_executions_for_maintenance'
+ t.index %i[concurrency_key priority job_id], name: 'index_solid_queue_blocked_executions_for_release'
+ end
+
+ create_table :solid_queue_failed_executions do |t|
+ t.references :job, null: false, index: { unique: true }
+ t.text :error
+ t.datetime :created_at, null: false
+ end
+
+ create_table :solid_queue_pauses do |t|
+ t.string :queue_name, null: false
+ t.datetime :created_at, null: false
+
+ t.index :queue_name, unique: true
+ end
+
+ create_table :solid_queue_processes do |t|
+ t.string :kind, null: false
+ t.datetime :last_heartbeat_at, null: false
+ t.bigint :supervisor_id
+ t.integer :pid, null: false
+ t.string :hostname
+ t.text :metadata
+ t.datetime :created_at, null: false
+ t.string :name, null: false
+
+ t.index :last_heartbeat_at
+ t.index %i[name supervisor_id], unique: true
+ t.index :supervisor_id
+ end
+
+ create_table :solid_queue_semaphores do |t|
+ t.string :key, null: false
+ t.integer :value, default: 1, null: false
+ t.datetime :expires_at, null: false
+ t.timestamps
+
+ t.index :key, unique: true
+ t.index %i[key value]
+ t.index :expires_at
+ end
+
+ create_table :solid_queue_recurring_tasks do |t|
+ t.string :key, null: false
+ t.string :schedule, null: false
+ t.string :command, limit: 2048
+ t.string :class_name
+ t.text :arguments
+ t.string :queue_name
+ t.integer :priority, default: 0
+ t.boolean :static, default: true, null: false
+ t.text :description
+ t.timestamps
+
+ t.index :key, unique: true
+ t.index :static
+ end
+
+ create_table :solid_queue_recurring_executions do |t|
+ t.references :job, null: false, index: { unique: true }
+ t.string :task_key, null: false
+ t.datetime :run_at, null: false
+ t.datetime :created_at, null: false
+
+ t.index %i[task_key run_at], unique: true
+ end
+
+ add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+ add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+ add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+ add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+ add_foreign_key :solid_queue_recurring_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+ add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f228a8ee7d..14466edc2e 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.2].define(version: 2026_06_23_150237) do
+ActiveRecord::Schema[7.2].define(version: 2026_07_15_145804) do
create_table "activity_logs", id: :integer, force: :cascade do |t|
t.string "action"
t.string "format"
@@ -1956,6 +1956,134 @@
t.text "description"
end
+ create_table "solid_queue_blocked_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.string "queue_name", null: false
+ t.integer "priority", default: 0, null: false
+ t.string "concurrency_key", null: false
+ t.datetime "expires_at", null: false
+ t.datetime "created_at", null: false
+ t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release"
+ t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance"
+ t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true
+ end
+
+ create_table "solid_queue_claimed_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.bigint "process_id"
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true
+ t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id"
+ end
+
+ create_table "solid_queue_failed_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.text "error"
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true
+ end
+
+ create_table "solid_queue_pauses", force: :cascade do |t|
+ t.string "queue_name", null: false
+ t.datetime "created_at", null: false
+ t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true
+ end
+
+ create_table "solid_queue_processes", force: :cascade do |t|
+ t.string "kind", null: false
+ t.datetime "last_heartbeat_at", null: false
+ t.bigint "supervisor_id"
+ t.integer "pid", null: false
+ t.string "hostname"
+ t.text "metadata"
+ t.datetime "created_at", null: false
+ t.string "name", null: false
+ t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at"
+ t.index ["name", "supervisor_id"], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true
+ t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id"
+ end
+
+ create_table "solid_queue_ready_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.string "queue_name", null: false
+ t.integer "priority", default: 0, null: false
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true
+ t.index ["priority", "job_id"], name: "index_solid_queue_poll_all"
+ t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue"
+ end
+
+ create_table "solid_queue_recurring_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.string "task_key", null: false
+ t.datetime "run_at", null: false
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true
+ t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true
+ end
+
+ create_table "solid_queue_recurring_tasks", force: :cascade do |t|
+ t.string "key", null: false
+ t.string "schedule", null: false
+ t.string "command", limit: 2048
+ t.string "class_name"
+ t.text "arguments"
+ t.string "queue_name"
+ t.integer "priority", default: 0
+ t.boolean "static", default: true, null: false
+ t.text "description"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["key"], name: "index_solid_queue_recurring_tasks_on_key", unique: true
+ t.index ["static"], name: "index_solid_queue_recurring_tasks_on_static"
+ end
+
+ create_table "solid_queue_scheduled_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.string "queue_name", null: false
+ t.integer "priority", default: 0, null: false
+ t.datetime "scheduled_at", null: false
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true
+ t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all"
+ end
+
+ # solid_queue_jobs is deliberately placed after every solid_queue_* table that holds a
+ # foreign key referencing it (rather than in alphabetical order), so that db:schema:load
+ # can safely drop-and-recreate on a MySQL database that already has the schema loaded.
+ # MySQL's DROP TABLE doesn't actually support CASCADE (the keyword is accepted but has no
+ # effect - see https://dev.mysql.com/doc/refman/8.4/en/drop-table.html), so a table with an
+ # active foreign key pointing at it can't be dropped until the referencing table is dropped
+ # first.
+ create_table "solid_queue_jobs", force: :cascade do |t|
+ t.string "queue_name", null: false
+ t.string "class_name", null: false
+ t.text "arguments"
+ t.integer "priority", default: 0, null: false
+ t.string "active_job_id"
+ t.datetime "scheduled_at"
+ t.datetime "finished_at"
+ t.string "concurrency_key"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id"
+ t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name"
+ t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at"
+ t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering"
+ t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting"
+ end
+
+ create_table "solid_queue_semaphores", force: :cascade do |t|
+ t.string "key", null: false
+ t.integer "value", default: 1, null: false
+ t.datetime "expires_at", null: false
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at"
+ t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value"
+ t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true
+ end
+
create_table "sop_auth_lookup", force: :cascade do |t|
t.integer "user_id"
t.integer "asset_id"
@@ -2389,4 +2517,10 @@
add_foreign_key "oauth_access_grants", "oauth_applications", column: "application_id"
add_foreign_key "oauth_access_tokens", "oauth_applications", column: "application_id"
+ add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
end
diff --git a/docker-compose.yml b/docker-compose.yml
index 405960a6ee..ac0f9bc086 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -94,7 +94,7 @@ services:
start_period: 20s
seek_workers:
- # The SEEK delayed job workers
+ # The SEEK Solid Queue workers
<<: *seek_base
container_name: seek-workers
environment:
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 10892304ba..5bfb4596c4 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -42,9 +42,9 @@ bundle exec puma -C docker/puma.rb &
# Workers and Cron
if [ -z $NO_ENTRYPOINT_WORKERS ] #Don't start if flag set, for use with docker-compose
then
- echo "STARTING WORKERS"
- bundle exec rake seek:workers:start &
-
+ echo "STARTING SOLID QUEUE"
+ bundle exec rake seek:workers:start
+
setup_and_start_cron
fi
diff --git a/docker/seek.crontab b/docker/seek.crontab
new file mode 100644
index 0000000000..3513bf6f57
--- /dev/null
+++ b/docker/seek.crontab
@@ -0,0 +1,9 @@
+# Crontab for supercronic, started by setup_and_start_cron in docker/shared_functions.sh.
+#
+# All periodic *application* work lives in config/recurring.yml and is run by Solid Queue's own
+# scheduler - not here. This file is only for OS-level shell maintenance that doesn't map onto an
+# ActiveJob or Ruby call.
+#
+# Reap LibreOffice (soffice.bin) processes left running longer than 30 minutes by document
+# conversion. This runs inside the SEEK container(s) only, since cron is only started under Docker.
+*/10 * * * * sh /seek/script/kill-long-running-soffice.sh
diff --git a/docker/shared_functions.sh b/docker/shared_functions.sh
index d9530a1a8a..600ccaa6b3 100644
--- a/docker/shared_functions.sh
+++ b/docker/shared_functions.sh
@@ -62,15 +62,12 @@ function start_search {
}
function setup_and_start_cron {
- echo "GENERATING CRONTAB"
- bundle exec whenever > /seek/seek.crontab
-
if [ -z $QUIET_SUPERCRONIC ]
then
echo "STARTING SUPERCRONIC"
- supercronic /seek/seek.crontab &
+ supercronic /seek/docker/seek.crontab &
else
echo "STARTING SUPERCRONIC (QUIET)"
- supercronic -quiet /seek/seek.crontab &
+ supercronic -quiet /seek/docker/seek.crontab &
fi
}
diff --git a/docker/start_workers.sh b/docker/start_workers.sh
index 42e7f1e005..20e47af7a2 100644
--- a/docker/start_workers.sh
+++ b/docker/start_workers.sh
@@ -13,7 +13,7 @@ start_search
# Cron
setup_and_start_cron
-echo "STARTING WORKERS"
+echo "STARTING SOLID QUEUE"
bundle exec rake seek:workers:start
# Ensure the workers have started up and the logs are available before tailing
diff --git a/lib/seek/config_setting_attributes.yml b/lib/seek/config_setting_attributes.yml
index 701b7b60e6..db7cff9187 100644
--- a/lib/seek/config_setting_attributes.yml
+++ b/lib/seek/config_setting_attributes.yml
@@ -271,8 +271,6 @@ bio_tools_enabled:
require_cookie_consent:
galaxy_tool_sources:
metadata_license:
-regular_job_offset:
- convert: :to_i
auto_activate_programmes:
auto_activate_site_managed_projects:
fair_data_station_enabled:
diff --git a/lib/seek/delayed_job_migrator.rb b/lib/seek/delayed_job_migrator.rb
new file mode 100644
index 0000000000..23e5b54e72
--- /dev/null
+++ b/lib/seek/delayed_job_migrator.rb
@@ -0,0 +1,126 @@
+module Seek
+ # One-off migration of any rows left in the `delayed_jobs` table (queued before the Solid Queue
+ # cutover) into Solid Queue's own tables. Invoked exactly once via `seek:upgrade` - see the
+ # `migrate_delayed_jobs_to_solid_queue` task in lib/tasks/seek_upgrades.rake.
+ #
+ # Pending and locked rows are re-enqueued onto Solid Queue, preserving queue, run_at, priority and
+ # attempts. A stale lock (`locked_at`/`locked_by`) is ignored: by this point the delayed_job workers
+ # are gone, so the lock is meaningless and the job simply needs running again. Rows that have
+ # already failed (`failed_at` set) are deleted rather than migrated.
+ #
+ # Queued reindexing jobs (`ReindexAllJob`/`ReindexingJob`) are dropped rather than migrated, and the
+ # `ReindexingQueue` table (which `ReindexingJob` pulls its batches from) is cleared: the upgrade runs
+ # a full `seek:reindex_all` immediately afterwards, which re-enqueues a complete reindex of every
+ # searchable type, so any reindex work queued before the cutover is redundant.
+ #
+ # The Solid Queue row is built directly from the raw ActiveJob payload (`job_data`) rather than by
+ # deserialising and re-serialising the job. Re-serialising would force the job's GlobalID arguments
+ # to be resolved, raising ActiveJob::DeserializationError - and aborting the whole migration - for
+ # any job whose referenced record has since been deleted. Building the row directly keeps the
+ # original serialised arguments intact; deserialisation then happens lazily when the worker runs the
+ # job, where ApplicationJob's rescue_from already swallows a missing-record DeserializationError.
+ class DelayedJobMigrator
+ # Job classes whose queued work is made redundant by the full `seek:reindex_all` that the upgrade
+ # runs immediately afterwards - dropped rather than migrated.
+ REINDEX_JOB_CLASSES = %w[ReindexAllJob ReindexingJob].freeze
+
+ Result = Struct.new(:migrated, :failed_deleted, :reindex_dropped, :reindex_queue_cleared, :skipped,
+ keyword_init: true) do
+ def summary
+ "#{migrated} migrated, #{failed_deleted} failed row(s) deleted, " \
+ "#{reindex_dropped} reindex job(s) dropped, #{reindex_queue_cleared} reindex queue entry(s) cleared, " \
+ "#{skipped} skipped"
+ end
+ end
+
+ def self.run(logger: nil)
+ new(logger: logger).run
+ end
+
+ def initialize(logger: nil)
+ @logger = logger
+ end
+
+ def run
+ result = Result.new(migrated: 0, failed_deleted: 0, reindex_dropped: 0, reindex_queue_cleared: 0,
+ skipped: 0)
+ result.reindex_queue_cleared = clear_reindexing_queue
+
+ return result unless delayed_jobs_available?
+
+ Delayed::Job.find_each do |dj|
+ if dj.failed_at.present?
+ dj.destroy!
+ result.failed_deleted += 1
+ next
+ end
+
+ job_data = job_data_for(dj)
+
+ if job_data.nil?
+ result.skipped += 1
+ elsif REINDEX_JOB_CLASSES.include?(job_data['job_class'])
+ dj.destroy!
+ result.reindex_dropped += 1
+ elsif migrate(dj, job_data)
+ result.migrated += 1
+ else
+ result.skipped += 1
+ end
+ end
+
+ result
+ end
+
+ private
+
+ def migrate(dj, job_data)
+ ActiveRecord::Base.transaction do
+ SolidQueue::Job.create!(
+ queue_name: dj.queue.presence || job_data['queue_name'].presence || 'default',
+ active_job_id: job_data['job_id'],
+ priority: dj.priority || job_data['priority'] || 0,
+ scheduled_at: dj.run_at,
+ class_name: job_data['job_class'],
+ arguments: job_data
+ )
+ dj.destroy!
+ end
+ true
+ rescue StandardError => e
+ log("Failed to migrate delayed_job #{dj.id} (#{job_data&.dig('job_class')}): #{e.class}: #{e.message}")
+ false
+ end
+
+ # Extracts the ActiveJob payload hash from the delayed_job handler, folding the delayed_job
+ # `attempts` count into ActiveJob's `executions` field. Returns nil for any row that isn't an
+ # ActiveJob wrapper (SEEK only ever enqueues via ActiveJob, so this is a defensive guard).
+ def job_data_for(dj)
+ payload = dj.payload_object
+ return nil unless payload.respond_to?(:job_data)
+
+ job_data = payload.job_data.dup
+ job_data['executions'] = [job_data['executions'].to_i, dj.attempts.to_i].max
+ job_data
+ rescue StandardError => e
+ log("Could not read delayed_job #{dj.id} handler: #{e.class}: #{e.message}")
+ nil
+ end
+
+ # Empties the ReindexingQueue - its pending batches are superseded by the full reindex_all the
+ # upgrade runs next. Returns the number of entries removed.
+ def clear_reindexing_queue
+ return 0 unless defined?(ReindexingQueue) && ReindexingQueue.table_exists?
+
+ ReindexingQueue.delete_all
+ end
+
+ def delayed_jobs_available?
+ defined?(Delayed::Job) && ActiveRecord::Base.connection.table_exists?('delayed_jobs')
+ end
+
+ def log(message)
+ @logger&.call(message)
+ end
+ end
+end
diff --git a/lib/seek/util.rb b/lib/seek/util.rb
index 5a63ac49c8..d45f342423 100644
--- a/lib/seek/util.rb
+++ b/lib/seek/util.rb
@@ -140,12 +140,26 @@ def self.database_type
ActiveRecord::Base.connection.instance_values['config'][:adapter]
end
- def self.delayed_job_pids
- directory = "#{Rails.root}/tmp/pids"
- Daemons::PidFile.find_files(directory, 'delayed_job', false, '').collect do |path|
- file = path.sub("#{directory}/", '').sub('.pid', '')
- Daemons::PidFile.new(directory, file)
- end
+ # The pid of the running Solid Queue supervisor, if its pidfile (config/initializers/solid_queue.rb)
+ # exists and the process is still alive, otherwise nil.
+ def self.solid_queue_supervisor_pid
+ live_pid_from_pidfile(SolidQueue.supervisor_pidfile)
+ end
+
+ # Read a pid from a pidfile, returning it only if the process is still alive (nil for a missing or
+ # stale pidfile). A process owned by another user counts as alive, since it exists.
+ def self.live_pid_from_pidfile(path)
+ return nil unless path && File.exist?(path)
+
+ pid = File.read(path).strip.to_i
+ return nil unless pid.positive?
+
+ Process.kill(0, pid)
+ pid
+ rescue Errno::ESRCH
+ nil # No process with this pid - the pidfile is stale.
+ rescue Errno::EPERM
+ pid # The process exists but is owned by another user, so we can't signal it. It is still running.
end
# Use this to avoid needlessly regenerating the url helper module each time a route needs to be accessed
diff --git a/lib/seek/workers.rb b/lib/seek/workers.rb
deleted file mode 100644
index 354f78a792..0000000000
--- a/lib/seek/workers.rb
+++ /dev/null
@@ -1,54 +0,0 @@
-require 'delayed/command'
-
-# module for handling interaction with delayed job workers
-module Seek
- module Workers
- def self.start
- commands = create_commands('start')
- daemonize_commands(commands)
- end
-
- def self.daemonize_commands(commands)
- commands.map { |command| Delayed::Command.new(command.split).daemonize }
- end
-
- def self.create_commands(action)
- commands = []
-
- active_queues.each_with_index do |queue_name, index|
- commands << command(queue_name, index + 1, 1, action)
- end
- commands
- end
-
- def self.active_queues
- queues = [QueueNames::DEFAULT]
- queues << QueueNames::MAILERS
- queues << QueueNames::AUTH_LOOKUP if Seek::Config.auth_lookup_enabled
- queues << QueueNames::REMOTE_CONTENT if Seek::Config.cache_remote_files
- queues << QueueNames::SAMPLES if Seek::Config.samples_enabled
- queues << QueueNames::INDEXING if Seek::Config.solr_enabled
- queues << QueueNames::TEMPLATES if Seek::Config.isa_json_compliance_enabled
- queues << QueueNames::DATAFILES if Seek::Config.data_files_enabled
- queues
- end
-
- def self.stop
- # will stop the first 15, not expecting more than that
- daemonize_commands(['stop -n 15'])
- end
-
- def self.status
- daemonize_commands(['status'])
- end
-
- def self.restart
- stop
- start
- end
-
- def self.command(queue_name, index, number_of_workers, action)
- "--queue=#{queue_name} -i #{index} -n #{number_of_workers} #{action}"
- end
- end
-end
diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake
index 8d149b3b95..54752fc037 100644
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -12,5 +12,4 @@ Rake::Task['db:schema:dump'].enhance do
rescue StandardError => e
puts "Failed to convert schema.rb to db agnostic - #{e.message}"
end
-end
-
+end
\ No newline at end of file
diff --git a/lib/tasks/jobs.rake b/lib/tasks/jobs.rake
new file mode 100644
index 0000000000..12887e536a
--- /dev/null
+++ b/lib/tasks/jobs.rake
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# Defines the `jobs:*` rake tasks against Solid Queue.
+#
+# The delayed_job gem is a dependency and its railtie also contributes `jobs:work`, `jobs:workoff`,
+# `jobs:clear` and `jobs:check`. Gem rake tasks load before the application's own lib/tasks/*.rake,
+# so those definitions already exist by the time this file runs; each is cleared before being
+# redefined below. The guard keeps this working whether or not delayed_job is present.
+%w[work workoff clear check].each do |name|
+ Rake::Task["jobs:#{name}"].clear if Rake::Task.task_defined?("jobs:#{name}")
+end
+
+namespace :jobs do
+ desc 'Start the Solid Queue supervisor (equivalent to bin/jobs)'
+ task work: :environment do
+ SolidQueue::Supervisor.start
+ end
+
+ desc 'Run all available Solid Queue jobs and exit when the queue is empty. QUEUES=a,b THREADS=n'
+ task workoff: :environment do
+ queues = ENV['QUEUES'].presence || ENV['QUEUE'].presence || '*'
+ threads = (ENV['THREADS'].presence || 3).to_i
+
+ # Move any scheduled jobs that are already due onto the ready queue - normally the dispatcher's
+ # job, but there isn't one running here. Only due jobs are picked up, so anything scheduled for
+ # the future is deliberately left alone. Jobs that were already ready need no dispatching, so this
+ # is routinely zero even when there is plenty of work waiting.
+ dispatched = 0
+ loop do
+ batch = SolidQueue::ScheduledExecution.dispatch_next_batch(500)
+ dispatched += batch
+ break if batch.zero?
+ end
+
+ ready = SolidQueue::ReadyExecution.all
+ ready = ready.where(queue_name: queues.split(',')) unless queues == '*'
+ waiting = ready.count
+ puts "Dispatched #{dispatched} due scheduled job(s); #{waiting} job(s) ready to run on queue(s) #{queues}"
+
+ # Count what actually runs. Solid Queue doesn't report this itself, and the job rows can't simply be
+ # counted afterwards: jobs may enqueue further jobs, and a job that fails is left unfinished. Note
+ # that the exception count stays at zero for SEEK's own jobs however badly they go wrong, because
+ # ApplicationJob's `rescue_from(Exception)` handles the exception inside `perform_now` - it is only
+ # reached by jobs that don't inherit from ApplicationJob, such as SolidQueue::RecurringJob.
+ performed = Concurrent::AtomicFixnum.new
+ failed = Concurrent::AtomicFixnum.new
+ subscriber = ActiveSupport::Notifications.subscribe('perform.active_job') do |*, payload|
+ performed.increment
+ failed.increment if payload[:exception] || payload[:exception_object]
+ end
+
+ started_at = Time.now
+ begin
+ # A worker in `inline` mode runs in the current process and shuts itself down as soon as the ready
+ # queue is empty, waiting for its thread pool to drain first. Jobs enqueued by the jobs being run
+ # are only picked up if they land before the queue empties.
+ worker = SolidQueue::Worker.new(queues: queues, threads: threads, polling_interval: 0.1)
+ worker.mode = :inline
+ worker.start
+ ensure
+ ActiveSupport::Notifications.unsubscribe(subscriber)
+ end
+
+ summary = "Ran #{performed.value} job(s) in #{(Time.now - started_at).round(1)}s"
+ summary += ", #{failed.value} raised an exception" if failed.value.positive?
+ puts summary
+ puts "#{SolidQueue::Job.where(finished_at: nil).count} unfinished job(s) remain (including any scheduled for later)"
+ end
+
+ desc 'Clear the Solid Queue queue by discarding every unfinished job'
+ task clear: :environment do
+ count = 0
+ SolidQueue::Job.where(finished_at: nil).find_each do |job|
+ job.discard
+ count += 1
+ end
+ puts "Discarded #{count} unfinished job(s)"
+ end
+
+ desc "Exit with error status if any jobs older than max_age seconds haven't been run yet"
+ task :check, [:max_age] => :environment do |_task, args|
+ args.with_defaults(max_age: 300)
+
+ # Measured from when the job became due rather than when it was created, so that jobs deliberately
+ # scheduled for later aren't reported as overdue.
+ unfinished = SolidQueue::Job.where(finished_at: nil)
+ due_by = ->(time) { unfinished.where('COALESCE(scheduled_at, created_at) <= ?', time) }
+ stale = due_by.call(Time.now - args[:max_age].to_i).count
+
+ raise "#{stale} jobs older than #{args[:max_age]} seconds have not been processed yet" if stale.positive?
+
+ puts "OK - no job has been waiting longer than #{args[:max_age]} seconds " \
+ "(#{unfinished.count} unfinished job(s), of which #{due_by.call(Time.now).count} due)"
+ end
+end
diff --git a/lib/tasks/seek_upgrades.rake b/lib/tasks/seek_upgrades.rake
index b302ec108f..387fe8e6de 100644
--- a/lib/tasks/seek_upgrades.rake
+++ b/lib/tasks/seek_upgrades.rake
@@ -18,6 +18,7 @@ namespace :seek do
db:seed:017_minimal_starter_isa_templates
db:seed:019_sop_type_controlled_vocab
db:seed:020_event_types
+ migrate_delayed_jobs_to_solid_queue
]
# these are the tasks that are executes for each upgrade as standard, and rarely change
@@ -68,6 +69,15 @@ namespace :seek do
end
end
+ desc('migrates any jobs left in the delayed_jobs table into Solid Queue (one-off, post-cutover)')
+ task(migrate_delayed_jobs_to_solid_queue: [:environment]) do
+ only_once('seek:migrate_delayed_jobs_to_solid_queue 1.19.0') do
+ puts '... migrating any remaining delayed_jobs into Solid Queue'
+ result = Seek::DelayedJobMigrator.run(logger: ->(message) { puts " #{message}" })
+ puts "... #{result.summary}"
+ end
+ end
+
task(strip_publication_abstracts: [:environment]) do
puts 'Stripping publication abstracts...'
updated_count = 0
diff --git a/lib/tasks/seek_workers.rake b/lib/tasks/seek_workers.rake
index c1178b4c57..98ebf123d8 100644
--- a/lib/tasks/seek_workers.rake
+++ b/lib/tasks/seek_workers.rake
@@ -1,27 +1,63 @@
# frozen_string_literal: true
-require 'rubygems'
-require 'rake'
-
+# The seek:workers:* tasks manage the Solid Queue supervisor (bin/jobs), which forks and monitors the
+# per-queue worker/dispatcher/scheduler subprocesses (config/queue.yml). They are the interface used
+# by the Docker entrypoints, the deployment scripts and any external init scripts, abstracting the
+# job backend away from those callers.
namespace :seek do
namespace :workers do
- desc 'Start the delayed job workers'
+ desc 'Start the Solid Queue supervisor in the background'
task start: :environment do
- Seek::Workers.start
+ if (pid = Seek::Util.solid_queue_supervisor_pid)
+ puts "Solid Queue supervisor is already running (pid #{pid})"
+ else
+ # Daemonise: a new process group detached from this task, writing to the Rails log, so bin/jobs
+ # keeps running after the task returns. The supervisor records its own pid in
+ # SolidQueue.supervisor_pidfile (config/initializers/solid_queue.rb), which stop/status read.
+ log = Rails.application.config.paths['log'].first
+ pid = Process.spawn('bundle', 'exec', 'bin/jobs',
+ chdir: Rails.root.to_s, pgroup: true,
+ in: File::NULL, out: [log, 'a'], err: [log, 'a'])
+ Process.detach(pid)
+ puts "Started Solid Queue supervisor in the background (pid #{pid})"
+ end
end
- desc 'Stop the delayed job workers'
+ desc 'Stop the Solid Queue supervisor'
task stop: :environment do
- Seek::Workers.stop
- end
+ pid = Seek::Util.solid_queue_supervisor_pid
+ if pid.nil?
+ puts 'No Solid Queue supervisor is running'
+ else
+ Process.kill('TERM', pid)
+ puts "Stopping Solid Queue supervisor (pid #{pid})..."
+ # Wait (up to ~15s) for the supervisor to shut its workers down and remove its pidfile, so that
+ # a following start (e.g. from restart) sees a clean state rather than the still-exiting process.
+ 30.times do
+ break if Seek::Util.solid_queue_supervisor_pid.nil?
- desc 'Get the status of the delayed job workers'
- task status: :environment do
- Seek::Workers.status
+ sleep 0.5
+ end
+ if Seek::Util.solid_queue_supervisor_pid
+ puts 'Solid Queue supervisor did not stop within 15s'
+ else
+ puts 'Solid Queue supervisor stopped'
+ end
+ end
+ rescue Errno::ESRCH
+ puts 'No Solid Queue supervisor is running' # exited between the check and the signal
end
+ desc 'Restart the Solid Queue supervisor'
task restart: :environment do
- Seek::Workers.restart
+ Rake::Task['seek:workers:stop'].invoke
+ Rake::Task['seek:workers:start'].invoke
+ end
+
+ desc 'Report whether the Solid Queue supervisor is running'
+ task status: :environment do
+ pid = Seek::Util.solid_queue_supervisor_pid
+ puts(pid ? "Solid Queue supervisor running (pid #{pid})" : 'Solid Queue supervisor is not running')
end
end
end
diff --git a/script/check_deployment.rb b/script/check_deployment.rb
index 3bee7a7abe..eeefa88d65 100644
--- a/script/check_deployment.rb
+++ b/script/check_deployment.rb
@@ -1,5 +1,5 @@
output = `curl --verbose --silent http://localhost:3000/statistics/application_status 2>&1`
-if $?.success? && output.include?('FAIRDOM-SEEK is running | search is enabled | 7 delayed jobs running')
+if $?.success? && output.include?('FAIRDOM-SEEK is running | search is enabled | 8 background job worker processes running')
exit 0
else
puts "::group::Docker logs"
diff --git a/script/check_worker_pids.sh b/script/check_worker_pids.sh
index f88dee52ae..aa6a37b312 100644
--- a/script/check_worker_pids.sh
+++ b/script/check_worker_pids.sh
@@ -1,25 +1,14 @@
#!/bin/bash
-expected_workers=$(bundle exec rails runner "puts Seek::Workers.active_queues.count")
+# Solid Queue's supervisor forks its own worker/dispatcher/scheduler subprocesses
+# internally (per config/queue.yml), so checking the single supervisor pidfile is
+# sufficient to tell whether background job processing is up.
+pid=$(bundle exec rails runner "puts Seek::Util.solid_queue_supervisor_pid")
-shopt -s nullglob || true 2>/dev/null
-
-# Fails if the number of running worker PID files is less than expected_workers
-pid_files=(tmp/pids/delayed_job.*.pid)
-pids=$(cat "${pid_files[@]}")
-running_workers=$(echo "$pids" 2>/dev/null | wc -l)
-if [ "$running_workers" -ne "$expected_workers" ]; then
- echo "Expected at least $expected_workers workers, but found $running_workers."
+if [ -z "$pid" ]; then
+ echo "Solid Queue supervisor is not running."
exit 1
-else
- for pid in $pids; do
- if ! kill -0 "$pid" 2>/dev/null; then
- echo "Worker with PID $pid is not running."
- exit 1
- fi
- done
fi
-# Log success
-echo "Found $running_workers running workers (Expected: $expected_workers)."
-exit 0
\ No newline at end of file
+echo "Solid Queue supervisor running (Process ID: $pid)."
+exit 0
diff --git a/script/mini-update-from-git.sh b/script/mini-update-from-git.sh
index 114964b5ee..9d1fa441f0 100755
--- a/script/mini-update-from-git.sh
+++ b/script/mini-update-from-git.sh
@@ -24,9 +24,6 @@ bundle exec rake assets:precompile # this task will take a while
echo "${GREEN} restart workers${NC}"
bundle exec rake seek:workers:restart
-echo "${GREEN} update crontab${NC}"
-bundle exec whenever --update-crontab
-
echo "${GREEN} restart server${NC}"
touch tmp/restart.txt
bundle exec rake tmp:clear
diff --git a/script/update-from-git.sh b/script/update-from-git.sh
index 34af284cf4..81d9f89c60 100755
--- a/script/update-from-git.sh
+++ b/script/update-from-git.sh
@@ -19,20 +19,18 @@ bundle install --deployment --without development test
echo "${GREEN}pip install${NC}"
python`cat .python-version` -m pip install -r requirements.txt
+echo "${GREEN} stop background job workers${NC}"
bundle exec rake seek:workers:stop
echo "${GREEN} seek:upgrade${NC}"
bundle exec rake seek:upgrade
sleep 5 # small delay to make sure SOLR has started up and ready
-bundle exec rake seek:workers:start &
+bundle exec rake seek:workers:start
echo "${GREEN} precompile assets${NC}"
bundle exec rake assets:precompile # this task will take a while
-echo "${GREEN} update crontab${NC}"
-bundle exec whenever --update-crontab &
-
echo "${GREEN} restart server${NC}"
touch tmp/restart.txt
bundle exec rake tmp:clear
diff --git a/test/functional/admin_controller_test.rb b/test/functional/admin_controller_test.rb
index d34a9a7a1d..b3f0c7bf47 100644
--- a/test/functional/admin_controller_test.rb
+++ b/test/functional/admin_controller_test.rb
@@ -30,14 +30,14 @@ def setup
assert_response :success
end
- test 'non admin cannot restart the delayed job' do
+ test 'non admin cannot restart the job workers' do
login_as(FactoryBot.create(:user))
- post :restart_delayed_job
+ post :restart_job_workers
refute_nil flash[:error]
end
- test 'admin can restart the delayed job' do
- post :restart_delayed_job
+ test 'admin can restart the job workers' do
+ post :restart_job_workers
assert_nil flash[:error]
end
@@ -299,20 +299,19 @@ def setup
end
test 'job statistics stats' do
- Delayed::Job.destroy_all
- dj = Delayed::Job.create(run_at: '2010 September 12', locked_at: '2010 September 13', failed_at: nil)
- dj.created_at = '2010 September 11'
- assert dj.save
+ SolidQueue::Job.destroy_all
+ job = SolidQueue::Job.create!(queue_name: 'default', class_name: 'TestJob', scheduled_at: '2010 September 12')
+ job.update_column(:created_at, '2010 September 11')
get :get_stats, xhr: true, params: { page: 'job_queue' }
assert_response :success
- assert_select 'h4', text: 'Total delayed jobs waiting = 1'
+ assert_select 'h4', text: 'Total jobs waiting = 1'
assert_select 'tr' do
assert_select 'td', text: /11th Sep 2010 at/, count: 1
assert_select 'td', text: /12th Sep 2010 at/, count: 1
- assert_select 'td', text: /13th Sep 2010 at/, count: 1
- assert_select "td > span[class='none_text']", text: /No date defined/, count: 1
+ assert_select 'td', text: 'default', count: 1
+ assert_select 'td', text: 'TestJob', count: 1
end
end
@@ -467,41 +466,41 @@ def setup
end
test 'clear failed jobs' do
- Delayed::Job.destroy_all
- job = Delayed::Job.create!
- job.update_column(:failed_at,Time.now)
- Delayed::Job.create!
- assert_equal 2,Delayed::Job.count
- assert_difference('Delayed::Job.count',-1) do
+ SolidQueue::Job.destroy_all
+ job = SolidQueue::Job.create!(queue_name: 'default', class_name: 'TestJob')
+ SolidQueue::FailedExecution.create!(job: job, error: 'boom')
+ SolidQueue::Job.create!(queue_name: 'default', class_name: 'TestJob')
+ assert_equal 2, SolidQueue::Job.count
+ assert_difference('SolidQueue::Job.count', -1) do
post :clear_failed_jobs, format: 'json'
end
- assert_equal 1,Delayed::Job.count
- assert_equal 0,Delayed::Job.where('failed_at IS NOT NULL').count
+ assert_equal 1, SolidQueue::Job.count
+ assert_equal 0, SolidQueue::Job.failed.count
end
test 'admin required to clear failed jobs' do
logout
person = FactoryBot.create(:person)
- Delayed::Job.destroy_all
- job = Delayed::Job.create!
- job.update_column(:failed_at,Time.now)
- Delayed::Job.create!
- assert_equal 2,Delayed::Job.count
+ SolidQueue::Job.destroy_all
+ job = SolidQueue::Job.create!(queue_name: 'default', class_name: 'TestJob')
+ SolidQueue::FailedExecution.create!(job: job, error: 'boom')
+ SolidQueue::Job.create!(queue_name: 'default', class_name: 'TestJob')
+ assert_equal 2, SolidQueue::Job.count
- assert_no_difference('Delayed::Job.count') do
+ assert_no_difference('SolidQueue::Job.count') do
post :clear_failed_jobs, format: 'json'
end
login_as(person)
- assert_no_difference('Delayed::Job.count') do
+ assert_no_difference('SolidQueue::Job.count') do
post :clear_failed_jobs, format: 'json'
end
- assert_equal 2,Delayed::Job.count
- assert_equal 1,Delayed::Job.where('failed_at IS NOT NULL').count
+ assert_equal 2, SolidQueue::Job.count
+ assert_equal 1, SolidQueue::Job.failed.count
end
test 'update branding' do
diff --git a/test/functional/statistics_controller_test.rb b/test/functional/statistics_controller_test.rb
index 6e03689658..c65d06a823 100644
--- a/test/functional/statistics_controller_test.rb
+++ b/test/functional/statistics_controller_test.rb
@@ -34,7 +34,7 @@ class StatisticsControllerTest < ActionController::TestCase
get :application_status
end
assert_response :success
- assert_match(/Euro SEEK is running \| search is enabled \| [0-9] delayed jobs running/, @response.body)
+ assert_match(/Euro SEEK is running \| search is enabled \| [0-9]+ background job worker processes running/, @response.body)
end
end
end
diff --git a/test/integration/recurring_test.rb b/test/integration/recurring_test.rb
new file mode 100644
index 0000000000..0f5e081177
--- /dev/null
+++ b/test/integration/recurring_test.rb
@@ -0,0 +1,158 @@
+require 'test_helper'
+require 'minitest/mock'
+
+class RecurringTest < ActiveSupport::TestCase
+ setup do
+ @config = ActiveSupport::ConfigurationFile.parse(Rails.root.join('config/recurring.yml')).deep_symbolize_keys
+ @tasks = @config[:production].deep_dup
+ end
+
+ test 'should read recurring schedule file' do
+ daily = pop_task(:periodic_subscription_email_daily)
+ assert_equal 'PeriodicSubscriptionEmailJob', daily[:class]
+ assert_equal ['daily'], daily[:args]
+ assert_equal '0 0 * * *', daily[:schedule]
+
+ weekly = pop_task(:periodic_subscription_email_weekly)
+ assert_equal 'PeriodicSubscriptionEmailJob', weekly[:class]
+ assert_equal ['weekly'], weekly[:args]
+ assert_equal '0 0 * * 0', weekly[:schedule]
+
+ monthly = pop_task(:periodic_subscription_email_monthly)
+ assert_equal 'PeriodicSubscriptionEmailJob', monthly[:class]
+ assert_equal ['monthly'], monthly[:args]
+ assert_equal '0 0 1 * *', monthly[:schedule]
+
+ regular = pop_task(:regular_maintenance)
+ assert_equal 'RegularMaintenanceJob', regular[:class]
+ assert_equal '0 */4 * * *', regular[:schedule]
+
+ auth = pop_task(:auth_lookup_maintenance)
+ assert_equal 'AuthLookupMaintenanceJob', auth[:class]
+ assert_equal '0 */8 * * *', auth[:schedule]
+
+ cache_cleanup = pop_task(:cache_overflow_cleanup)
+ assert_equal 'CacheOverflowCleanupJob', cache_cleanup[:class]
+ assert_equal '0 4 * * *', cache_cleanup[:schedule]
+
+ life_monitor = pop_task(:life_monitor_status)
+ assert_equal 'LifeMonitorStatusJob', life_monitor[:class]
+ assert_equal '0 2 * * *', life_monitor[:schedule]
+
+ news_refresh = pop_task(:news_feed_refresh)
+ assert_equal 'NewsFeedRefreshJob', news_refresh[:class]
+ assert_equal 3, news_refresh[:priority]
+ assert_equal NewsFeedRefreshJob.cron_schedule, news_refresh[:schedule]
+
+ queue_timed = pop_task(:queue_timed_jobs)
+ assert_equal 'ApplicationJob.queue_timed_jobs', queue_timed[:command]
+ assert_equal '*/10 * * * *', queue_timed[:schedule]
+
+ clear_finished = pop_task(:clear_finished_jobs)
+ assert_equal 'SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)', clear_finished[:command]
+ assert_equal '12 * * * *', Fugit.parse(clear_finished[:schedule].to_s).to_cron_s
+
+ app_status = pop_task(:application_status_refresh)
+ assert_equal 'ApplicationStatus.instance.refresh', app_status[:command]
+ assert_equal '* * * * *', app_status[:schedule]
+
+ tool_map_refresh = pop_task(:galaxy_tool_map_refresh)
+ assert_equal 'Galaxy::ToolMap.instance.refresh', tool_map_refresh[:command]
+ assert_equal '0 3 * * *', tool_map_refresh[:schedule]
+
+ data_dump = pop_task(:bioschema_data_dump_generate)
+ assert_equal 'Seek::BioSchema::DataDump.generate_dumps', data_dump[:command]
+ assert_equal '10 0 * * *', data_dump[:schedule]
+
+ sitemap = pop_task(:sitemap_refresh)
+ assert_equal 'SitemapRefreshJob', sitemap[:class]
+ assert_equal '45 0 * * *', sitemap[:schedule]
+
+ assert_empty @tasks, "Found untested recurring task(s): #{@tasks.keys.join(', ')}"
+ end
+
+ test 'news feed refresh schedule stays a valid cron as the cache timeout changes' do
+ # A bare "*/n" cron is only valid for n in 1..59; larger intervals must become an hour/day step,
+ # or Solid Queue's Fugit-based validation rejects the schedule and stops the scheduler.
+ {
+ 30 => '*/30 * * * *', # minute interval
+ 90 => '0 */2 * * *', # rounded up to a two-hour step
+ 731 => '0 */12 * * *' # large interval a bare */n cron could not express
+ }.each do |timeout, expected|
+ with_config_value(:home_feeds_cache_timeout, timeout) do
+ schedule = production_tasks[:news_feed_refresh][:schedule]
+ assert_equal expected, schedule
+ assert Fugit.parse_cron(schedule), "#{schedule.inspect} is not a valid cron"
+ end
+ end
+ end
+
+ test 'schedules are all valid cron expressions' do
+ @tasks.each do |key, options|
+ assert Fugit.parse(options[:schedule].to_s),
+ "#{key}: #{options[:schedule].inspect} did not parse as a valid schedule"
+ end
+ end
+
+ test 'each task resolves to a valid recurring task, with a queue that has a configured worker' do
+ # Recurring tasks aren't feature-flag gated, but some of the queues they target are
+ # (config/queue.yml only starts a worker for e.g. `authlookup` when auth_lookup_enabled is
+ # on) - so this needs every optional feature enabled to see the full set of queues a fully
+ # configured instance would have workers for.
+ all_features_enabled = {
+ auth_lookup_enabled: true, cache_remote_files: true, samples_enabled: true,
+ solr_enabled: true, isa_json_compliance_enabled: true, data_files_enabled: true
+ }
+ with_config_values(all_features_enabled) do
+ queue_config = ActiveSupport::ConfigurationFile.parse(Rails.root.join('config/queue.yml')).deep_symbolize_keys
+ configured_queues = queue_config[:production][:workers].map { |w| w[:queues] }
+
+ @tasks.each do |key, options|
+ task = SolidQueue::RecurringTask.from_configuration(key.to_s, **options)
+ assert task.valid?, "#{key}: #{task.errors.full_messages.join(', ')}"
+
+ # class: entries use that job class's own queue_as; command:-only entries fall back to
+ # SolidQueue::RecurringJob's queue_as (:solid_queue_recurring) - both need a worker
+ # configured in config/queue.yml, or the task enqueues but is never picked up.
+ job_class = options[:class]&.safe_constantize || SolidQueue::RecurringJob
+ assert_includes configured_queues, job_class.queue_name,
+ "#{key} enqueues onto '#{job_class.queue_name}', which has no worker in config/queue.yml"
+ end
+ end
+ end
+
+ test 'executes recurring tasks without error' do
+ with_config_value(:email_enabled, true) do
+ with_config_value(:openbis_enabled, true) do
+ VCR.use_cassette('galaxy/fetch_tools_trimmed') do
+ VCR.use_cassette('bio_tools/fetch_galaxy_tool_names') do
+ # SitemapRefreshJob generates files and pings search engines over HTTP; stub both so this
+ # test doesn't write sitemaps or make real network requests (SitemapRefreshJobTest covers
+ # that it calls them).
+ SitemapGenerator::Interpreter.stub(:run, nil) do
+ SitemapGenerator::Sitemap.stub(:ping_search_engines, nil) do
+ perform_enqueued_jobs do
+ assert_nothing_raised do
+ @tasks.each do |key, options|
+ SolidQueue::RecurringTask.from_configuration(key.to_s, **options).enqueue(at: Time.current)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ private
+
+ def production_tasks
+ ActiveSupport::ConfigurationFile.parse(Rails.root.join('config/recurring.yml')).deep_symbolize_keys[:production]
+ end
+
+ def pop_task(key)
+ @tasks.delete(key)
+ end
+end
diff --git a/test/integration/schedule_test.rb b/test/integration/schedule_test.rb
deleted file mode 100644
index 1c11a5d869..0000000000
--- a/test/integration/schedule_test.rb
+++ /dev/null
@@ -1,147 +0,0 @@
-require 'test_helper'
-
-class ScheduleTest < ActionDispatch::IntegrationTest
- setup do
- @schedule = Whenever::Test::Schedule.new(file: 'config/schedule.rb')
- end
-
- test 'should read schedule file' do
- runners = @schedule.jobs[:runner]
-
- # Periodic emails
- daily = pop_task(runners, "PeriodicSubscriptionEmailJob.new('daily').queue_job")
- weekly = pop_task(runners, "PeriodicSubscriptionEmailJob.new('weekly').queue_job")
- monthly = pop_task(runners, "PeriodicSubscriptionEmailJob.new('monthly').queue_job")
- assert daily
- assert_equal [1.day, { at: '12:00am' }], daily[:every]
- assert weekly
- assert_equal [1.week, { at: '12:00am' }], weekly[:every]
- assert monthly
- assert_equal [1.month, { at: '12:00am' }], monthly[:every]
-
- # RegularMaintenanceJob
- regular = pop_task(runners, "RegularMaintenanceJob.perform_later")
- assert regular
- assert_equal [RegularMaintenanceJob::RUN_PERIOD, { at: '1:00am' }], regular[:every]
-
- # AuthLookupMaintenanceJob
- auth = pop_task(runners, "AuthLookupMaintenanceJob.perform_later")
- assert auth
- assert_equal [AuthLookupMaintenanceJob::RUN_PERIOD, { at: '1:00am' }], auth[:every]
-
- # LifeMonitor status
- lm_status = pop_task(runners, "LifeMonitorStatusJob.perform_later")
- assert lm_status
- assert_equal [LifeMonitorStatusJob::PERIOD, { at: '2:00am' }], lm_status[:every]
-
- # Newsfeed refresh
- news_refresh = pop_task(runners, "NewsFeedRefreshJob.set(priority: 3).perform_later")
- assert news_refresh
- assert_equal [Seek::Config.home_feeds_cache_timeout.minutes], news_refresh[:every]
-
- # General
- general = pop_task(runners, "ApplicationJob.queue_timed_jobs")
- assert general
- assert_equal [10.minutes], general[:every]
-
- # ApplicationStatus
- app_status = pop_task(runners, "ApplicationStatus.instance.refresh")
- assert app_status
- assert_equal [1.minute], app_status[:every]
-
- # Galaxy::ToolMap.instance.refresh
- tool_map_refresh = pop_task(runners, "Galaxy::ToolMap.instance.refresh")
- assert tool_map_refresh
- assert_equal [1.day, { at: '3:00am' }], tool_map_refresh[:every]
-
- # CacheOverflowCleanupJob
- cache_cleanup = pop_task(runners, "CacheOverflowCleanupJob.perform_later")
- assert cache_cleanup
- assert_equal [CacheOverflowCleanupJob::RUN_PERIOD, { at: '4:00am' }], cache_cleanup[:every]
-
- # Data dumps
- data_dump = pop_task(runners, 'Seek::BioSchema::DataDump.generate_dumps')
- assert data_dump
- assert_equal [1.day, { at: '12:10 am' }], data_dump[:every]
-
- assert_empty runners, "Found untested runner(s) in schedule"
- end
-
- test 'executes tasks in schedule' do
- # Executes all the tasks to see if any of them throw error
- with_config_value(:email_enabled, true) do
- with_config_value(:openbis_enabled, true) do
- assert_nothing_raised do
- VCR.use_cassette('galaxy/fetch_tools_trimmed') do
- VCR.use_cassette('bio_tools/fetch_galaxy_tool_names') do
- @schedule.jobs[:runner].each { |job| instance_eval job[:task] }
- end
- end
- end
- end
- end
- end
-
- test 'executes tasks in schedule and runs jobs' do
- # Executes all the tasks, and also runs the jobs to see if any of them throw errors
- with_config_value(:email_enabled, true) do
- with_config_value(:openbis_enabled, true) do
- perform_enqueued_jobs do
- assert_nothing_raised do
- VCR.use_cassette('galaxy/fetch_tools_trimmed') do
- VCR.use_cassette('bio_tools/fetch_galaxy_tool_names') do
- @schedule.jobs[:runner].each { |job| instance_eval job[:task] }
- end
- end
- end
- end
- end
- end
- end
-
- test 'news feed refresh changes with config' do
- with_config_value(:home_feeds_cache_timeout, 731) do
- news_refresh = Whenever::Test::Schedule.new(file: 'config/schedule.rb').jobs[:runner].detect { |job| job[:task] == "NewsFeedRefreshJob.set(priority: 3).perform_later" }
- assert_equal [Seek::Config.home_feeds_cache_timeout.minutes], news_refresh[:every]
- assert_equal [731.minutes], news_refresh[:every]
- end
- end
-
- test 'should offset daily job runtime by configured amount' do
- plus_43_schedule = nil
- with_config_value(:regular_job_offset, 43) do
- plus_43_schedule = Whenever::Test::Schedule.new(file: 'config/schedule.rb')
- end
- plus_43_runners = plus_43_schedule.jobs[:runner]
-
- minus_237_schedule = nil
- with_config_value(:regular_job_offset, -237) do
- minus_237_schedule = Whenever::Test::Schedule.new(file: 'config/schedule.rb')
- end
- minus_237_runners = minus_237_schedule.jobs[:runner]
-
- # For jobs that are not run daily, such as this one, which is run every 4 hours, only the minute offsets are applied.
- assert_equal [RegularMaintenanceJob::RUN_PERIOD, { at: '1:43am' }],
- pop_task(plus_43_runners, "RegularMaintenanceJob.perform_later")[:every]
- assert_equal [RegularMaintenanceJob::RUN_PERIOD, { at: '9:03pm' }],
- pop_task(minus_237_runners, "RegularMaintenanceJob.perform_later")[:every]
-
- assert_equal [LifeMonitorStatusJob::PERIOD, { at: '2:43am' }],
- pop_task(plus_43_runners, "LifeMonitorStatusJob.perform_later")[:every]
- assert_equal [LifeMonitorStatusJob::PERIOD, { at: '10:03pm' }],
- pop_task(minus_237_runners, "LifeMonitorStatusJob.perform_later")[:every]
-
- assert_equal [1.day, { at: '3:43am' }],
- pop_task(plus_43_runners, "Galaxy::ToolMap.instance.refresh")[:every]
- assert_equal [1.day, { at: '11:03pm' }],
- pop_task(minus_237_runners, "Galaxy::ToolMap.instance.refresh")[:every]
- end
-
- private
-
- def pop_task(runners, task)
- i = runners.index { |job| job[:task] == task }
- return runners.delete_at(i) if i
- nil
- end
-end
diff --git a/test/unit/application_status_test.rb b/test/unit/application_status_test.rb
index 7a1306275e..bd780f0029 100644
--- a/test/unit/application_status_test.rb
+++ b/test/unit/application_status_test.rb
@@ -32,10 +32,24 @@ def setup
app = ApplicationStatus.instance
app.refresh
app.reload
- assert_equal Seek::Util.delayed_job_pids.count, app.running_jobs
+ alive_since = SolidQueue.process_alive_threshold.ago
+ expected = SolidQueue::Process.where(kind: 'Worker').where('last_heartbeat_at > ?', alive_since).count
+ assert_equal expected, app.running_jobs
assert_equal Seek::Config.solr_enabled, app.search_enabled
end
+ test 'refresh only counts workers, not other solid queue process kinds' do
+ common = { hostname: 'test', last_heartbeat_at: Time.current, supervisor_id: nil, metadata: {} }
+ SolidQueue::Process.create!(**common, kind: 'Worker', name: 'worker-1', pid: 1)
+ SolidQueue::Process.create!(**common, kind: 'Dispatcher', name: 'dispatcher-1', pid: 2)
+ SolidQueue::Process.create!(**common, kind: 'Supervisor', name: 'supervisor-1', pid: 3)
+
+ app = ApplicationStatus.instance
+ app.refresh
+ app.reload
+ assert_equal 1, app.running_jobs
+ end
+
test 'search_enabled' do
with_config_value(:solr_enabled, true) do
assert ApplicationStatus.instance.search_enabled
diff --git a/test/unit/delayed_job_migrator_test.rb b/test/unit/delayed_job_migrator_test.rb
new file mode 100644
index 0000000000..3ca86d434d
--- /dev/null
+++ b/test/unit/delayed_job_migrator_test.rb
@@ -0,0 +1,155 @@
+require 'test_helper'
+
+class DelayedJobMigratorTest < ActiveSupport::TestCase
+ def setup
+ Delayed::Job.delete_all
+ SolidQueue::Job.delete_all
+ SolidQueue::ReadyExecution.delete_all
+ SolidQueue::ScheduledExecution.delete_all
+ ReindexingQueue.delete_all
+ @previous_delay_jobs = Delayed::Worker.delay_jobs
+ Delayed::Worker.delay_jobs = true # ensure enqueue writes a row rather than running inline
+ end
+
+ def teardown
+ Delayed::Worker.delay_jobs = @previous_delay_jobs
+ end
+
+ test 'migrates a pending job into Solid Queue preserving queue, priority and run_at' do
+ run_at = 5.minutes.from_now
+ job = build_job(AuthLookupUpdateJob.new, queue: 'authlookup', priority: 3)
+ dj = create_delayed_job(job, run_at: run_at)
+
+ result = Seek::DelayedJobMigrator.run
+
+ assert_equal 1, result.migrated
+ assert_equal 0, Delayed::Job.count
+ assert_equal 1, SolidQueue::Job.count
+
+ sq = SolidQueue::Job.last
+ assert_equal 'AuthLookupUpdateJob', sq.class_name
+ assert_equal 'authlookup', sq.queue_name
+ assert_equal 3, sq.priority
+ assert_equal job.job_id, sq.active_job_id
+ assert_in_delta run_at.to_f, sq.scheduled_at.to_f, 1
+ assert_equal 'AuthLookupUpdateJob', sq.arguments['job_class']
+ refute Delayed::Job.exists?(dj.id)
+ end
+
+ test 'a past-due job becomes ready and a future job stays scheduled' do
+ due = build_job(AuthLookupUpdateJob.new, queue: 'authlookup', priority: 3)
+ later = build_job(AuthLookupUpdateJob.new, queue: 'authlookup', priority: 3)
+ due_dj = create_delayed_job(due, run_at: 5.minutes.ago)
+ later_dj = create_delayed_job(later, run_at: 5.minutes.from_now)
+
+ Seek::DelayedJobMigrator.run
+
+ due_sq = SolidQueue::Job.find_by(active_job_id: due.job_id)
+ later_sq = SolidQueue::Job.find_by(active_job_id: later.job_id)
+
+ assert SolidQueue::ReadyExecution.exists?(job_id: due_sq.id), 'past-due job should be ready to run'
+ assert SolidQueue::ScheduledExecution.exists?(job_id: later_sq.id), 'future job should stay scheduled'
+ refute Delayed::Job.exists?(due_dj.id)
+ refute Delayed::Job.exists?(later_dj.id)
+ end
+
+ test 'folds the delayed_job attempts count into ActiveJob executions' do
+ job = build_job(AuthLookupUpdateJob.new, queue: 'authlookup', priority: 3)
+ create_delayed_job(job, attempts: 2)
+
+ Seek::DelayedJobMigrator.run
+
+ assert_equal 2, SolidQueue::Job.last.arguments['executions']
+ end
+
+ test 'deletes already-failed rows instead of migrating them' do
+ ok = build_job(AuthLookupUpdateJob.new, queue: 'authlookup', priority: 3)
+ failed = build_job(AuthLookupUpdateJob.new, queue: 'authlookup', priority: 3)
+ create_delayed_job(ok)
+ failed_dj = create_delayed_job(failed, failed_at: 1.hour.ago)
+
+ result = Seek::DelayedJobMigrator.run
+
+ assert_equal 1, result.migrated
+ assert_equal 1, result.failed_deleted
+ assert_equal 0, Delayed::Job.count
+ assert_equal 1, SolidQueue::Job.count
+ assert_equal ok.job_id, SolidQueue::Job.last.active_job_id
+ refute Delayed::Job.exists?(failed_dj.id)
+ end
+
+ test 'drops queued reindexing jobs instead of migrating them (reindex_all supersedes them)' do
+ keep = build_job(AuthLookupUpdateJob.new, queue: 'authlookup', priority: 3)
+ reindex_all = build_job(ReindexAllJob.new('DataFile'))
+ reindexing = build_job(ReindexingJob.new)
+ create_delayed_job(keep)
+ create_delayed_job(reindex_all)
+ create_delayed_job(reindexing)
+
+ result = Seek::DelayedJobMigrator.run
+
+ assert_equal 1, result.migrated
+ assert_equal 2, result.reindex_dropped
+ assert_equal 0, Delayed::Job.count
+ assert_equal 1, SolidQueue::Job.count
+ assert_equal keep.job_id, SolidQueue::Job.last.active_job_id
+ assert_empty SolidQueue::Job.where(class_name: %w[ReindexAllJob ReindexingJob])
+ end
+
+ test 'clears the ReindexingQueue (superseded by reindex_all)' do
+ sop = FactoryBot.create(:sop)
+ document = FactoryBot.create(:document)
+ ReindexingQueue.delete_all # ignore anything auto-enqueued when the records were created
+ with_config_value(:solr_enabled, true) do
+ ReindexingQueue.enqueue(sop, document, queue_job: false)
+ end
+ assert_equal 2, ReindexingQueue.count
+
+ result = Seek::DelayedJobMigrator.run
+
+ assert_equal 2, result.reindex_queue_cleared
+ assert_equal 0, ReindexingQueue.count
+ end
+
+ test 'migrates a job whose argument record has since been deleted without raising' do
+ content_blob = FactoryBot.create(:content_blob)
+ job = build_job(RemoteContentFetchingJob.new(content_blob))
+ create_delayed_job(job)
+ # the referenced record is gone by the time the migration runs
+ ContentBlob.where(id: content_blob.id).delete_all
+
+ result = nil
+ assert_nothing_raised { result = Seek::DelayedJobMigrator.run }
+
+ assert_equal 1, result.migrated
+ assert_equal 0, Delayed::Job.count
+ sq = SolidQueue::Job.last
+ assert_equal 'RemoteContentFetchingJob', sq.class_name
+ # the dangling GlobalID is preserved verbatim; it is only resolved (and handled) when run
+ assert_match(/ContentBlob\/#{content_blob.id}/, sq.arguments['arguments'].to_s)
+ end
+
+ test 'is a no-op when there are no delayed jobs' do
+ result = Seek::DelayedJobMigrator.run
+
+ assert_equal 0, result.migrated
+ assert_equal 0, result.failed_deleted
+ assert_equal 0, SolidQueue::Job.count
+ end
+
+ private
+
+ def build_job(active_job, queue: nil, priority: nil)
+ active_job.queue_name = queue if queue
+ active_job.priority = priority if priority
+ active_job
+ end
+
+ def create_delayed_job(active_job, run_at: 5.minutes.from_now, attempts: 0, failed_at: nil)
+ wrapper = ActiveJob::QueueAdapters::DelayedJobAdapter::JobWrapper.new(active_job.serialize)
+ dj = Delayed::Job.enqueue(wrapper, queue: active_job.queue_name, priority: active_job.priority, run_at: run_at)
+ dj.update_columns(attempts: attempts) unless attempts.zero?
+ dj.update_columns(failed_at: failed_at) if failed_at
+ dj
+ end
+end
diff --git a/test/unit/jobs/auth_lookup_maintenance_job_test.rb b/test/unit/jobs/auth_lookup_maintenance_job_test.rb
index 102aa8c5e2..b40509faf7 100644
--- a/test/unit/jobs/auth_lookup_maintenance_job_test.rb
+++ b/test/unit/jobs/auth_lookup_maintenance_job_test.rb
@@ -13,10 +13,6 @@ def setup
AuthLookupUpdateQueue.destroy_all
end
- test 'run period' do
- assert_equal 8.hours, AuthLookupMaintenanceJob::RUN_PERIOD
- end
-
test 'priority' do
assert_equal 3, AuthLookupMaintenanceJob.priority
end
diff --git a/test/unit/jobs/cache_overflow_cleanup_job_test.rb b/test/unit/jobs/cache_overflow_cleanup_job_test.rb
index c4957bf340..b918580428 100644
--- a/test/unit/jobs/cache_overflow_cleanup_job_test.rb
+++ b/test/unit/jobs/cache_overflow_cleanup_job_test.rb
@@ -5,10 +5,6 @@
class CacheOverflowCleanupJobTest < ActiveSupport::TestCase
MAX_SIZE = 200
- test 'run period' do
- assert_equal 1.day, CacheOverflowCleanupJob::RUN_PERIOD
- end
-
test 'removes only expired entries from the file overflow side' do
redis_store = ActiveSupport::Cache::RedisCacheStore.new(redis: MockRedis.new,
namespace: 'test-cache-overflow-job')
diff --git a/test/unit/jobs/news_feed_refresh_job_test.rb b/test/unit/jobs/news_feed_refresh_job_test.rb
new file mode 100644
index 0000000000..84dbd4c457
--- /dev/null
+++ b/test/unit/jobs/news_feed_refresh_job_test.rb
@@ -0,0 +1,29 @@
+require 'test_helper'
+require 'fugit'
+
+class NewsFeedRefreshJobTest < ActiveSupport::TestCase
+ test 'cron_schedule converts the cache timeout into a valid cron' do
+ {
+ 1 => '*/1 * * * *',
+ 30 => '*/30 * * * *',
+ 59 => '*/59 * * * *', # last value expressible as a minute step
+ 60 => '0 */1 * * *', # an hour - switches to an hourly step
+ 90 => '0 */2 * * *', # rounded up to two hours
+ 731 => '0 */12 * * *', # ~12 hours
+ 1440 => '0 0 */1 * *', # a day - switches to a daily step
+ 100_000 => '0 0 */31 * *' # capped at a monthly-ish day step
+ }.each do |timeout, expected|
+ with_config_value(:home_feeds_cache_timeout, timeout) do
+ assert_equal expected, NewsFeedRefreshJob.cron_schedule
+ assert Fugit.parse_cron(NewsFeedRefreshJob.cron_schedule),
+ "#{NewsFeedRefreshJob.cron_schedule.inspect} is not a valid cron for timeout #{timeout}"
+ end
+ end
+ end
+
+ test 'cron_schedule copes with a non-positive timeout' do
+ with_config_value(:home_feeds_cache_timeout, 0) do
+ assert_equal '*/1 * * * *', NewsFeedRefreshJob.cron_schedule
+ end
+ end
+end
diff --git a/test/unit/jobs/regular_maintenance_job_test.rb b/test/unit/jobs/regular_maintenance_job_test.rb
index e8e209796c..9b32bd6cf5 100644
--- a/test/unit/jobs/regular_maintenance_job_test.rb
+++ b/test/unit/jobs/regular_maintenance_job_test.rb
@@ -5,10 +5,6 @@ def setup
ContentBlob.destroy_all
end
- test 'run period' do
- assert_equal 4.hours, RegularMaintenanceJob::RUN_PERIOD
- end
-
test 'removes dangling content blobs' do
assert_equal 8.hours, RegularMaintenanceJob::REMOVE_DANGLING_BLOB_GRACE_PERIOD
to_go, keep1, keep2, keep3, keep4 = nil
diff --git a/test/unit/jobs/sitemap_refresh_job_test.rb b/test/unit/jobs/sitemap_refresh_job_test.rb
new file mode 100644
index 0000000000..020e6ceb9c
--- /dev/null
+++ b/test/unit/jobs/sitemap_refresh_job_test.rb
@@ -0,0 +1,22 @@
+require 'test_helper'
+require 'minitest/mock'
+
+class SitemapRefreshJobTest < ActiveSupport::TestCase
+ test 'regenerates the sitemap and pings search engines' do
+ ran = false
+ pinged = false
+
+ SitemapGenerator::Interpreter.stub(:run, ->(*) { ran = true }) do
+ SitemapGenerator::Sitemap.stub(:ping_search_engines, ->(*) { pinged = true }) do
+ SitemapRefreshJob.perform_now
+ end
+ end
+
+ assert ran, 'expected the sitemap to be regenerated'
+ assert pinged, 'expected search engines to be pinged'
+ end
+
+ test 'uses the default queue' do
+ assert_equal QueueNames::DEFAULT, SitemapRefreshJob.new.queue_name
+ end
+end
diff --git a/test/unit/util_test.rb b/test/unit/util_test.rb
index d9a1a9eacb..157e741837 100644
--- a/test/unit/util_test.rb
+++ b/test/unit/util_test.rb
@@ -1,4 +1,5 @@
require 'test_helper'
+require 'minitest/mock'
class UtilTest < ActiveSupport::TestCase
@@ -216,5 +217,33 @@ def teardown
assert_equal expected, Seek::Util.schema_org_supported_types.map(&:name)
end
+ test 'solid_queue_supervisor_pid returns nil when the pidfile is missing' do
+ SolidQueue.stub(:supervisor_pidfile, '/no/such/pidfile') do
+ assert_nil Seek::Util.solid_queue_supervisor_pid
+ end
+ end
+
+ test 'solid_queue_supervisor_pid interprets the running process' do
+ Tempfile.create('supervisor.pid') do |file|
+ file.write("12345\n")
+ file.rewind
+ SolidQueue.stub(:supervisor_pidfile, file.path) do
+ # Live process - signal 0 succeeds
+ Process.stub(:kill, 1) do
+ assert_equal 12_345, Seek::Util.solid_queue_supervisor_pid
+ end
+
+ # Stale pidfile - no such process
+ Process.stub(:kill, ->(*) { raise Errno::ESRCH }) do
+ assert_nil Seek::Util.solid_queue_supervisor_pid
+ end
+
+ # Process exists but owned by another user - still running
+ Process.stub(:kill, ->(*) { raise Errno::EPERM }) do
+ assert_equal 12_345, Seek::Util.solid_queue_supervisor_pid
+ end
+ end
+ end
+ end
end