seek4science · stuzart · Jul 15, 2026 · Jul 15, 2026 · Jul 15, 2026 · Jul 15, 2026
diff --git a/AGENTS.md b/AGENTS.md
@@ -12,8 +12,7 @@ FAIRDOM-SEEK (v1.18) is a Rails 7.2 / Ruby 3.3 web platform for sharing scientif
 
 ```bash
 bundle exec rails server          # Start the app (default: port 3000)
-bundle exec rake seek:workers:start  # Start delayed_job background workers
-bundle exec rake seek:workers:stop
+bundle exec bin/jobs               # Start Solid Queue background workers
 ```
 
 ### Testing
@@ -116,7 +115,7 @@ The REST API uses `active_model_serializers` with `BaseSerializer` (`app/seriali
 
 ### Background Jobs
 
-All async work uses `delayed_job` (ActiveRecord backend). Jobs are in `app/jobs/`. Start/stop workers via `rake seek:workers:start/stop`. Key jobs: `AuthLookupUpdateJob`, `ReindexingJob`, subscription email jobs, RDF generation.
+All async work uses `Solid Queue` (`solid_queue` gem, tables in the primary database). Jobs are in `app/jobs/`. Worker/dispatcher/scheduler processes are started via `bin/jobs` (see `config/queue.yml` for topology, `config/recurring.yml` for scheduled jobs); `script/run_solid_queue.sh` wraps this with automatic restart-on-exit for deployment. Key jobs: `AuthLookupUpdateJob`, `ReindexingJob`, subscription email jobs, RDF generation. (Migrated from `delayed_job` - the `delayed_job_active_record` gem and its tables remain installed as a rollback safety net for now.)
 
 ### Semantic / RDF
 

diff --git a/Dockerfile b/Dockerfile
@@ -86,7 +86,7 @@ RUN mkdir -p /var/www
 
 # Fix permissions
 RUN chown -R www-data:www-data /var/www config docker db/schema.rb public solr sqlite3-db
-RUN chmod -R 755 docker/upgrade.sh docker/start_workers.sh
+RUN chmod -R 755 docker/upgrade.sh docker/start_workers.sh script/run_solid_queue.sh
 
 # Python dependencies from requirements.txt
 RUN PYTHON_VERSION=$(cat .python-version) && \

diff --git a/Gemfile b/Gemfile
@@ -20,12 +20,12 @@ gem 'bundler'
 gem 'caxlsx'
 gem 'caxlsx_rails'
 gem 'cff'
+gem 'chronic' # AdminController parses human-readable duration settings (e.g. error grouping timeout) with Chronic
 gem 'citeproc-ruby'
 gem 'commonmarker'
 gem 'country_select'
 gem 'csl-styles'
 gem 'csv'
-gem 'daemons'
 gem 'delayed_job_active_record'
 gem 'docsplit', git: 'https://github.com/documentcloud/docsplit.git'
 gem 'doorkeeper'
@@ -55,6 +55,8 @@ gem 'loofah'
 gem 'mechanize'
 gem 'metainspector'
 gem 'mimemagic'
+# Experimental: web dashboard for inspecting/managing Solid Queue jobs (on the mission-control branch)
+gem 'mission_control-jobs'
 gem 'mutex_m'
 gem 'mysql2'
 gem 'net-ftp'
@@ -99,13 +101,13 @@ gem 'sass-rails'
 gem 'seedbank'
 gem 'simple-spreadsheet-extractor'
 gem 'sitemap_generator'
+gem 'solid_queue'
 gem 'sprockets-rails'
 gem 'sqlite3'
 gem 'sunspot_rails'
 gem 'terrapin'
 gem 'terser'
 gem 'validate_url'
-gem 'whenever', require: false
 gem 'will_paginate'
 gem 'yaml_db'
 gem 'zenodo-client', git: 'https://github.com/seek4science/zenodo-client.git'
@@ -145,7 +147,6 @@ group :test do
   gem 'test-prof'
   gem 'selenium-webdriver', '~> 4.44'
   gem 'vcr'
-  gem 'whenever-test'
 end
 
 group :test, :development do

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -226,7 +226,6 @@ GEM
     csl-styles (2.0.1)
       csl (~> 2.0)
     csv (3.3.3)
-    daemons (1.4.1)
     database_cleaner (2.1.0)
       database_cleaner-active_record (>= 2, < 3)
     database_cleaner-active_record (2.2.0)
@@ -265,6 +264,8 @@ GEM
     erb (6.0.4)
     erubi (1.13.1)
     erubis (2.7.0)
+    et-orbi (1.4.0)
+      tzinfo
     exception_notification (5.0.0)
       actionmailer (>= 7.1, < 9)
       activesupport (>= 7.1, < 9)
@@ -300,6 +301,9 @@ GEM
     ffi (1.17.1-x86_64-linux-gnu)
     flamegraph (0.9.5)
     forwardable (1.3.3)
+    fugit (1.13.0)
+      et-orbi (~> 1.4)
+      raabro (~> 1.4)
     gem-licenses (0.2.2)
     gitlab_omniauth-ldap (2.2.0)
       net-ldap (~> 0.16)
@@ -338,6 +342,10 @@ GEM
       concurrent-ruby (~> 1.0)
     i18n-js (3.9.2)
       i18n (>= 0.6.6)
+    importmap-rails (2.2.3)
+      actionpack (>= 6.0.0)
+      activesupport (>= 6.0.0)
+      railties (>= 6.0.0)
     indefinite_article (0.2.5)
       activesupport
     io-console (0.8.2)
@@ -499,6 +507,16 @@ GEM
       builder
       minitest (>= 5.0)
       ruby-progressbar
+    mission_control-jobs (1.1.0)
+      actioncable (>= 7.1)
+      actionpack (>= 7.1)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      importmap-rails (>= 1.2.1)
+      irb (~> 1.13)
+      railties (>= 7.1)
+      stimulus-rails
+      turbo-rails
     mock_redis (0.55.0)
       redis (~> 5)
     msgpack (1.8.2)
@@ -626,6 +644,7 @@ GEM
     puma (7.2.1)
       nio4r (~> 2.0)
     pyu-ruby-sasl (0.0.3.3)
+    raabro (1.4.0)
     racc (1.8.1)
     rack (3.2.6)
     rack-attack (6.7.0)
@@ -942,6 +961,13 @@ GEM
     snaky_hash (2.0.6)
       hashie (>= 0.1.0, < 6)
       version_gem (>= 1.1.8, < 3)
+    solid_queue (1.4.0)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11)
+      railties (>= 7.1)
+      thor (>= 1.3.1)
     sparql (3.3.2)
       builder (~> 3.2, >= 3.2.4)
       ebnf (~> 2.5)
@@ -967,6 +993,8 @@ GEM
     sqlite3 (2.9.5-arm64-darwin)
     sqlite3 (2.9.5-x86_64-linux-gnu)
     stackprof (0.2.27)
+    stimulus-rails (1.3.4)
+      railties (>= 6.0.0)
     stringio (3.2.0)
     sunspot (2.7.1)
       bigdecimal
@@ -1001,6 +1029,9 @@ GEM
       date
     timeout (0.6.1)
     tsort (0.2.0)
+    turbo-rails (2.0.23)
+      actionpack (>= 7.1.0)
+      railties (>= 7.1.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     ucf (3.0.0)
@@ -1040,10 +1071,6 @@ GEM
       base64
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    whenever (1.0.0)
-      chronic (>= 0.6.3)
-    whenever-test (1.0.1)
-      whenever
     will_paginate (4.0.1)
     yaml-ld (0.0.3)
       json-ld (~> 3.3)
@@ -1087,12 +1114,12 @@ DEPENDENCIES
   caxlsx
   caxlsx_rails
   cff
+  chronic
   citeproc-ruby
   commonmarker
   country_select
   csl-styles
   csv
-  daemons
   database_cleaner
   delayed_job_active_record
   docsplit!
@@ -1130,6 +1157,7 @@ DEPENDENCIES
   minitar
   minitest (~> 5.4)
   minitest-reporters
+  mission_control-jobs
   mock_redis
   mutex_m
   mysql2
@@ -1193,6 +1221,7 @@ DEPENDENCIES
   simple-spreadsheet-extractor
   simplecov
   sitemap_generator
+  solid_queue
   sprockets-rails
   sqlite3
   stackprof
@@ -1207,8 +1236,6 @@ DEPENDENCIES
   vcr
   web-console
   webmock
-  whenever
-  whenever-test
   will_paginate
   yaml_db
   zenodo-client!

diff --git a/app/controllers/admin_controller.rb b/app/controllers/admin_controller.rb
@@ -1,5 +1,3 @@
-require 'delayed/command'
-
 class AdminController < ApplicationController
   include CommonSweepers
 
@@ -376,10 +374,15 @@ def restart_server
     redirect_with_status(error, 'server')
   end
 
-  def restart_delayed_job
-    command = "bundle exec rake seek:workers:restart"
-    error = execute_command(command)
-    redirect_with_status(error, 'background job workers')
+  def restart_job_workers
+    pid = Seek::Util.solid_queue_supervisor_pid
+    if pid
+      error = execute_command("kill -TERM #{pid}")
+      redirect_with_status(error, 'background job workers')
+    else
+      flash[:error] = 'There are no background job workers running to restart.'
+      redirect_to action: :show
+    end
   end
 
   def clear_cache
@@ -390,17 +393,6 @@ def clear_cache
     end
   end
 
-  # give it up to 5 seconds to start up, otherwise the page reloads too quickly and says it is not running
-  def wait_for_delayed_job_to_start
-    sleep(0.5)
-    pid = Daemons::PidFile.new("#{Rails.root}/tmp/pids", 'delayed_job.0')
-    count = 0
-    while !pid.running? && (count < 10)
-      sleep(0.5)
-      count += 1
-    end
-  end
-
   def edit_tag
     @tag = TextValue.find(params[:id])
     if request.post?
@@ -627,9 +619,9 @@ def array_remove_at(array, index)
     return array.slice(0,index) + array.slice(index+1,array.length)
   end
 
-  # this destroys any failed Delayed::Jobs
+  # this destroys any failed SolidQueue::Jobs
   def clear_failed_jobs
-    Delayed::Job.where('failed_at IS NOT NULL').destroy_all
+    SolidQueue::Job.failed.destroy_all
     respond_to do |format|
       format.json { head :ok }
     end

diff --git a/app/controllers/mission_control_jobs_controller.rb b/app/controllers/mission_control_jobs_controller.rb
@@ -0,0 +1,8 @@
+# Base controller for the Mission Control - Jobs dashboard (mounted at /jobs). The engine's own
+# controllers inherit from this, so gating it here reuses SEEK's existing admin authentication
+# (login_required + is_user_admin_auth) rather than the gem's default HTTP Basic auth - only a
+# logged-in admin can reach the queue dashboard.
+class MissionControlJobsController < ApplicationController
+  before_action :login_required
+  before_action :is_user_admin_auth
+end
diff --git a/app/jobs/auth_lookup_maintenance_job.rb b/app/jobs/auth_lookup_maintenance_job.rb
@@ -1,6 +1,4 @@
 class AuthLookupMaintenanceJob < ApplicationJob
-  RUN_PERIOD = 8.hours.freeze
-
   queue_as QueueNames::AUTH_LOOKUP
   queue_with_priority  3
 

diff --git a/app/jobs/cache_overflow_cleanup_job.rb b/app/jobs/cache_overflow_cleanup_job.rb
@@ -4,8 +4,6 @@
 # rising is the signal that the shared Redis instance is under enough memory pressure that session
 # keys could, in principle, start being evicted too.
 class CacheOverflowCleanupJob < ApplicationJob
-  RUN_PERIOD = 1.day.freeze
-
   def perform
     Rails.cache.cleanup
     log_redis_memory_stats

diff --git a/app/jobs/life_monitor_status_job.rb b/app/jobs/life_monitor_status_job.rb
@@ -1,6 +1,4 @@
 class LifeMonitorStatusJob < ApplicationJob
-  PERIOD = 1.day.freeze
-
   def perform(offset = 0)
     return unless Seek::Config.life_monitor_enabled
     token = LifeMonitor::Oauth2::Client.new.get_token

diff --git a/app/jobs/regular_maintenance_job.rb b/app/jobs/regular_maintenance_job.rb
@@ -1,12 +1,11 @@
 # A job that runs regularly and performs certain general maintenance tasks
-# the jobs is run periodically according to RUN_PERIOD
+# the job is run periodically, according to the schedule set in config/recurring.yml
 # short running, simple, maintenance operations can be added here, complex, or longer running operations should spawn a
 # new job specific to the operation
 
 require 'rake'
 
 class RegularMaintenanceJob < ApplicationJob
-  RUN_PERIOD = 4.hours.freeze
   REMOVE_DANGLING_BLOB_GRACE_PERIOD = 8.hours.freeze
   REMOVE_DELETED_BLOB_GRACE_PERIOD = 24.hours.freeze
   REPO_GRACE_PERIOD = 8.hours.freeze

diff --git a/app/jobs/sitemap_refresh_job.rb b/app/jobs/sitemap_refresh_job.rb
@@ -0,0 +1,8 @@
+# Regenerates the XML sitemap from config/sitemap.rb and pings search engines. Scheduled
+# periodically via config/recurring.yml. verbose: false keeps it quiet in the job logs.
+class SitemapRefreshJob < ApplicationJob
+  def perform
+    SitemapGenerator::Interpreter.run(verbose: false)
+    SitemapGenerator::Sitemap.ping_search_engines
+  end
+end
diff --git a/app/models/application_status.rb b/app/models/application_status.rb
@@ -4,8 +4,9 @@ class ApplicationStatus < ApplicationRecord
     before_create :validate_singleton
 
     def refresh
+        alive_since = SolidQueue.process_alive_threshold.ago
         update(
-            running_jobs: Seek::Util.delayed_job_pids.count
+            running_jobs: SolidQueue::Process.where(kind: 'Worker').where('last_heartbeat_at > ?', alive_since).count
          )
     end
 

diff --git a/app/views/admin/_general.html.erb b/app/views/admin/_general.html.erb
@@ -29,6 +29,7 @@
       <h4>Management</h4>
       <ul>
         <li><%= link_to 'Dashboard', dashboard_admin_stats_path -%></li>
+        <li><%= link_to 'Job queue dashboard', mission_control_jobs_path -%></li>
         <li><%= link_to 'Announcements', site_announcements_path -%></li>
         <li><%= link_to 'Manage Assay Types', suggested_assay_types_path %></li>
         <li><%= link_to 'Manage Technology Types', suggested_technology_types_path %></li>