From 7b9edc4de76d7ac01332f58c3860a82883399091 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Fri, 26 Jun 2026 12:24:27 -0700 Subject: [PATCH 01/20] [scheduler/cuebot] Replace Redis-backed accounting with in-memory store + PG LISTEN/NOTIFY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The accounting subsystem coordinated Cuebot and the Rust scheduler through Redis to enable horizontal scaling across N scheduler instances. The scheduler is and will remain single-instance (N=1), so Redis's only unique benefit is unreachable while it manufactured an entire class of accounting-drift bugs (limit-seeding fail-closed, mass dispatch rejection, double-booking, CAS starvation). A single in-process counter is the source of truth that makes that bug class structurally impossible. All shows were already drained to Cuebot-managed, so the cutover is clean (deploy into an idle role, then flip shows back one at a time via `b_scheduler_managed`). **Scheduler** — Redis → in-memory `Store`: - `accounting/store.rs`: one `Mutex`, atomic check-and-increment across the three enforced vertices (subscription burst, folder/job max cores+gpus). Layer/point were incremented but never read, so they're dropped. - Live updates via PG `LISTEN/NOTIFY` (`accounting/listener.rs`): `acct_release` and `acct_limit_change`. - Recompute from `SUM(proc)` is the backstop (absolute overwrite, no CAS), carrying in-flight bookings forward via an epoch double-buffer so it can never erase a not-yet-snapshot-visible booking → never over-books a hard cap. - Blocking seeds gate dispatch: bootstrap and managed-flip both seed caps **and** booked counters before enforcing. - Deleted `redis_client.rs`, `lua.rs`, `acct:seq`/CAS, the `redis` dependency. **Cuebot** — `LettuceAccountingRedisPublisher` → `AccountingNotifier`: - Transactional `pg_notify` on proc release (same txn as `DELETE proc` → delivered iff it commits, a stronger model than the old afterCommit publish) and on the five enforced admin cap changes. - `accounting.redis.*` and the Lettuce dep removed; replaced by a safe `accounting.notify.enabled` kill-switch (off → scheduler degrades to recompute-only, which under-books, never over-books). The old over-booking startup guardrail is gone. **Docs** — `redis-accounting.md` → `scheduler-accounting.md` (full rewrite) plus `scheduler.md`, `deploying-scheduler.md`, stress-testing, and properties. - Caps are hard (license/OOM); every failure mode is safe-direction: a dropped NOTIFY leaves a counter reading high → under-book → healed by the next recompute. - N=1 is now an assumption of the in-memory design; multi-scheduler would need a shared store again (revisit trigger documented). - Rust: 177 lib tests pass (incl. straddle, managed-flip, dropped-NOTIFY invariants); clippy clean; stress suite compiles; bin builds. - Cuebot: `compileJava`/`compileTestJava`/`spotlessJavaCheck` pass (JDK 11). rollback). --- cuebot/build.gradle | 4 - .../com/imageworks/spcue/dao/ShowDao.java | 4 +- .../spcue/dao/postgres/GroupDaoJdbc.java | 17 + .../spcue/dao/postgres/JobDaoJdbc.java | 17 + .../spcue/dao/postgres/ProcDaoJdbc.java | 62 +- .../dao/postgres/SubscriptionDaoJdbc.java | 17 + .../spcue/service/AccountingNotifier.java | 212 +++++ .../service/AccountingRedisPublisher.java | 44 - .../LettuceAccountingRedisPublisher.java | 184 ---- .../V45__show_scheduler_managed.sql | 2 +- .../spring/applicationContext-accounting.xml | 18 +- cuebot/src/main/resources/opencue.properties | 16 +- .../spcue/test/dao/postgres/ProcDaoTests.java | 4 +- .../test/service/AccountingNotifierTests.java | 160 ++++ .../LettuceAccountingRedisPublisherTests.java | 207 ----- .../_docs/developer-guide/redis-accounting.md | 603 -------------- .../developer-guide/scheduler-accounting.md | 515 ++++++++++++ .../scheduler-stress-testing.md | 79 +- docs/_docs/developer-guide/scheduler.md | 217 ++--- .../getting-started/deploying-scheduler.md | 2 +- rust/crates/scheduler/Cargo.toml | 7 +- .../scheduler/src/accounting/booking_delta.rs | 100 +-- .../scheduler/src/accounting/bootstrap.rs | 22 +- rust/crates/scheduler/src/accounting/dao.rs | 186 ++--- rust/crates/scheduler/src/accounting/error.rs | 42 +- .../scheduler/src/accounting/limit_reseed.rs | 216 ++--- .../scheduler/src/accounting/listener.rs | 239 ++++++ rust/crates/scheduler/src/accounting/lua.rs | 141 ---- .../scheduler/src/accounting/managed_shows.rs | 74 +- rust/crates/scheduler/src/accounting/mod.rs | 162 ++-- .../scheduler/src/accounting/recompute.rs | 486 ++--------- .../scheduler/src/accounting/redis_client.rs | 313 ------- rust/crates/scheduler/src/accounting/store.rs | 786 ++++++++++++++++++ rust/crates/scheduler/src/cluster.rs | 2 +- rust/crates/scheduler/src/cluster_key.rs | 2 +- rust/crates/scheduler/src/config/mod.rs | 44 +- rust/crates/scheduler/src/dao/job_dao.rs | 2 +- rust/crates/scheduler/src/metrics/mod.rs | 6 +- rust/crates/scheduler/src/models/core_size.rs | 4 +- .../src/pipeline/dispatcher/actor.rs | 105 +-- .../src/pipeline/dispatcher/error.rs | 12 - .../src/pipeline/dispatcher/messages.rs | 2 +- .../scheduler/src/pipeline/entrypoint.rs | 12 +- rust/crates/scheduler/src/pipeline/matcher.rs | 73 +- .../scheduler/src/pipeline/placement.rs | 12 +- .../scheduler/tests/active_scan_superset.rs | 5 +- .../scheduler/tests/redis_integration.rs | 481 ----------- rust/crates/scheduler/tests/stress_tests.rs | 53 +- rust/crates/scheduler/tests/util/stress.rs | 168 ++-- 49 files changed, 2715 insertions(+), 3426 deletions(-) create mode 100644 cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/service/AccountingRedisPublisher.java delete mode 100644 cuebot/src/main/java/com/imageworks/spcue/service/LettuceAccountingRedisPublisher.java create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/service/AccountingNotifierTests.java delete mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/service/LettuceAccountingRedisPublisherTests.java delete mode 100644 docs/_docs/developer-guide/redis-accounting.md create mode 100644 docs/_docs/developer-guide/scheduler-accounting.md create mode 100644 rust/crates/scheduler/src/accounting/listener.rs delete mode 100644 rust/crates/scheduler/src/accounting/lua.rs delete mode 100644 rust/crates/scheduler/src/accounting/redis_client.rs create mode 100644 rust/crates/scheduler/src/accounting/store.rs delete mode 100644 rust/crates/scheduler/tests/redis_integration.rs diff --git a/cuebot/build.gradle b/cuebot/build.gradle index d8ca0ed757..6e30c1e8cb 100644 --- a/cuebot/build.gradle +++ b/cuebot/build.gradle @@ -38,10 +38,6 @@ dependencies { implementation group: 'com.sun.mail', name: 'mailapi', version: '1.5.4' implementation group: 'commons-lang', name: 'commons-lang', version: '2.6' implementation group: 'io.grpc', name: 'grpc-all', version: "${grpcVersion}" - implementation group: 'io.lettuce', name: 'lettuce-core', version: '6.3.2.RELEASE' - // Lettuce 6.3.x requires reactor-core 3.6.x (for reactor.core.publisher.Sinks). - // Spring Boot 2.2.1's dependency management would otherwise pin this to 3.3.x. - implementation group: 'io.projectreactor', name: 'reactor-core', version: '3.6.5' implementation group: 'org.apache.activemq', name: 'activemq-pool', version: activemqVersion implementation group: 'org.apache.velocity', name: 'velocity', version: '1.7' implementation group: 'org.jdom', name: 'jdom', version: '1.1.3' diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java index 0eac3ee0ee..25a55b5059 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/ShowDao.java @@ -152,8 +152,8 @@ public interface ShowDao { boolean isSchedulerManaged(String showId); /** - * Return the number of shows currently flagged scheduler-managed. Used at boot by the Redis - * accounting publisher to emit the deployment-invariant misconfiguration warning. + * Return the number of shows currently flagged scheduler-managed. Used at boot by the + * accounting NOTIFY publisher to emit a visibility warning when publishing is disabled. * * @return count of rows with b_scheduler_managed = true */ diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java index f9dc43c9ac..5aa06d1ec0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.CallableStatementCreator; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.SqlParameter; @@ -36,7 +37,9 @@ import com.imageworks.spcue.JobInterface; import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.dao.GroupDao; +import com.imageworks.spcue.dao.ShowDao; import com.imageworks.spcue.grpc.job.JobState; +import com.imageworks.spcue.service.AccountingNotifier; import com.imageworks.spcue.util.CueUtil; import com.imageworks.spcue.util.SqlUtil; @@ -44,6 +47,12 @@ public class GroupDaoJdbc extends JdbcDaoSupport implements GroupDao { private static final int MAX_NESTING_LEVEL = 10; + @Autowired + private ShowDao showDao; + + @Autowired + private AccountingNotifier accountingNotifier; + @Override public String getRootGroupId(ShowInterface show) { return getJdbcTemplate().queryForObject( @@ -200,6 +209,10 @@ public void updateMaxCores(GroupInterface group, int value) { getJdbcTemplate().update("UPDATE folder_resource SET int_max_cores=? WHERE pk_folder=?", value, group.getId()); + + if (accountingNotifier.isEnabled() && showDao.isSchedulerManaged(group.getShowId())) { + accountingNotifier.notifyFolderMaxCores(group.getId(), value); + } } @Override @@ -258,6 +271,10 @@ public void updateMaxGpus(GroupInterface group, int value) { getJdbcTemplate().update("UPDATE folder_resource SET int_max_gpus=? WHERE pk_folder=?", value, group.getId()); + + if (accountingNotifier.isEnabled() && showDao.isSchedulerManaged(group.getShowId())) { + accountingNotifier.notifyFolderMaxGpus(group.getId(), value); + } } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index 28db874454..a35c84ba6c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -29,6 +29,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; @@ -49,8 +50,10 @@ import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.TaskEntity; import com.imageworks.spcue.dao.JobDao; +import com.imageworks.spcue.dao.ShowDao; import com.imageworks.spcue.grpc.job.FrameState; import com.imageworks.spcue.grpc.job.JobState; +import com.imageworks.spcue.service.AccountingNotifier; import com.imageworks.spcue.util.CueUtil; import com.imageworks.spcue.util.JobLogUtil; import com.imageworks.spcue.util.SqlUtil; @@ -58,6 +61,12 @@ public class JobDaoJdbc extends JdbcDaoSupport implements JobDao { private static final Pattern LAST_JOB_STRIP_PATTERN = Pattern.compile("_v*([_0-9]*$)"); + @Autowired + private ShowDao showDao; + + @Autowired + private AccountingNotifier accountingNotifier; + /* * Maps a row to a DispatchJob object */ @@ -370,6 +379,10 @@ public void updateMinCores(JobInterface j, int v) { public void updateMaxCores(JobInterface j, int v) { getJdbcTemplate().update("UPDATE job_resource SET int_max_cores=? WHERE pk_job=?", v, j.getJobId()); + + if (accountingNotifier.isEnabled() && showDao.isSchedulerManaged(j.getShowId())) { + accountingNotifier.notifyJobMaxCores(j.getJobId(), v); + } } @Override @@ -398,6 +411,10 @@ public void updateMinGpus(JobInterface j, int v) { public void updateMaxGpus(JobInterface j, int v) { getJdbcTemplate().update("UPDATE job_resource SET int_max_gpus=? WHERE pk_job=?", v, j.getJobId()); + + if (accountingNotifier.isEnabled() && showDao.isSchedulerManaged(j.getShowId())) { + accountingNotifier.notifyJobMaxGpus(j.getJobId(), v); + } } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index b769e33a8e..3400fd5e46 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -33,8 +33,6 @@ import org.springframework.jdbc.core.PreparedStatementCreator; import org.springframework.jdbc.core.RowMapper; import org.springframework.jdbc.core.support.JdbcDaoSupport; -import org.springframework.transaction.support.TransactionSynchronizationAdapter; -import org.springframework.transaction.support.TransactionSynchronizationManager; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.HostInterface; @@ -51,7 +49,7 @@ import com.imageworks.spcue.dispatcher.ResourceDuplicationFailureException; import com.imageworks.spcue.dispatcher.ResourceReservationFailureException; import com.imageworks.spcue.grpc.host.HardwareState; -import com.imageworks.spcue.service.AccountingRedisPublisher; +import com.imageworks.spcue.service.AccountingNotifier; import com.imageworks.spcue.util.SqlUtil; public class ProcDaoJdbc extends JdbcDaoSupport implements ProcDao { @@ -65,7 +63,7 @@ public class ProcDaoJdbc extends JdbcDaoSupport implements ProcDao { private ShowDao showDao; @Autowired - private AccountingRedisPublisher accountingRedisPublisher; + private AccountingNotifier accountingNotifier; // spotless:off private static final String VERIFY_RUNNING_PROC = @@ -759,9 +757,9 @@ public void updateReservedMemory(ProcInterface p, long value) { * For shows flagged {@code b_scheduler_managed=true} (and non-local dispatch), the five PG * accounting tables (subscription / layer_resource / job_resource / folder_resource / point) * are not decremented here — the standalone Rust scheduler owns recompute from - * {@code SUM(proc)} on a few minutes cadence. Instead, a Redis delta is published after the - * surrounding Postgres transaction commits. The host idle counters always update because - * Cuebot's own host-report path consumes them regardless of who owns dispatch. + * {@code SUM(proc)} on a few minutes cadence. Instead, a release delta is emitted via Postgres + * {@code NOTIFY} inside this transaction. The host idle counters always update because Cuebot's + * own host-report path consumes them regardless of who owns dispatch. * *

* Local dispatches are always Cuebot-managed, regardless of the show flag. @@ -802,9 +800,10 @@ private void procDestroyed(VirtualProc proc) { } if (showDao.isSchedulerManaged(proc.getShowId())) { - // Skip the five PG accounting tables; the Rust scheduler owns recompute. Publish a - // release delta to Redis once the surrounding transaction commits. - registerAfterCommitRedisPublish(proc); + // Skip the five PG accounting tables; the Rust scheduler owns recompute. Emit a release + // delta via NOTIFY inside this (the unbook) transaction so it is delivered iff the + // DELETE proc commits. + accountingNotifier.notifyRelease(proc); return; } @@ -837,49 +836,6 @@ private void procDestroyed(VirtualProc proc) { proc.coresReserved, proc.gpusReserved, proc.getJobId(), proc.getJobId()); } - /** - * Register an afterCommit hook to publish the Redis release delta. The Redis publish must not - * run on rollback or it would over-decrement. - * - *

- * folderId, deptId, and allocationId are read directly from the {@link VirtualProc} fields - * hydrated by {@link #VIRTUAL_PROC_MAPPER}. A defensive fallback populates folderId/deptId from - * the job table and allocationId from the host table if a caller built the proc manually - * instead of going through a SELECT. - */ - private void registerAfterCommitRedisPublish(final VirtualProc proc) { - if (proc.folderId == null || proc.deptId == null) { - Map jobMeta = getJdbcTemplate().queryForMap( - "SELECT pk_folder, pk_dept FROM job WHERE pk_job=?", proc.getJobId()); - proc.folderId = (String) jobMeta.get("pk_folder"); - proc.deptId = (String) jobMeta.get("pk_dept"); - } - - // allocationId is sourced from host.pk_alloc (see VIRTUAL_PROC_MAPPER), not the job row, so - // backfill it separately to avoid publishing to acct:sub::null. - if (proc.getAllocationId() == null) { - proc.allocationId = getJdbcTemplate().queryForObject( - "SELECT pk_alloc FROM host WHERE pk_host=?", String.class, proc.getHostId()); - } - - if (TransactionSynchronizationManager.isSynchronizationActive()) { - TransactionSynchronizationManager - .registerSynchronization(new TransactionSynchronizationAdapter() { - @Override - public void afterCommit() { - accountingRedisPublisher.publishRelease(proc); - } - }); - } else { - // procDestroyed is always called from within a transactional service method - // (DispatchSupportService is @Transactional). Defensive fallback if that invariant - // changes. - logger.warn("procDestroyed called outside a transaction; publishing Redis delta " - + "synchronously for proc {}", proc.getProcId()); - accountingRedisPublisher.publishRelease(proc); - } - } - /** * Updates proc counts for the host, subscription, layer, job, folder, and proc point when a new * proc is created. diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SubscriptionDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SubscriptionDaoJdbc.java index 520f67b678..f04126a865 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SubscriptionDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SubscriptionDaoJdbc.java @@ -18,6 +18,7 @@ import java.sql.ResultSet; import java.sql.SQLException; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.jdbc.core.RowMapper; @@ -29,11 +30,19 @@ import com.imageworks.spcue.SubscriptionEntity; import com.imageworks.spcue.SubscriptionInterface; import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dao.ShowDao; import com.imageworks.spcue.dao.SubscriptionDao; +import com.imageworks.spcue.service.AccountingNotifier; import com.imageworks.spcue.util.SqlUtil; public class SubscriptionDaoJdbc extends JdbcDaoSupport implements SubscriptionDao { + @Autowired + private ShowDao showDao; + + @Autowired + private AccountingNotifier accountingNotifier; + // spotless:off private static final String IS_SHOW_OVER_SIZE = "SELECT " @@ -232,5 +241,13 @@ public void updateSubscriptionSize(SubscriptionInterface sub, int size) { public void updateSubscriptionBurst(SubscriptionInterface sub, int size) { getJdbcTemplate().update("UPDATE subscription SET int_burst=? WHERE pk_subscription=?", size, sub.getSubscriptionId()); + + // SubscriptionInterface extends both ShowInterface and AllocationInterface, so the show and + // alloc ids are available directly (the gRPC path passes a fully-hydrated + // SubscriptionEntity). Emit the burst cap change in this transaction for managed shows. + if (accountingNotifier.isEnabled() && showDao.isSchedulerManaged(sub.getShowId())) { + accountingNotifier.notifySubscriptionBurst(sub.getShowId(), sub.getAllocationId(), + size); + } } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java new file mode 100644 index 0000000000..6b6b2b188c --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java @@ -0,0 +1,212 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.service; + +import java.util.Map; + +import javax.annotation.PostConstruct; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.jdbc.core.support.JdbcDaoSupport; + +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dao.ShowDao; + +/** + * Publishes per-release and admin cap-change accounting deltas to the standalone Rust scheduler via + * Postgres {@code LISTEN/NOTIFY}. Every notification is emitted with {@code pg_notify(channel, + * payload)} inside the same transaction as the DB write it describes, so the payload is + * delivered if and only if that transaction commits (this replaces the old afterCommit Redis + * publish with a transactional, stronger failure model). + * + *

+ * See the Scheduler Accounting Reference at + * {@code docs/_docs/developer-guide/scheduler-accounting.md} for the wire contract. + * + *

+ * Gated by {@code accounting.notify.enabled} (default true). When false, every method is a no-op; + * the scheduler then degrades to recompute-only from {@code SUM(proc)}, which under-books rather + * than over-books, so disabling is safe (visibility-only WARN on startup, not a hard guard). + * + *

+ * Unit invariant: Cuebot stores cores as centicores (cores × 100). The scheduler accounts in cores, + * so cores are integer-divided by 100 on the way out. GPUs are stored in whole units and pass + * through unconverted. The {@code -1} "unlimited" sentinel on max caps is preserved. + */ +public class AccountingNotifier extends JdbcDaoSupport { + + private static final Logger logger = LogManager.getLogger(AccountingNotifier.class); + + /** NOTIFY channel for proc releases (negated cores/gpus deltas). */ + static final String CHANNEL_RELEASE = "acct_release"; + + /** NOTIFY channel for enforced admin cap changes (subscription burst, folder/job max). */ + static final String CHANNEL_LIMIT_CHANGE = "acct_limit_change"; + + /** + * Cuebot stores cores as centicores (cores × 100; see VirtualProc.java). The scheduler accounts + * in cores, so we divide on the way out. The divide is exact: VirtualProc forces coresReserved + * to a multiple of 100 at booking time, and cap values are whole-core multiples of 100. + */ + static final int CENTICORES_PER_CORE = 100; + + @Autowired + private Environment env; + + @Autowired + private ShowDao showDao; + + private volatile boolean notifyEnabled = true; + + @PostConstruct + public void initialize() { + notifyEnabled = env.getProperty("accounting.notify.enabled", Boolean.class, true); + if (!notifyEnabled) { + // Visibility only (NOT a hard guard): flag-off degrades the scheduler to + // recompute-only from SUM(proc), which under-books rather than over-books. + int managedCount = showDao.countSchedulerManagedShows(); + if (managedCount > 0) { + logger.warn("accounting.notify.enabled=false but {} scheduler-managed show(s) " + + "exist; the standalone scheduler will rely on periodic recompute only " + + "(no live release/limit-change deltas).", managedCount); + } + logger.info("Accounting NOTIFY publishing disabled (accounting.notify.enabled=false)"); + } else { + logger.info("Accounting NOTIFY publishing enabled"); + } + } + + /** True when this notifier actually emits pg_notify. */ + public boolean isEnabled() { + return notifyEnabled; + } + + /** + * Emit an {@code acct_release} delta for a destroyed proc on the {@code acct_release} channel. + * Cores and GPUs are NEGATED (a release returns resources). Must be called from within the same + * transaction as the {@code DELETE proc} it describes. + * + *

+ * folderId, deptId, and allocationId are read directly from the {@link VirtualProc} fields + * hydrated by {@code ProcDaoJdbc.VIRTUAL_PROC_MAPPER}. A defensive fallback populates + * folderId/deptId from the job table and allocationId from the host table if a caller built the + * proc manually instead of going through a SELECT. + */ + public void notifyRelease(VirtualProc proc) { + if (!notifyEnabled) { + return; + } + + if (proc.folderId == null || proc.deptId == null) { + Map jobMeta = getJdbcTemplate().queryForMap( + "SELECT pk_folder, pk_dept FROM job WHERE pk_job=?", proc.getJobId()); + proc.folderId = (String) jobMeta.get("pk_folder"); + proc.deptId = (String) jobMeta.get("pk_dept"); + } + + // allocationId is sourced from host.pk_alloc (see VIRTUAL_PROC_MAPPER), not the job row, so + // backfill it separately to avoid publishing an alloc of null. + if (proc.getAllocationId() == null) { + proc.allocationId = getJdbcTemplate().queryForObject( + "SELECT pk_alloc FROM host WHERE pk_host=?", String.class, proc.getHostId()); + } + + int cores = -proc.coresReserved / CENTICORES_PER_CORE; + int gpus = -proc.gpusReserved; + String payload = String.format( + "{\"show\":\"%s\",\"alloc\":\"%s\",\"folder\":\"%s\",\"job\":\"%s\"," + + "\"layer\":\"%s\",\"dept\":\"%s\",\"cores\":%d,\"gpus\":%d}", + proc.getShowId(), proc.getAllocationId(), proc.folderId, proc.getJobId(), + proc.getLayerId(), proc.deptId, cores, gpus); + notify(CHANNEL_RELEASE, payload); + } + + /** Emit a subscription-burst cap change. {@code burst} is in centicores. */ + public void notifySubscriptionBurst(String showId, String allocId, int burst) { + if (!notifyEnabled) { + return; + } + String payload = + String.format("{\"vertex\":\"sub\",\"show\":\"%s\",\"alloc\":\"%s\",\"burst\":%d}", + showId, allocId, burst / CENTICORES_PER_CORE); + notify(CHANNEL_LIMIT_CHANGE, payload); + } + + /** + * Emit a folder max-cores cap change. {@code value} is in centicores; <0 means unlimited. + */ + public void notifyFolderMaxCores(String folderId, int value) { + if (!notifyEnabled) { + return; + } + int maxCores = value < 0 ? -1 : value / CENTICORES_PER_CORE; + String payload = String.format("{\"vertex\":\"folder\",\"id\":\"%s\",\"max_cores\":%d}", + folderId, maxCores); + notify(CHANNEL_LIMIT_CHANGE, payload); + } + + /** + * Emit a folder max-gpus cap change. GPUs pass through unconverted ({@code -1} = unlimited). + */ + public void notifyFolderMaxGpus(String folderId, int value) { + if (!notifyEnabled) { + return; + } + String payload = String.format("{\"vertex\":\"folder\",\"id\":\"%s\",\"max_gpus\":%d}", + folderId, value); + notify(CHANNEL_LIMIT_CHANGE, payload); + } + + /** Emit a job max-cores cap change. {@code value} is in centicores; <0 means unlimited. */ + public void notifyJobMaxCores(String jobId, int value) { + if (!notifyEnabled) { + return; + } + int maxCores = value < 0 ? -1 : value / CENTICORES_PER_CORE; + String payload = String.format("{\"vertex\":\"job\",\"id\":\"%s\",\"max_cores\":%d}", jobId, + maxCores); + notify(CHANNEL_LIMIT_CHANGE, payload); + } + + /** Emit a job max-gpus cap change. GPUs pass through unconverted ({@code -1} = unlimited). */ + public void notifyJobMaxGpus(String jobId, int value) { + if (!notifyEnabled) { + return; + } + String payload = + String.format("{\"vertex\":\"job\",\"id\":\"%s\",\"max_gpus\":%d}", jobId, value); + notify(CHANNEL_LIMIT_CHANGE, payload); + } + + /** + * Issue the {@code pg_notify} inside the current transaction. All payload fields are UUIDs or + * integers (built via String.format), so no JSON escaping is required. + * + *

+ * {@code pg_notify} is a {@code SELECT} that returns a (void) result set, so it MUST be run as + * a query, not via {@code update()}/{@code executeUpdate()} — pgjdbc throws "A result was + * returned when none was expected" from {@code executeUpdate()}, which would roll back the + * surrounding unbook/admin transaction. {@code queryForList} routes through + * {@code executeQuery()} and discards the single void row; the NOTIFY still queues against the + * transaction-bound connection and is delivered on commit. + */ + private void notify(String channel, String payload) { + getJdbcTemplate().queryForList("SELECT pg_notify(?, ?)", channel, payload); + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingRedisPublisher.java b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingRedisPublisher.java deleted file mode 100644 index bf825e9b25..0000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingRedisPublisher.java +++ /dev/null @@ -1,44 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.imageworks.spcue.service; - -import com.imageworks.spcue.VirtualProc; - -/** - * Publishes per-release accounting deltas to Redis for scheduler-managed shows. Implementations may - * be no-op (when {@code accounting.redis.enabled=false}) or Lettuce-backed. - * - * See the Redis-Backed Accounting Reference at - * {@code docs/_docs/developer-guide/redis-accounting.md} for the protocol. The decrement is applied - * atomically across {@code acct:sub:*}, {@code acct:folder:*}, {@code acct:job:*}, - * {@code acct:layer:*}, {@code acct:point:*} and bumps {@code acct:seq}. - */ -public interface AccountingRedisPublisher { - - /** - * Publish a release delta. Must be invoked only after the surrounding Postgres transaction has - * committed (typically from {@code TransactionSynchronization.afterCommit}). - * - * @param proc the released VirtualProc; provides showId, jobId, layerId, allocationId, - * folderId, deptId, coresReserved, gpusReserved. Callers must ensure folderId and deptId - * are populated (production hydration paths do this via {@code - * ProcDaoJdbc.VIRTUAL_PROC_MAPPER}). - */ - void publishRelease(VirtualProc proc); - - /** True when this publisher actually writes to Redis. */ - boolean isEnabled(); -} diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/LettuceAccountingRedisPublisher.java b/cuebot/src/main/java/com/imageworks/spcue/service/LettuceAccountingRedisPublisher.java deleted file mode 100644 index 0796fcbd19..0000000000 --- a/cuebot/src/main/java/com/imageworks/spcue/service/LettuceAccountingRedisPublisher.java +++ /dev/null @@ -1,184 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.imageworks.spcue.service; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; - -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisNoScriptException; -import io.lettuce.core.RedisURI; -import io.lettuce.core.ScriptOutputType; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; - -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.dao.ShowDao; - -/** - * Lettuce-backed implementation of {@link AccountingRedisPublisher}. Gated by the - * {@code accounting.redis.enabled} property; when false this class loads but {@link #initialize()} - * skips connection setup and {@link #publishRelease} short-circuits. - * - *

- * Atomicity: each release applies five HINCRBY decrements and an INCR on {@code acct:seq} in a - * single Lua script. The sequence guard is required by the reseed protocol (see the "acct:seq - * sequence-number guard" section of the Redis-Backed Accounting Reference at - * {@code docs/_docs/developer-guide/redis-accounting.md}); without it, a reseed running - * concurrently with a release would silently clobber the decrement. - * - *

- * Failure mode: a publish failure leaves Redis missing a decrement, which the Rust scheduler's - * few-minutes recompute heals from {@code SUM(proc)} (see "Failure modes and drift bounds" in the - * same reference). We therefore swallow exceptions with a WARN log rather than letting them - * propagate back into the (already-committed) caller. - */ -public class LettuceAccountingRedisPublisher implements AccountingRedisPublisher { - - private static final Logger logger = - LogManager.getLogger(LettuceAccountingRedisPublisher.class); - - /** - * Redis key layout. - * - * 

-     * KEYS[1] = acct:sub:{show_id}:{alloc_id}    ARGV[1] = -cores  ARGV[2]  = -gpus
-     * KEYS[2] = acct:folder:{folder_id}          ARGV[3] = -cores  ARGV[4]  = -gpus
-     * KEYS[3] = acct:job:{job_id}                ARGV[5] = -cores  ARGV[6]  = -gpus
-     * KEYS[4] = acct:layer:{layer_id}            ARGV[7] = -cores  ARGV[8]  = -gpus
-     * KEYS[5] = acct:point:{dept_id}:{show_id}   ARGV[9] = -cores  ARGV[10] = -gpus
-     * KEYS[6] = acct:seq
-     *
- */ - static final String RELEASE_LUA = "redis.call('HINCRBY', KEYS[1], 'int_cores', ARGV[1]); " - + "redis.call('HINCRBY', KEYS[1], 'int_gpus', ARGV[2]); " - + "redis.call('HINCRBY', KEYS[2], 'int_cores', ARGV[3]); " - + "redis.call('HINCRBY', KEYS[2], 'int_gpus', ARGV[4]); " - + "redis.call('HINCRBY', KEYS[3], 'int_cores', ARGV[5]); " - + "redis.call('HINCRBY', KEYS[3], 'int_gpus', ARGV[6]); " - + "redis.call('HINCRBY', KEYS[4], 'int_cores', ARGV[7]); " - + "redis.call('HINCRBY', KEYS[4], 'int_gpus', ARGV[8]); " - + "redis.call('HINCRBY', KEYS[5], 'int_cores', ARGV[9]); " - + "redis.call('HINCRBY', KEYS[5], 'int_gpus', ARGV[10]); " - + "return redis.call('INCR', KEYS[6]);"; - - @Autowired - private Environment env; - - @Autowired - private ShowDao showDao; - - private RedisClient client; - private StatefulRedisConnection connection; - private RedisCommands commands; - private volatile String scriptSha; - private volatile boolean enabled = false; - - @PostConstruct - public void initialize() { - enabled = env.getProperty("accounting.redis.enabled", Boolean.class, false); - - // Deployment invariant (§4.1): scheduler-managed shows exist but publishing is off -> - // counters drift up monotonically. Loud warning routed through Sentry via log4j2. - int managedCount = showDao.countSchedulerManagedShows(); - if (managedCount > 0 && !enabled) { - logger.warn("Scheduler-managed shows exist ({} found) but " - + "accounting.redis.enabled=false. Booking releases will not be published to " - + "Redis; the standalone scheduler's accounting will silently over-count. Set " - + "accounting.redis.enabled=true on every Cuebot, OR run " - + "`cueadmin -scheduler-managed off` on each managed show before the " - + "scheduler starts.", managedCount); - } - - if (!enabled) { - logger.info("Redis accounting publishing disabled (accounting.redis.enabled=false)"); - return; - } - - String host = env.getRequiredProperty("accounting.redis.host", String.class); - int port = env.getRequiredProperty("accounting.redis.port", Integer.class); - client = RedisClient.create(RedisURI.create(host, port)); - connection = client.connect(); - commands = connection.sync(); - scriptSha = commands.scriptLoad(RELEASE_LUA); - logger.info("Redis accounting publishing enabled, connected to {}:{} (script SHA {})", host, - port, scriptSha); - } - - @PreDestroy - public void shutdown() { - if (connection != null) { - connection.close(); - } - if (client != null) { - client.shutdown(); - } - } - - @Override - public void publishRelease(VirtualProc proc) { - if (!enabled) { - return; - } - try { - evalRelease(proc); - } catch (RedisNoScriptException ns) { - // Lua script flushed (e.g., Redis restart with SCRIPT FLUSH). Reload and retry once. - try { - scriptSha = commands.scriptLoad(RELEASE_LUA); - evalRelease(proc); - } catch (Exception retry) { - logger.warn( - "Redis publish retry failed for proc {} (show {}): {}; recompute " - + "will heal", - proc.getProcId(), proc.getShowId(), retry.getMessage()); - } - } catch (Exception e) { - // §4.3 row 1: publish failure -> Redis missing a decrement -> next recompute - // (≤2 min) heals from SUM(proc). Log + swallow. - logger.warn("Redis publish failed for proc {} (show {}): {}; recompute will heal", - proc.getProcId(), proc.getShowId(), e.getMessage()); - } - } - - /** - * Cuebot stores cores as centicores (cores × 100; see VirtualProc.java:143). Redis stores cores - * (design §0 unit invariant), so we divide on the way out. The divide is exact: VirtualProc - * forces coresReserved to a multiple of 100 at booking time. - */ - static final int CENTICORES_PER_CORE = 100; - - private void evalRelease(VirtualProc proc) { - String[] keys = new String[] {"acct:sub:" + proc.getShowId() + ":" + proc.getAllocationId(), - "acct:folder:" + proc.folderId, "acct:job:" + proc.getJobId(), - "acct:layer:" + proc.getLayerId(), - "acct:point:" + proc.deptId + ":" + proc.getShowId(), "acct:seq"}; - String negCores = String.valueOf(-proc.coresReserved / CENTICORES_PER_CORE); - String negGpus = String.valueOf(-proc.gpusReserved); - String[] argv = new String[] {negCores, negGpus, negCores, negGpus, negCores, negGpus, - negCores, negGpus, negCores, negGpus}; - commands.evalsha(scriptSha, ScriptOutputType.INTEGER, keys, argv); - } - - @Override - public boolean isEnabled() { - return enabled; - } -} diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql index 284ce1cfdd..7efeff0925 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql @@ -12,7 +12,7 @@ -- recalculate_subs() is rewritten here (CREATE OR REPLACE) so that the -- 2-hour maintenance task skips scheduler-managed shows, per the -- "recalculate_subs() show-awareness" section of --- docs/_docs/developer-guide/redis-accounting.md. The body is otherwise identical to +-- docs/_docs/developer-guide/scheduler-accounting.md. The body is otherwise identical to -- V20__recalculate_subs_gpu.sql with two narrow changes: -- 1) the initial UPDATE-to-zero of subscription is restricted to shows -- with b_scheduler_managed = false, so Rust-owned rows are never diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-accounting.xml b/cuebot/src/main/resources/conf/spring/applicationContext-accounting.xml index 23e5eb2494..cd4de948a7 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-accounting.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-accounting.xml @@ -21,15 +21,19 @@ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd"> - - + + - - - + + + + + - + + + diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index bc3caf10c9..4b9a596e98 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -276,13 +276,11 @@ monitoring.elasticsearch.port=${MONITORING_ELASTICSEARCH_PORT:9200} monitoring.elasticsearch.scheme=${MONITORING_ELASTICSEARCH_SCHEME:http} # ================================================================================== -# Redis-backed accounting (see docs/_docs/developer-guide/redis-accounting.md) +# Scheduler accounting NOTIFY (see docs/_docs/developer-guide/scheduler-accounting.md) # ================================================================================== -# Enable Redis accounting publishing. When false, Cuebot operates as today. -# When true, Cuebot publishes per-show release deltas to Redis for shows -# whose b_scheduler_managed=true is owned by the standalone Rust scheduler. -accounting.redis.enabled=${ACCOUNTING_REDIS_ENABLED:false} - -# Redis host and port (single-node, no persistence -- see decisions doc Q6e/Q8f). -accounting.redis.host=${ACCOUNTING_REDIS_HOST:localhost} -accounting.redis.port=${ACCOUNTING_REDIS_PORT:6379} +# Emit per-release and enforced cap-change deltas to the standalone Rust scheduler +# via Postgres pg_notify, inside the same transaction as each DB write. Only affects +# shows whose b_scheduler_managed=true is owned by the scheduler. When false, the +# scheduler degrades to periodic recompute from SUM(proc) (under-books, never +# over-books), so disabling is safe. +accounting.notify.enabled=${ACCOUNTING_NOTIFY_ENABLED:true} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java index 4cad14ca2a..9e9ef6db09 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java @@ -904,8 +904,8 @@ public void testProcDestroyedCuebotManagedShowDecrementsAccountingTables() { /** * Scheduler-managed show: deleteVirtualProc must not decrement the five PG accounting * tables. The Rust scheduler's recompute will rewrite them from SUM(proc) on a 2-min cadence - * (PR-C). For PR-B this only asserts the SQL chokepoint behavior; the Redis publish itself is - * covered by {@code LettuceAccountingRedisPublisherTests}. + * (PR-C). For PR-B this only asserts the SQL chokepoint behavior; the release NOTIFY itself is + * covered by {@code AccountingNotifierTests}. */ @Test @Transactional diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/AccountingNotifierTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/AccountingNotifierTests.java new file mode 100644 index 0000000000..eee5fd5216 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/AccountingNotifierTests.java @@ -0,0 +1,160 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.test.service; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.util.ReflectionTestUtils; + +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.service.AccountingNotifier; + +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +/** + * Unit tests for {@link AccountingNotifier}. Mocks the {@link JdbcTemplate} (injected into the + * {@code JdbcDaoSupport} base field) and asserts the exact {@code SELECT pg_notify} channel and + * JSON payload for a release and for each enforced cap change. This pins the wire contract the Rust + * scheduler's NOTIFY listener parses (see docs/_docs/developer-guide/scheduler-accounting.md). + */ +public class AccountingNotifierTests { + + private static final String NOTIFY_SQL = "SELECT pg_notify(?, ?)"; + private static final String CHANNEL_RELEASE = "acct_release"; + private static final String CHANNEL_LIMIT = "acct_limit_change"; + + private static final String SHOW_ID = "show-uuid"; + private static final String JOB_ID = "job-uuid"; + private static final String LAYER_ID = "layer-uuid"; + private static final String ALLOC_ID = "alloc-uuid"; + private static final String FOLDER_ID = "folder-uuid"; + private static final String DEPT_ID = "dept-uuid"; + + private AccountingNotifier notifier; + private JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + private VirtualProc proc; + + @Before + public void setUp() { + notifier = new AccountingNotifier(); + // Bypass @PostConstruct (no Environment); enable and inject the mock JdbcTemplate held by + // the JdbcDaoSupport base class. + ReflectionTestUtils.setField(notifier, "notifyEnabled", true); + ReflectionTestUtils.setField(notifier, "jdbcTemplate", jdbcTemplate); + + proc = new VirtualProc(); + proc.id = "proc-uuid"; + proc.showId = SHOW_ID; + proc.jobId = JOB_ID; + proc.layerId = LAYER_ID; + proc.allocationId = ALLOC_ID; + proc.folderId = FOLDER_ID; + proc.deptId = DEPT_ID; + // 400 centicores = 4 cores; converted to cores on the way out and negated for a release. + // 4 != 2 (gpus) keeps the two negated values disambiguated in the payload assertion. + proc.coresReserved = 400; + proc.gpusReserved = 2; + } + + @Test + public void disabledNotifierIsNoop() { + ReflectionTestUtils.setField(notifier, "notifyEnabled", false); + + notifier.notifyRelease(proc); + notifier.notifySubscriptionBurst(SHOW_ID, ALLOC_ID, 1500); + notifier.notifyFolderMaxCores(FOLDER_ID, 800); + notifier.notifyFolderMaxGpus(FOLDER_ID, 3); + notifier.notifyJobMaxCores(JOB_ID, 1600); + notifier.notifyJobMaxGpus(JOB_ID, 4); + + verifyNoInteractions(jdbcTemplate); + } + + @Test + public void notifyReleaseEmitsNegatedDelta() { + notifier.notifyRelease(proc); + + // Cores converted to cores (400/100=4) then negated; gpus pass through then negated. + String expected = "{\"show\":\"show-uuid\",\"alloc\":\"alloc-uuid\"," + + "\"folder\":\"folder-uuid\",\"job\":\"job-uuid\",\"layer\":\"layer-uuid\"," + + "\"dept\":\"dept-uuid\",\"cores\":-4,\"gpus\":-2}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_RELEASE), eq(expected)); + } + + @Test + public void notifySubscriptionBurstConvertsCentiCores() { + notifier.notifySubscriptionBurst(SHOW_ID, ALLOC_ID, 1500); + + String expected = "{\"vertex\":\"sub\",\"show\":\"show-uuid\",\"alloc\":\"alloc-uuid\"," + + "\"burst\":15}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_LIMIT), eq(expected)); + } + + @Test + public void notifyFolderMaxCoresConvertsCentiCores() { + notifier.notifyFolderMaxCores(FOLDER_ID, 800); + + String expected = "{\"vertex\":\"folder\",\"id\":\"folder-uuid\",\"max_cores\":8}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_LIMIT), eq(expected)); + } + + @Test + public void notifyFolderMaxCoresPreservesUnlimitedSentinel() { + notifier.notifyFolderMaxCores(FOLDER_ID, -1); + + String expected = "{\"vertex\":\"folder\",\"id\":\"folder-uuid\",\"max_cores\":-1}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_LIMIT), eq(expected)); + } + + @Test + public void notifyFolderMaxGpusPassesThrough() { + notifier.notifyFolderMaxGpus(FOLDER_ID, 3); + + String expected = "{\"vertex\":\"folder\",\"id\":\"folder-uuid\",\"max_gpus\":3}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_LIMIT), eq(expected)); + } + + @Test + public void notifyJobMaxCoresConvertsCentiCores() { + notifier.notifyJobMaxCores(JOB_ID, 1600); + + String expected = "{\"vertex\":\"job\",\"id\":\"job-uuid\",\"max_cores\":16}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_LIMIT), eq(expected)); + } + + @Test + public void notifyJobMaxCoresPreservesUnlimitedSentinel() { + notifier.notifyJobMaxCores(JOB_ID, -100); + + String expected = "{\"vertex\":\"job\",\"id\":\"job-uuid\",\"max_cores\":-1}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_LIMIT), eq(expected)); + } + + @Test + public void notifyJobMaxGpusPassesThrough() { + notifier.notifyJobMaxGpus(JOB_ID, 4); + + String expected = "{\"vertex\":\"job\",\"id\":\"job-uuid\",\"max_gpus\":4}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_LIMIT), eq(expected)); + verify(jdbcTemplate, never()).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_RELEASE), + org.mockito.ArgumentMatchers.anyString()); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/LettuceAccountingRedisPublisherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/LettuceAccountingRedisPublisherTests.java deleted file mode 100644 index aad6a0a595..0000000000 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/LettuceAccountingRedisPublisherTests.java +++ /dev/null @@ -1,207 +0,0 @@ - -/* - * Copyright Contributors to the OpenCue Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.imageworks.spcue.test.service; - -import java.util.Arrays; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import io.lettuce.core.RedisNoScriptException; -import io.lettuce.core.api.sync.RedisCommands; -import org.junit.Before; -import org.junit.Test; -import org.springframework.test.util.ReflectionTestUtils; - -import com.imageworks.spcue.VirtualProc; -import com.imageworks.spcue.service.LettuceAccountingRedisPublisher; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Unit tests for {@link LettuceAccountingRedisPublisher}. Uses a Mockito-backed - * {@code RedisCommands} stub with a custom Answer so we don't have to fight Mockito's vararg - * matching for the {@code evalsha(String, ScriptOutputType, K[], V...)} signature. - */ -public class LettuceAccountingRedisPublisherTests { - - private static final String SHA = "deadbeef"; - private static final String NEW_SHA = "cafebabe"; - - private LettuceAccountingRedisPublisher publisher; - @SuppressWarnings("unchecked") - private RedisCommands commands = mock(RedisCommands.class); - - private VirtualProc proc; - private static final String SHOW_ID = "show-uuid"; - private static final String JOB_ID = "job-uuid"; - private static final String LAYER_ID = "layer-uuid"; - private static final String ALLOC_ID = "alloc-uuid"; - private static final String FOLDER_ID = "folder-uuid"; - private static final String DEPT_ID = "dept-uuid"; - - /** Captured evalsha invocations across test scenarios. */ - private final AtomicInteger evalshaCount = new AtomicInteger(); - private final AtomicReference lastSha = new AtomicReference<>(); - private final AtomicReference lastKeys = new AtomicReference<>(); - private final AtomicReference lastArgv = new AtomicReference<>(); - - @Before - public void setUp() { - publisher = new LettuceAccountingRedisPublisher(); - // Bypass @PostConstruct so we don't dial Redis; manually inject test state. - ReflectionTestUtils.setField(publisher, "enabled", true); - ReflectionTestUtils.setField(publisher, "commands", commands); - ReflectionTestUtils.setField(publisher, "scriptSha", SHA); - - proc = new VirtualProc(); - proc.id = "proc-uuid"; - proc.showId = SHOW_ID; - proc.jobId = JOB_ID; - proc.layerId = LAYER_ID; - proc.allocationId = ALLOC_ID; - proc.folderId = FOLDER_ID; - proc.deptId = DEPT_ID; - // 400 centicores = 4 cores. Cuebot stores cores × 100; the publisher converts to - // cores on the way to Redis per design §0 unit invariant. 4 != 2 (gpusReserved) - // keeps the -cores/-gpus argv pair disambiguated in assertions below. - proc.coresReserved = 400; - proc.gpusReserved = 2; - - // Mockito's vararg matching is fiddly for evalsha(String, ScriptOutputType, K[], V...). - // Side-step it with an Answer that records the args; tests can then inspect or - // overwrite the behavior via setEvalshaBehavior(...). - evalshaCount.set(0); - when(commands.evalsha(org.mockito.ArgumentMatchers.anyString(), - org.mockito.ArgumentMatchers.any(io.lettuce.core.ScriptOutputType.class), - org.mockito.ArgumentMatchers.any(String[].class), - org.mockito.ArgumentMatchers.any())).thenAnswer(invocation -> { - evalshaCount.incrementAndGet(); - lastSha.set(invocation.getArgument(0)); - lastKeys.set(invocation.getArgument(2)); - Object[] all = invocation.getArguments(); - String[] argv = new String[all.length - 3]; - for (int i = 3; i < all.length; i++) { - argv[i - 3] = (String) all[i]; - } - lastArgv.set(argv); - return 0L; - }); - } - - /** Replace the evalsha stub with an Answer that always throws the given exception. */ - private void evalshaAlwaysThrows(RuntimeException ex) { - when(commands.evalsha(org.mockito.ArgumentMatchers.anyString(), - org.mockito.ArgumentMatchers.any(io.lettuce.core.ScriptOutputType.class), - org.mockito.ArgumentMatchers.any(String[].class), - org.mockito.ArgumentMatchers.any())).thenThrow(ex); - } - - /** Evalsha throws only when the SHA matches the given value; otherwise records as normal. */ - private void evalshaThrowsOnSha(String shaToFail, RuntimeException ex) { - when(commands.evalsha(org.mockito.ArgumentMatchers.anyString(), - org.mockito.ArgumentMatchers.any(io.lettuce.core.ScriptOutputType.class), - org.mockito.ArgumentMatchers.any(String[].class), - org.mockito.ArgumentMatchers.any())).thenAnswer(invocation -> { - String sha = invocation.getArgument(0); - if (sha.equals(shaToFail)) { - throw ex; - } - evalshaCount.incrementAndGet(); - lastSha.set(sha); - lastKeys.set(invocation.getArgument(2)); - Object[] all = invocation.getArguments(); - String[] argv = new String[all.length - 3]; - for (int i = 3; i < all.length; i++) { - argv[i - 3] = (String) all[i]; - } - lastArgv.set(argv); - return 0L; - }); - } - - @Test - public void disabledPublisherIsNoop() { - ReflectionTestUtils.setField(publisher, "enabled", false); - assertFalse(publisher.isEnabled()); - - publisher.publishRelease(proc); - - assertEquals(0, evalshaCount.get()); - } - - @Test - public void publishReleaseSendsExpectedKeysAndArgs() { - assertTrue(publisher.isEnabled()); - - publisher.publishRelease(proc); - - assertEquals(1, evalshaCount.get()); - assertEquals(SHA, lastSha.get()); - - // Validates the Redis schema documented in the Redis-Backed Accounting Reference - // (docs/_docs/developer-guide/redis-accounting.md) and pins the alloc_id-not-name - // decision (the subscription key uses the allocation UUID). - String[] expectedKeys = new String[] {"acct:sub:" + SHOW_ID + ":" + ALLOC_ID, - "acct:folder:" + FOLDER_ID, "acct:job:" + JOB_ID, "acct:layer:" + LAYER_ID, - "acct:point:" + DEPT_ID + ":" + SHOW_ID, "acct:seq"}; - assertArrayEquals(expectedKeys, lastKeys.get()); - - // Five pairs of (-cores, -gpus), one per accounting table. Cores are sent in - // Redis cores (centicores / 100), so 400 -> "-4". GPUs pass through unconverted. - String[] expectedArgv = - new String[] {"-4", "-2", "-4", "-2", "-4", "-2", "-4", "-2", "-4", "-2"}; - assertArrayEquals(expectedArgv, lastArgv.get()); - } - - @Test - public void noScriptExceptionTriggersReloadAndRetry() { - // First evalsha (with stale SHA) throws; reload returns new SHA; retry succeeds. - evalshaThrowsOnSha(SHA, new RedisNoScriptException("NOSCRIPT")); - when(commands.scriptLoad(org.mockito.ArgumentMatchers.anyString())).thenReturn(NEW_SHA); - - publisher.publishRelease(proc); - - // The retry (with NEW_SHA) was recorded by the Answer. - assertEquals(1, evalshaCount.get()); - assertEquals(NEW_SHA, lastSha.get()); - // scriptSha field updated for subsequent calls. - assertEquals(NEW_SHA, ReflectionTestUtils.getField(publisher, "scriptSha")); - } - - @Test - public void genericExceptionIsSwallowed() { - evalshaAlwaysThrows(new RuntimeException("redis unreachable")); - - // Should not propagate; recompute heals the missing decrement per §4.3 row 1. - // Reaching the next line is the assertion. - publisher.publishRelease(proc); - } - - /** Sanity check that our test fixture's Lua script-content captor really sees varargs. */ - @Test - public void varargCaptureSmokeTest() { - publisher.publishRelease(proc); - assertEquals(10, lastArgv.get().length); - // First pair is from acct:sub key — uses -cores then -gpus. - assertEquals(Arrays.asList("-4", "-2"), - Arrays.asList(lastArgv.get()[0], lastArgv.get()[1])); - } -} diff --git a/docs/_docs/developer-guide/redis-accounting.md b/docs/_docs/developer-guide/redis-accounting.md deleted file mode 100644 index 00c6f8dc3b..0000000000 --- a/docs/_docs/developer-guide/redis-accounting.md +++ /dev/null @@ -1,603 +0,0 @@ ---- -title: "Redis-Backed Accounting Reference" -nav_order: 101 -parent: Reference -layout: default -linkTitle: "Redis-Backed Accounting" -date: 2026-05-22 -description: > - Technical reference for the Redis-backed accounting subsystem shared by Cuebot and the Rust scheduler ---- - -# Redis-Backed Accounting Reference - -## How Cuebot and the Rust scheduler coordinate per-show resource accounting through Redis - ---- - -## Overview - -The accounting subsystem tracks how much of each resource pool (subscription, -folder, job, layer, department point) is currently booked. Every dispatch -decision the [Rust scheduler](/docs/developer-guide/scheduler/) makes is gated -on these counters - if a job is already at its `int_max_cores`, the scheduler -must not book another frame against it. - -Historically these counters lived only in PostgreSQL: Cuebot's dispatcher -updated five accounting tables transactionally on every booking and every -release. As the Rust scheduler took over dispatch, two problems emerged: - -1. **Postgres lock contention.** The scheduler's hot path was hammering the - same accounting rows Cuebot's `HostReportHandler` writes to. Lock waits on - `subscription`, `folder_resource`, and `job_resource` started limiting - throughput. -2. **Horizontal scaling.** An earlier in-process accounting cache (a - `HashMap` inside the scheduler) cannot be shared across N scheduler - instances. Any path to multi-scheduler deployment needs a shared store. - -This subsystem replaces both with a Redis-backed accounting layer that both -Cuebot and the Rust scheduler write through on the hot path. PostgreSQL -remains the durable system of record; Redis is the live operational view. - -Source: `rust/crates/scheduler/src/accounting/` and -`cuebot/src/main/java/com/imageworks/spcue/service/AccountingRedisPublisher.java`. - ---- - -## Source-of-truth model - -This is the load-bearing design choice; everything else falls out of it. - -| Component | Hot path | Slow path | -|---|---|---| -| Rust scheduler (booking) | Atomic Lua against Redis (`check + 5×HINCRBY + INCR acct:seq`), then `INSERT proc` in PG transactionally | Periodic recompute (every 2 min) writes PG accounting tables for scheduler-managed shows from `SUM(proc)` | -| Cuebot release path (`ProcDaoJdbc.unbookProc`) | For scheduler-managed shows: only `DELETE proc` transactionally, then `afterCommit` publishes the release delta to Redis. For Cuebot-managed shows: unchanged transactional UPDATEs against accounting tables | - | -| Cuebot admin paths (size/burst/min/max changes) | Unchanged transactional UPDATEs against accounting tables; **no Redis publish** | - | -| CueGUI | Reads PG accounting tables (unchanged) | - | - -Three properties hold: - -- **`proc` is canonical for bookings.** Both sides write `proc` transactionally. - Anything else can be reconstructed from `SELECT SUM(int_cores_reserved) FROM - proc GROUP BY pk_show, pk_alloc`. -- **PG accounting tables are derived.** For scheduler-managed shows they are - refreshed by the recompute loop. For Cuebot-managed shows they are written - transactionally by Cuebot's dispatcher (unchanged from before). -- **Redis is the live operational view.** Both sides feed it on the hot path; - it is rebuilt from `proc` and from the PG accounting tables on a schedule. - -### Why a hybrid, not Redis-only or PG-only - -Three alternatives were considered and rejected: - -- **Redis as a derived index of Postgres** (Cuebot writes PG transactionally; - `afterCommit` copies the delta to Redis; the scheduler reads Redis on the hot - path but writes PG transactionally). This is conservative and safe under a - Redis outage, but it leaves the scheduler's hot path holding PG locks - which - is most of the lock-contention pressure we were trying to relieve. -- **Both sides write Redis only; an async drainer persists to Postgres.** - Maximally decoupled, but Redis loss becomes data loss unless AOF and - replication are bulletproof, and the drainer becomes a new SPOF. Too far - ahead of where we trust Redis in this stack today. -- **Cuebot writes both Postgres and Redis inline in the same code path with no - transaction coordination.** Partial failures silently diverge with no recovery - story. Anti-pattern, rejected outright. - -The chosen design sits between the first two: Redis is the live operational -view both sides write through; Postgres is durably correct via Cuebot's -transactional writes (for Cuebot-managed shows) or the scheduler's periodic -recompute (for scheduler-managed shows). - ---- - -## Show ownership: the per-show partition - -Within a single show, exactly one of Cuebot or Rust owns the accounting write -path. There is no double-write and no per-key arbitration. The flag lives on -the `show` table: - -```sql -ALTER TABLE show ADD COLUMN b_scheduler_managed BOOLEAN NOT NULL DEFAULT false; -``` - -- `b_scheduler_managed = false` (default): the show is Cuebot-managed. - Cuebot's dispatcher books and releases against PG accounting tables - transactionally, exactly as before. Redis is not consulted. -- `b_scheduler_managed = true`: the show is scheduler-managed. The Rust - scheduler books against Redis on the hot path; Cuebot's release path only - deletes the `proc` row and publishes the release delta to Redis via - `afterCommit`. PG accounting tables for this show are refreshed by the - scheduler's 2-min recompute loop. - -The flag is **per-show, not per-allocation.** A show is either -scheduler-managed or it isn't; mixed-mode shows would force per-row arbitration -in both Cuebot and the scheduler, and the simplification was worth more than -the flexibility. - -The flag also replaces the older `dispatcher.exclusion_list` and -`dispatcher.scheduler_manages_resources` properties in `opencue.properties`, -both removed. Migration: any show previously named in `exclusion_list` must be -flipped via cueadmin (see [Operator workflow](#operator-workflow) below). - -### Looking up the flag - -Cuebot's `ProcDaoJdbc.unbookProc` checks the flag on every release. Hitting PG -on every release would defeat the purpose, so `ShowDao` caches the flag with a -~30 s TTL. After a `setSchedulerManaged` toggle, the new value is visible -across the cluster within ~30 s. The brief stale window is acceptable - -transient drift heals via the next recompute (see [Failure modes](#failure-modes-and-drift-bounds)). - ---- - -## Redis schema - -Six key namespaces, one per accounting table plus a global sequence counter: - -| Key | Type | Fields | -|---|---|---| -| `acct:sub:{show_id}:{alloc_id}` | HASH | `size, burst, int_cores, int_gpus` | -| `acct:folder:{folder_id}` | HASH | `int_min_cores, int_max_cores, int_min_gpus, int_max_gpus, int_cores, int_gpus, show_id` | -| `acct:job:{job_id}` | HASH | `int_max_cores, int_max_gpus, int_priority, int_cores, int_gpus` | -| `acct:layer:{layer_id}` | HASH | `int_cores, int_gpus` (plus any per-layer caps the scheduler reads) | -| `acct:point:{dept_id}:{show_id}` | HASH | `int_min_cores, int_max_cores, int_cores, int_gpus` | -| `acct:seq` | STRING (INCR) | global mutation sequence number | - -`job_resource` and `layer_resource` both carry caps the scheduler enforces on -the hot path - neither can be omitted from Redis without breaking per-job or -per-layer cap enforcement. - -### The unit invariant: cores, not centicores - -PostgreSQL stores cores as **centicores** (cores × 100; the `int_*cores*` -columns and `proc.int_cores_reserved`). Redis stores cores in **unmultiplied -units** (1 = 1 core). Conversion happens at every PG↔Redis and Cuebot↔Redis -boundary - the limit reseed, the booked-counter recompute, the Cuebot release -publisher, and the Rust booking delta. Inside Redis - and inside the Lua -scripts - no centicore arithmetic ever happens. - -Two reasons: - -1. Redis is the live operational view. Operators reading `redis-cli HGETALL - acct:sub:...` should see numbers that match what they typed into cueadmin - (e.g. `cueadmin -create-subscription -size 100` should show `size = 100`, - not `10000`). -2. The Rust scheduler's hot-path arithmetic is already in cores - (`CoreSize`). Pushing the conversion to the PG and Cuebot edges keeps the - hot path free of unit-juggling. - -GPU fields and `int_priority` pass through verbatim - no unit conversion. - -The `-1` "unlimited" sentinel on `folder_resource.int_max_cores` and -`job_resource.int_max_cores` is preserved verbatim across the conversion. The -hot-path Lua guard (`> 0`) gates the comparison either way, but passing the -sentinel through unchanged keeps `redis-cli` output faithful to the PG meaning. - ---- - -## The `acct:seq` sequence-number guard - -`acct:seq` is a monotonic counter in Redis that protects every reseed from a -silent-loss race against concurrent hot-path writes. It is not optional -machinery added later - it is the reseed contract. - -### The race it prevents - -A reseed has two operations that cannot be made atomic from the outside: - -1. **SQL read**: e.g. `SELECT SUM(int_cores_reserved) FROM proc GROUP BY - pk_show, pk_alloc`. -2. **Redis write**: write each computed total back to the corresponding - `acct:*` hash. - -Between (1) and (2), live hot-path mutations are still happening on Redis. -Without a guard, the reseed clobbers them: - -| t | Event | Redis `acct:sub:S:A.int_cores` | proc rows for (S,A) | -|---|---|---|---| -| t0 | start | 50 | 5 rows × 10 cores | -| t1 | Reseed reads PG → `SUM = 50` | 50 | 5 rows | -| t2 | Rust books a frame: Lua `HINCRBY +10` → `INSERT proc` | 60 | 6 rows | -| t3 | Reseed writes Redis from its in-memory snapshot: `HSET ... 50` | **50** ← booking lost | 6 rows | - -At t3, the booking from t2 is silently lost in Redis. `proc` is correct, but -Redis under-counts → the next dispatch over-books. This does **not** self-heal -- every reseed cycle reopens the same window. - -### The protocol - -Every mutating Lua script (booking, force-rollback, Cuebot release publisher) -increments `acct:seq` as part of the same script. Reseed becomes a -compare-and-swap on the entire state: - -1. `GET acct:seq` → store as `seq_before`. -2. `SELECT SUM(...) FROM proc` (or read accounting tables, for the limit - reseed). -3. Compute the new Redis values in memory. -4. Atomic CAS via Lua: *if `GET acct:seq == seq_before` then write the new - values, else return RETRY*. -5. On RETRY: loop back to (1). After a bounded number of retries under - sustained load, skip this reseed cycle. Hot-path writes are keeping Redis - fresh; a reseed that can't make progress is the wrong tool. - -The same trace with the guard: - -| t | Event | `acct:seq` | Redis `int_cores` | -|---|---|---|---| -| t0 | start | 100 | 50 | -| t1 | Reseed reads `seq_before=100`, SELECT SUM=50 | 100 | 50 | -| t2 | Booking Lua: HINCRBY +10, INCR seq | **101** | 60 | -| t3 | Reseed CAS: seq is 101 ≠ 100 → RETRY | 101 | 60 | -| t4 | Reseed re-reads `seq_before=101`, SELECT SUM=60 | 101 | 60 | -| t5 | No mutations during window | 101 | 60 | -| t6 | Reseed CAS succeeds: HSET ... 60 | 101 | 60 | - -No write is clobbered. - -The mechanism is the same as the `AtomicU64` sequence guard from the earlier -in-process design - with the counter moved into Redis so it's visible across -processes. Once N>1 schedulers run, this property generalises directly. - ---- - -## Hot path: atomic booking Lua - -The scheduler's per-frame booking is a single Lua script that runs against -Redis, executing five updates atomically: - -```text -1. Read current state of acct:sub / acct:folder / acct:job / acct:layer / acct:point -2. Check booking would not exceed any limit (size, burst, max_cores, etc.) -3. If OK: 5 × HINCRBY (int_cores, int_gpus) + INCR acct:seq, return {1} -4. If over a limit: return structured failure {0, table_name, current, limit} -5. Then transactionally INSERT proc in Postgres (outside Lua) -``` - -The Lua script returns a structured shape on failure (`{0, table_name, -current, limit}`) so observability and metrics can attribute the rejection to -the right table without re-reading state. - -### Rollback (`force` mode) - -The same script supports a `force` flag that skips the limit checks and applies -the delta unconditionally. This is the rollback path: if the PG `INSERT proc` -fails after the Lua succeeded, the scheduler calls the script again with -`force=true` and negated deltas to undo the Redis-side change. - -One script, two modes - no separate rollback script to keep in sync. - -### No idempotency tokens - -The booking script has no dedup mechanism. A network blip that causes the -caller to retry a successful booking will double-count in Redis. This is -accepted because: - -- The recompute loop (≤ 2 min) heals double-counts from `proc`. -- Adding idempotency tokens would require a write-once log in Redis with its - own eviction story. -- In practice the duplicate-booking rate from caller retries is far below the - threshold where it would affect dispatch correctness. - -If observed rates change this calculus, idempotency tokens are listed under -[known limitations](#known-limitations-and-future-work). - ---- - -## Reseed loops - -Three loops keep Redis convergent with PG. They are explicitly designed to be -the recovery mechanism - hot-path writes drive correctness in the common case, -reseeds drive correctness after failure. - -### Booked-counter recompute (every 2 min) - -For every scheduler-managed show, the scheduler runs: - -```sql -SELECT pk_show, pk_alloc, SUM(int_cores_reserved), SUM(int_gpus_reserved) -FROM proc -WHERE pk_show IN () -GROUP BY pk_show, pk_alloc; -``` - -The result is dual-written to: - -- **PG accounting tables** - so CueGUI's view stays fresh for scheduler-managed - shows. CueGUI reads PG unchanged; numbers may lag the actual booking state by - up to one recompute interval. -- **Redis `int_cores` / `int_gpus`** - guarded by the `acct:seq` CAS described - above. - -The dual write happens under the same SUM query for consistency: both PG and -Redis end up showing the same snapshot. - -#### Zero-convergence for drained keys - -`SUM(proc)` only returns keys that **still have procs**. A key whose booked -counter drifted stale-high (e.g. a lost decrement) and whose procs then drained -to zero would vanish from the snapshot entirely - so a snapshot-only reseed -could never reset it, and the stale value would wedge the key forever (for a -subscription/folder/job this means falsely failing the burst/cap check in the -booking Lua with no path to recovery). - -To close this, the Redis reseed overlays the snapshot on a **zero-baseline** of -every enumerable key. Before folding in the proc sums, it seeds `int_cores=0`/ -`int_gpus=0` for every `acct:sub`/`acct:folder`/`acct:job`/`acct:point` key that -exists for a scheduler-managed show - enumerated by reusing the same limit-table -queries the limit reseed runs (`subscription`, `folder_resource`, -`job_resource` non-`FINISHED`, `point`). A key present in the baseline but -absent from the snapshot has no procs, so it emits a resetting `0`; a key with -procs emits its true sum. This is the same key universe, and same order of -magnitude of ops, the limit reseed already writes each cycle. - -`acct:layer` is intentionally **not** zero-baselined: layers have no limit table -to enumerate from, and the booking Lua never reads the layer counter (it is -`HINCRBY`-only), so residual layer drift is cosmetic. Orphaned `acct:job`/ -`acct:layer` keys for `FINISHED` jobs are likewise left in place (never read -again); reclaiming that memory is tracked as future work. - -### Limit reseed (every 5 min) - -For every scheduler-managed show, the scheduler reads limit fields (`size`, -`burst`, `int_min_cores`, `int_max_cores`, priorities) from the PG accounting -tables and writes them to Redis. This catches changes from Cuebot admin -operations (size/burst/folder cap changes) that don't go through the -afterCommit hook. Five minutes of staleness on cap changes is the documented -drift bound for this path. - -### Bootstrap reseed (blocking at startup) - -When the scheduler starts, it runs both reseeds end-to-end before accepting -work. The booking pipeline does not start until Redis is fully populated. - -Redis is configured without persistence (single node, AOF off), so a Redis -restart shows as an empty store on reconnect. The scheduler detects empty -Redis and re-runs the bootstrap. This is the recovery path: Redis dies → -scheduler stops dispatching → Redis comes back → scheduler reseeds and -resumes. - -### Why recompute, not batched additive deltas - -The recompute model was chosen over a batched-additive model (where the -scheduler accumulates per-show deltas in memory and flushes them to PG -periodically) for two reasons: - -1. **No lost-batch-on-crash failure mode.** A batched flush that fails after - the scheduler crashes loses every booking in that batch from the PG view. - Recompute from `SUM(proc)` cannot lose bookings - `proc` is always correct. -2. **A safety net exists.** Recompute is self-correcting: any drift from any - cause is bounded by the recompute interval. Additive deltas have no such - property; once they diverge, they stay diverged. - ---- - -## Cuebot integration - -### Release publisher (`AccountingRedisPublisher`) - -For scheduler-managed shows, `ProcDaoJdbc.unbookProc` only `DELETE`s the proc -row transactionally; the accounting-table UPDATEs from the legacy code path are -skipped (they would race the recompute loop). On `afterCommit`, the release -delta is published to Redis via the `AccountingRedisPublisher` interface. - -Two implementations: - -- **`LettuceAccountingRedisPublisher`** - wired in when - `accounting.redis.enabled=true`. Runs a single Lua script that applies five - `HINCRBY` decrements and increments `acct:seq` atomically. -- **No-op publisher** - wired in when `accounting.redis.enabled=false`. - Deployments without Redis use this and the unmodified legacy behavior. - -Publish failures (network blip, Redis briefly unavailable) are logged at WARN -and swallowed. The recompute loop heals the missing decrement on the next -cycle. - -### `recalculate_subs()` show-awareness - -Cuebot's 2-hour periodic task that recomputes subscription aggregates already -existed before this subsystem. It is updated to skip rows where -`b_scheduler_managed = true`, so it doesn't fight the scheduler's recompute -loop on those shows. - -### Startup guardrail - -The combination "any show has `b_scheduler_managed=true` but at least one -Cuebot has `accounting.redis.enabled=false`" is a silent over-booking trap: -that Cuebot's releases never reach Redis, so the scheduler sees counts that -only ever grow. - -At startup, Cuebot queries `SELECT COUNT(*) FROM show WHERE b_scheduler_managed -= true`. If the count is > 0 and `accounting.redis.enabled=false`, Cuebot: - -- Logs a loud WARN ("Scheduler-managed shows exist but Redis publishing is - disabled; bookings will silently over-count."). -- Exposes the `cuebot_redis_publish_misconfigured` metric for deploy-time - alerting. - -Cuebot does **not** refuse to start - a misconfigured Cuebot must still serve -gRPC traffic. The signal is loud enough to catch in deploy validation. - -### Operator workflow - -Transitioning a show between modes is a single CLI operation: - -```bash -cueadmin -show -setSchedulerManaged true # move to scheduler -cueadmin -show -setSchedulerManaged false # move back to Cuebot -``` - -Backed by a `ShowInterface.setSchedulerManaged(show_id, bool)` gRPC method and -a pycue wrapper. Cuebot flips `show.b_scheduler_managed` in a single UPDATE; -the `ShowDao` cache picks up the change within ~30 s; from that moment, -`unbookProc` and `recalculate_subs()` branch on the new value. - -No drain or quiesce step is required: - -- In-flight bookings continue executing; their releases flow through whichever - branch is active at release time. -- Transient PG drift after the flip - possibly transiently negative - `int_cores` going one way, phantom-positive going the other - heals via the - next recompute (≤ 2 min). -- Redis state is fed by hot-path writes from both sides regardless of the flag, - so scheduler decisions stay correct across the transition. - -The command is safe to run during production hours. - ---- - -## Failure modes and drift bounds - -| Failure | Effect | Recovery | -|---|---|---| -| Cuebot `afterCommit` Redis publish fails | Redis missing a decrement; over-counts by 1 booking | Next recompute (≤ 2 min) reseeds booked counters from `proc` | -| Rust scheduler dies between Lua and `proc` INSERT | Redis over-counted by 1; no proc row | Next recompute (≤ 2 min) reseeds Redis from `proc` | -| Rust scheduler dies after `proc` INSERT, before reply to caller | Caller may retry; no Redis dedup → Redis over-counted by 1 | Same as above | -| Cuebot admin operation (size/burst change) | Redis stale on limit fields | Next limit reseed (≤ 5 min) heals from accounting tables | -| Redis itself dies | Scheduler stops dispatching scheduler-managed shows | On Redis recovery, scheduler detects empty Redis, reseeds, resumes | -| `b_scheduler_managed` toggle mid-flight | PG accounting transiently wrong (possibly negative) for that show | Next recompute (≤ 2 min) reseeds from `proc` | -| Cuebot Redis publish enabled but scheduler off | Redis filled but unused; no harm | n/a | -| Scheduler on, Cuebot Redis publish off on *any* Cuebot | Redis missing decrements progressively → silent over-booking | **Deployment invariant** (below); guarded at Cuebot startup | - -### Deployment invariant - -If any show has `b_scheduler_managed = true`, every Cuebot in the cluster must -have `accounting.redis.enabled = true`. Cuebot's startup guardrail surfaces -violations as a WARN log and the `cuebot_redis_publish_misconfigured` metric. - -### CueGUI staleness - -CueGUI reads PG accounting tables unchanged. For scheduler-managed shows, -numbers are stale by at most the recompute interval (2 min) plus any in-flight -bookings since the last recompute. For Cuebot-managed shows, accounting is -updated transactionally as before. Acceptable for the existing CueGUI contract. - ---- - -## Configuration - -### Cuebot - -Spring properties (typically in `opencue.properties` or environment overrides): - -```properties -accounting.redis.enabled=true -accounting.redis.host=redis.internal -accounting.redis.port=6379 -``` - -When `accounting.redis.enabled=false`, the no-op publisher bean is wired and -Cuebot behaves exactly as before this subsystem existed. - -### Rust scheduler - -YAML config (per the scheduler's standard config format): - -```yaml -accounting: - redis_url: "redis://redis.internal:6379" - recompute_interval_seconds: 120 - limit_reseed_interval_seconds: 300 - redis_pool_size: 20 -``` - -### Redis topology - -Current deployment is **single-node, no persistence**. Redis restart equals -empty store, which the scheduler detects on reconnect and recovers from via -bootstrap reseed. Redis is therefore a new SPOF for scheduler-managed shows; -during a Redis outage, the scheduler stops dispatching those shows. See -[known limitations](#known-limitations-and-future-work). - ---- - -## Design decisions and trade-offs - -Where the design picked one path and rejected another, the reasoning is here. -Brief versions; the trade-off matters more than the alternative chosen. - -| Question | Decision | Trade-off | -|---|---|---| -| Why Redis at all (vs an in-process cache) | Shared store enables horizontal scaling across N scheduler instances; hook-fed deltas are more reliable than reseed-as-primary for convergence | Operational dependency on Redis; new SPOF until HA lands | -| Where the show-ownership flag lives | New `b_scheduler_managed` column on `show`; replaces `dispatcher.exclusion_list` | One-time migration cost for deployments using the old property | -| Per-show vs per-allocation granularity | Per-show only | Mixed-mode shows not supported | -| How PG accounting tables stay current for scheduler-managed shows | Periodic recompute from `SUM(proc)` | CueGUI lag bounded by recompute interval (2 min) | -| How Cuebot release path stays current in Redis | `afterCommit` publish on the release path only - **no** publish on admin/lifecycle paths | Cuebot admin changes propagate to Redis via the 5-min limit reseed, not instantly | -| Concurrency safety on reseeds | `acct:seq` CAS guard on every reseed | Rare RETRY loop under sustained load; bounded by a max-retries cap | -| Idempotency on the booking Lua | None - trust caller retry semantics | Duplicate bookings double-count in Redis until next recompute heals | -| Bootstrap behavior | Blocking reseed at startup, always | Scheduler startup time increases by the bootstrap reseed duration | -| Redis client choice | `redis-rs` (Rust, async with built-in pooling); Lettuce (Java, composes with Spring DI and `TransactionSynchronization.afterCommit`) | Two clients to maintain awareness of | -| Cuebot rollout shape | No two-phase rollout; ship Cuebot with a no-op-when-disabled switch from the start | Cuebot deployments without Redis must explicitly set `accounting.redis.enabled=false` | -| Cuebot per-release flag lookup cost | `ShowDao` cache with ~30 s TTL | Brief stale window after `setSchedulerManaged` toggle | - -### Why the per-show partition is load-bearing - -Within a single show, exactly one of Cuebot or Rust owns the accounting write -path. There is no double-write and no per-key arbitration. The transition -between modes is brief and self-heals via the next recompute. - -The alternative - per-allocation or per-row arbitration - would force every -write on both sides to check ownership, every recompute to handle partial -state, and every operator toggle to specify which dimension was flipping. The -per-show simplification is the reason the rest of the design fits on one page. - ---- - -## Known limitations and future work - -These are documented gaps. Each must be addressed before the situation in the -"becomes load-bearing" column arises. - -| Item | Why deferred | When it becomes load-bearing | -|---|---|---| -| Leader election for the recompute and limit-reseed loops | Single scheduler instance for now | Before deploying > 1 scheduler instance - two schedulers running the recompute concurrently would race the CAS guard but waste cycles | -| Multi-scheduler bootstrap race | Single scheduler for now | Same | -| Redis HA (Sentinel or Cluster) | Single-node Redis accepted as new SPOF | If "scheduler stops, reseed on recovery" outage tolerance becomes unacceptable in production | -| Cuebot admin afterCommit hooks (size/burst/folder caps) | Drift heals via 5-min limit reseed | If 5-minute limit drift becomes problematic for operators | -| Idempotency tokens on the booking Lua | Recompute heals double-counts within 2 min | If duplicate-booking rate is observed to materially affect dispatch correctness | -| CueGUI surfacing of `b_scheduler_managed` | cueadmin CLI is enough for now | Whenever operator UX is prioritised | - -The recompute and limit-reseed loop entry points in `pipeline/entrypoint.rs` -carry `// TODO: gate behind leader-election when multi-scheduler lands` -comments as a pin against accidentally rolling out > 1 scheduler without -addressing leader election first. - ---- - -## Source layout - -| Path | Purpose | -|---|---| -| `rust/crates/scheduler/src/accounting/mod.rs` | Module root; orchestrates booking, recompute, limit reseed, bootstrap | -| `rust/crates/scheduler/src/accounting/redis_client.rs` | Redis connection management, Lua script wiring | -| `rust/crates/scheduler/src/accounting/lua.rs` | Booking + force-rollback Lua sources | -| `rust/crates/scheduler/src/accounting/recompute.rs` | 2-min `SUM(proc)` → PG + Redis dual write | -| `rust/crates/scheduler/src/accounting/limit_reseed.rs` | 5-min accounting tables → Redis | -| `rust/crates/scheduler/src/accounting/bootstrap.rs` | Blocking startup reseed | -| `rust/crates/scheduler/src/accounting/managed_shows.rs` | Cached lookup of `b_scheduler_managed = true` shows | -| `rust/crates/scheduler/src/accounting/booking_delta.rs` | Per-booking delta carried through the dispatch pipeline | -| `rust/crates/scheduler/src/accounting/dao.rs` | PG accounting-table queries used by the reseeds | -| `cuebot/.../service/AccountingRedisPublisher.java` | Java interface for the `afterCommit` release publisher | -| `cuebot/.../service/LettuceAccountingRedisPublisher.java` | Lettuce-backed implementation; the Lua release script lives here | -| `cuebot/.../dao/postgres/ProcDaoJdbc.java` | Hosts the show-aware branch in `unbookProc` | -| `cuebot/.../dao/postgres/ShowDaoJdbc.java` | Hosts the `b_scheduler_managed` lookup + cache and the startup count | - -## Glossary - -- **Accounting tables**: the five PG tables that track reserved resources at - each hierarchy level - `subscription`, `folder_resource`, `job_resource`, - `layer_resource`, `point`. -- **`acct:seq`**: monotonic Redis counter incremented by every mutating Lua - script; the CAS guard for reseeds. -- **Booked counters**: the `int_cores` / `int_gpus` fields - "how much is - currently reserved", as opposed to limit fields (size, burst, max_cores). -- **CAS guard**: compare-and-swap on `acct:seq` to detect concurrent mutations - between a reseed's read and write. -- **Limit fields**: `size`, `burst`, `int_min_cores`, `int_max_cores`, - `int_priority` - configured by operators, changed by admin paths, not by - bookings. -- **Recompute**: the 2-min loop that rebuilds booked counters from - `SUM(proc)`. -- **Limit reseed**: the 5-min loop that copies limit fields from PG to Redis. -- **Scheduler-managed show**: a show with `b_scheduler_managed = true` - - dispatch is owned by the Rust scheduler, releases go through Redis. -- **Cuebot-managed show**: a show with `b_scheduler_managed = false` - legacy - behavior, Redis not consulted. diff --git a/docs/_docs/developer-guide/scheduler-accounting.md b/docs/_docs/developer-guide/scheduler-accounting.md new file mode 100644 index 0000000000..20afe71da3 --- /dev/null +++ b/docs/_docs/developer-guide/scheduler-accounting.md @@ -0,0 +1,515 @@ +--- +title: "Scheduler Accounting Reference" +nav_order: 101 +parent: Reference +layout: default +linkTitle: "Scheduler Accounting" +date: 2026-06-26 +description: > + Technical reference for the in-memory accounting subsystem in the Rust scheduler and the + PostgreSQL LISTEN/NOTIFY feed from Cuebot +--- + +# Scheduler Accounting Reference + +## How the Rust scheduler enforces per-show resource limits with in-memory counters and a PostgreSQL NOTIFY feed from Cuebot + +--- + +## Overview + +The accounting subsystem tracks how much of each resource pool is currently +booked so that every dispatch decision the [Rust scheduler](/docs/developer-guide/scheduler/) +makes can be gated on a hard cap. If a job is already at its `int_max_cores`, +the scheduler must not book another frame against it. + +The single source of truth for a scheduler-managed show's booked counters is an +**in-memory `Store` inside the scheduler process**. PostgreSQL remains the +durable system of record for the `proc` rows the counters are derived from, and +Cuebot keeps the PG accounting tables fresh for CueGUI, but no external store +sits on the booking hot path. A booking is a lock-guarded, in-process atomic +check-and-increment. + +This replaces an earlier Redis-backed design. Redis was introduced to let the +counters be shared across N scheduler instances, but the scheduler is and will +remain **single-instance (N=1)**. At N=1, Redis's only unique benefit (splitting +one show across instances) is unreachable, while it manufactured an entire class +of accounting-drift bugs (limit-seeding fail-closed, mass dispatch rejection, +double-booking, CAS starvation). A single in-process counter makes that bug +class *structurally impossible*: there is exactly one writer and one reader of +the booked state, and the check and the increment happen under the same lock. + +Source: +- `rust/crates/scheduler/src/accounting/` (the store, listener, and backstop loops) +- `cuebot/.../service/AccountingNotifier.java` (the PG `pg_notify` emitter) + +--- + +## Source-of-truth model + +This is the load-bearing design choice; everything else falls out of it. + +| Component | Hot path | Backstop | +|---|---|---| +| Rust scheduler (booking) | In-process atomic check + increment in `Store` under one lock, then `INSERT proc` in PG transactionally | Recompute (~15 s) overwrites booked counters from `SUM(proc)`, carrying in-flight bookings forward | +| Cuebot release path (`ProcDaoJdbc.procDestroyed`) | For scheduler-managed shows: `DELETE proc` plus a transactional `pg_notify('acct_release', …)` in the **same** transaction. For Cuebot-managed shows: unchanged transactional accounting-table UPDATEs | Recompute heals any missed notification | +| Cuebot admin cap changes (burst / folder & job max cores/gpus) | Transactional accounting-table UPDATE plus a transactional `pg_notify('acct_limit_change', …)` in the same transaction | Limit reseed (~5 min) re-reads the caps from PG | +| CueGUI | Reads PG accounting tables (refreshed by the scheduler's recompute for managed shows; written transactionally by Cuebot for Cuebot-managed shows) | - | + +Three properties hold: + +- **`proc` is canonical for bookings.** Both sides write `proc` + transactionally. The booked counters can always be reconstructed from + `SELECT SUM(int_cores_reserved) FROM proc GROUP BY pk_show, pk_alloc` (and the + folder/job groupings). +- **The in-memory `Store` is the live enforced state.** It is what the booking + check reads and increments. It is seeded from PG at startup and kept fresh by + the NOTIFY feed, reconciled to `SUM(proc)` by the recompute. +- **The PG accounting tables are derived, for display only.** For + scheduler-managed shows they are refreshed by the recompute loop; for + Cuebot-managed shows Cuebot writes them transactionally as before. Nothing on + the scheduler's hot path reads them. + +### Why in-memory, not Redis or PG-on-the-hot-path + +- **PG-on-the-hot-path** is what the scheduler was built to escape: the + scheduler's booking rate hammered the same accounting rows Cuebot's + `HostReportHandler` writes, and lock waits on `subscription`, + `folder_resource`, and `job_resource` limited throughput. +- **Redis** decoupled the hot path from PG locks and could in principle be + shared across schedulers, but at N=1 it bought nothing the in-process store + does not, and every reseed had to defend against a read/write race with live + hot-path writes (the `acct:seq` compare-and-swap). That race only exists + because the counter lives in a separate process from the writer. Move the + counter in-process and the race — and the CAS, and the retry loop, and the + starvation failure mode — all disappear. + +The in-process store keeps the hot path off PG locks (the win Redis gave) while +removing the cross-process coordination Redis required (the cost Redis added). + +--- + +## Show ownership: the per-show partition + +Within a single show, exactly one of Cuebot or the scheduler owns the accounting +write path. There is no double-write and no per-key arbitration. The flag lives +on the `show` table (migration `V45__show_scheduler_managed.sql`, unchanged by +this rewrite): + +```sql +ALTER TABLE show ADD COLUMN b_scheduler_managed BOOLEAN NOT NULL DEFAULT false; +``` + +- `b_scheduler_managed = false` (default): the show is **Cuebot-managed**. + Cuebot books and releases against the PG accounting tables transactionally, + exactly as before. The scheduler's store is not consulted, and Cuebot emits no + NOTIFY for it. +- `b_scheduler_managed = true`: the show is **scheduler-managed**. The scheduler + books against its in-memory store on the hot path; Cuebot's release path only + deletes the `proc` row and emits an `acct_release` notification. The PG + accounting tables for this show are refreshed by the scheduler's recompute. + +The flag is **per-show, not per-allocation.** A show is either +scheduler-managed or it isn't. + +### Looking up the flag + +The scheduler caches the set of `b_scheduler_managed = true` show ids in +`ManagedShowsCache`, refreshed on a TTL (`managed_shows_ttl`, default 30 s). +`apply_booking` consults this cache and is a no-op for shows the scheduler does +not currently manage. Cuebot independently caches the flag for its release path. +The brief stale window after a toggle is safe in both directions — see +[Managed-flip seed](#managed-flip-seed) and [Failure modes](#failure-modes-and-drift). + +--- + +## The store + +`accounting/store.rs` holds the entire enforced state in process behind a single +`Mutex`: + +| Map | Key | Holds | +|---|---|---| +| `sub` | `(show_id, alloc_id)` | booked `cores`/`gpus` + `pending` deltas | +| `folder` | `folder_id` | booked `cores`/`gpus` + `pending` deltas | +| `job` | `job_id` | booked `cores`/`gpus` + `pending` deltas | +| `sub_burst` | `(show_id, alloc_id)` | subscription burst cap, in cores | +| `folder_caps` | `folder_id` | folder `max_cores` / `max_gpus`, in cores | +| `job_caps` | `job_id` | job `max_cores` / `max_gpus`, in cores | + +### Only three vertices are enforced + +The five accounting tables in PG are subscription, folder, job, layer, and +department point. The scheduler tracks and enforces only **three**: subscription +(burst), folder (`int_max_cores`/`int_max_gpus`), and job +(`int_max_cores`/`int_max_gpus`). The booking Lua this replaces incremented +layer and point counters too, but the booking check never *read* them, so they +are not kept in the store. (Layer/point limits are still visible to CueGUI via +PG, unchanged.) + +### One lock, pure in-memory critical sections + +A single `Mutex` guards the whole store. Every critical section is pure +in-memory arithmetic — no I/O, no `.await` — so contention is negligible at this +scale and the multi-vertex check-and-increment is trivially atomic. A booking +touches subscription, folder, and job under one lock acquisition; there is no +window in which a concurrent booking could see a half-applied increment. + +### The booking hot path + +`Store::book(&BookingDelta)` performs, under the lock, an atomic +check-and-increment across the three enforced vertices: + +```text +1. If cores delta > 0: + check subscription burst -> reject "subscription" if over + check folder int_max_cores -> reject "folder" if over + check job int_max_cores -> reject "job" if over +2. If gpus delta > 0: + check folder int_max_gpus -> reject "folder_gpus" if over + check job int_max_gpus -> reject "job_gpus" if over +3. On success: increment all three vertices and record the delta as `pending`. +``` + +It returns `Applied` or `LimitExceeded { table, current, limit }`, where `table` +names the offending vertex for metrics and operator-facing rejection messages. +The `INSERT proc` happens in PG **outside** the lock, after `book` returns +`Applied`. + +#### Cap conventions + +A subscription **burst** cap of `0` means "reject all" — a missing burst entry +(before the bootstrap seed) reads as `0` and fails closed, matching Cuebot's +`IS_SHOW_OVER_BURST` convention. A folder/job **max** of `0` or any non-positive +value (notably the `-1` unlimited sentinel) is treated as *unbounded* — the +`> 0` guard skips the comparison. The two conventions are distinct on purpose: +an unseeded subscription must reject (we don't know its real burst yet), whereas +an unset hard cap genuinely means unlimited. + +### Confirm and rollback + +After `book` returns `Applied`, the dispatcher carries the delta through to the +`proc` transaction's outcome: + +- **`confirm`** runs once the `proc` row is committed and RQD has launched the + frame. It drops the `pending` portion of the delta but keeps the booked + increment. +- **`rollback`** runs if the `proc` INSERT or RQD launch fails. It undoes both + the booked increment and the `pending` delta (the in-process equivalent of the + old force-rollback Lua). + +Exactly one of `confirm`/`rollback` runs per successful `book`. Both ignore +current managed status: if `apply_booking` applied a delta, it must be settled +even if the show flipped to Cuebot-managed in between. + +If the scheduler crashes between `book` and the `proc` INSERT, the booked +increment is simply lost from memory — but so is the whole store, which the +[bootstrap seed](#bootstrap-blocking-startup-seed) rebuilds from `SUM(proc)` on +restart. There is no durable over-count to clean up. + +--- + +## The two NOTIFY channels + +Cuebot is the only writer of `proc` deletions and cap changes, so it is the +source of the two live signals that keep the scheduler's store fresh between +recompute passes. Each notification is emitted with `SELECT pg_notify(channel, +payload)` **in the same transaction** as the PG write it describes. + +`pg_notify` is transactional: the notification is delivered if and only if the +enclosing transaction commits, and is discarded on rollback. This is strictly +better than the old Redis `afterCommit` publish, which ran *after* the commit +and so had a partial-failure window (commit succeeds, publish fails). Here there +is no such window — the DELETE and its release signal are atomic. + +The scheduler listens on both channels with a dedicated `PgListener` +(`accounting/listener.rs`), separate from the query pool. On any connection +drop it reconnects with a fixed backoff; missed notifications during the gap are +healed by the backstop loops (see [Failure modes](#failure-modes-and-drift)). + +### `acct_release` + +Emitted per proc on the scheduler-managed branch of `ProcDaoJdbc.procDestroyed`. +The payload carries the show/alloc/folder/job ids and **signed** core/gpu deltas +(negative for a release), in cores: + +```json +{ + "show": "", + "alloc": "", + "folder": "", + "job": "", + "layer": "", + "dept": "", + "cores": -10, + "gpus": -1 +} +``` + +The scheduler decrements the subscription/folder/job counters by the delta. +`layer` and `dept` are included for symmetry and debuggability; the listener +ignores them (those vertices are not enforced). The decrement is unconditional +and pending-free — a release is always for a long-committed booking. + +### `acct_limit_change` + +Emitted in the same transaction as a cueadmin cap change. Exactly one vertex per +message; core values are in cores (`-1` = unlimited), GPUs pass through: + +```json +{ "vertex": "sub", "show": "", "alloc": "", "burst": 200 } +{ "vertex": "folder", "id": "", "max_cores": 20, "max_gpus": 4 } +{ "vertex": "job", "id": "", "max_cores": -1 } +``` + +For `folder` and `job`, `max_cores` and `max_gpus` are each optional; a message +may set one, the other, or both (the listener expands a both-fields message into +two cap changes). Only the **five enforced caps** are wired to emit: +subscription burst, folder max cores/gpus, and job max cores/gpus. Size, +min-cores, priority, layer, and point caps are not enforced by the scheduler and +emit nothing. + +Cuebot emits a cap change only for scheduler-managed shows: each cap DAO resolves +the owning show id (the subscription/group/job interfaces all expose it, so no +extra query is needed) and gates the `pg_notify` on `ShowDao.isSchedulerManaged` +(short-circuited by the `accounting.notify.enabled` flag). The scheduler's listener +applies every cap change it receives without re-filtering — correctness comes from +the emit-side gate. Admin cap changes are rare, so this costs nothing in practice. + +--- + +## The recompute backstop and the pending carry-forward + +The recompute loop (`accounting/recompute.rs`, every `recompute_interval`, +default **15 s**) is the correctness backstop, not the primary path. Each pass: + +1. **PG side (for CueGUI):** the existing `RECOMPUTE_*_FROM_PROC` UPDATEs rewrite + the PG accounting tables from `SUM(proc)` for scheduler-managed shows, so + CueGUI's view stays within one interval of reality. (Scoped to managed shows; + an empty managed set is a no-op, so it never clobbers Cuebot's accounting.) +2. **Store side:** one `SUM(proc)` snapshot, grouped by (show, alloc, folder, + job), is overlaid on a **zero-baseline** of every enumerable key and written + **absolutely** into the store — but each key's in-flight `pending` delta is + carried forward: `counter = SUM(proc) + pending`. + +The zero-baseline (every enumerable sub/folder/job key seeded at 0 before the +sums fold in) means a key that drifted stale-high and then drained to zero procs +is reset, rather than wedged at its stale value forever — `SUM(proc)` alone only +returns keys that still have procs. + +### Why pending carry-forward is required + +The recompute reads `SUM(proc)` (step 2 above) and then overwrites the in-memory +counter. Those two operations are not atomic with respect to the booking hot +path. Consider a booking that lands in the **straddle window** — after the +snapshot's `SELECT` has read the rows but before its `proc` INSERT is visible to +that read: + +| t | Event | `proc` visible to snapshot | store `job.cores` | +|---|---|---|---| +| t0 | recompute `SELECT SUM(proc)` for job J reads 0 | 0 | 0 | +| t1 | `book(+8)` for J: store → 8, recorded as `pending` | 0 (INSERT not yet committed/visible) | 8 | +| t2 | recompute overwrites J from snapshot | 0 | **?** | + +A naive absolute overwrite would write `0` at t2 and erase the just-booked 8 +cores. The counter would then read low, and the *next* booking could push J over +its hard cap — the **one** way an absolute overwrite can over-book. + +The carry-forward closes this: at t2 the store writes `SUM(proc) + pending = +0 + 8 = 8`. The booking survives. + +### "When in doubt, keep the booking" is always safe + +`pending` is cleared by `confirm`/`rollback`, which run only after the dispatcher +knows the `proc` outcome. So a delta is `pending` exactly while its `proc` +visibility is uncertain. If the snapshot happened to *already* include that +proc, the carry-forward double-counts it for one interval — the counter reads +**high**, which can only cause a too-conservative *under*-book, which the next +recompute corrects. The asymmetry is deliberate: over-counting is self-healing +and harmless to a hard cap; under-counting can breach a hard cap. When in doubt, +keep the booking. + +This single invariant — `counter = SUM(proc) + Σ(in-flight bookings)` — subsumes +the old `acct:seq` compare-and-swap entirely. There is **no CAS, no retry loop, +no `acct:seq`, and no starvation floor.** The live store is the primary record; +the recompute only reconciles. + +--- + +## The other backstops and seeds + +### Limit reseed (cap-change backstop) + +`accounting/limit_reseed.rs`, every `limit_reseed_interval` (default 5 min), +re-reads the five enforced caps from PG and writes them into the store. The +`acct_limit_change` NOTIFY propagates cueadmin changes immediately; this loop +heals any notification missed during a listener reconnect, within one interval. +Only the enforced caps are read (subscription burst, folder/job max cores+gpus). + +### Bootstrap (blocking startup seed) + +`accounting/bootstrap.rs` runs **before the scheduler accepts any work**: it +seeds the enforced caps (limit reseed), then the booked counters (recompute +reseed), from PG. Because the store is the only copy of this state, the gate is +mandatory — dispatching against empty counters would book every hard cap wide +open, and (because an unseeded burst reads as 0 = reject-all) would +simultaneously reject every subscription. The entrypoint runs the bootstrap to +completion before spawning the recompute, limit-reseed, and listener loops. + +### Managed-flip seed + +When a show becomes scheduler-managed *after* startup, `ManagedShowsCache` +performs a **blocking seed of both its caps and its booked counters before +publishing the show into the cache**. The booked seed is essential: a flipped show +often already has live Cuebot procs, so seeding the booked counters from +`SUM(proc)` first means the hot path enforces against real usage from the very +first booking, not against 0 (which would leave a full burst of headroom free → +over-book a hard cap). The booked seed is a one-shot absolute set per show — it +does not bump the recompute epoch or touch the settled buckets, so it cannot +interfere with the single recompute driver's begin/overwrite sequencing (the show +has no in-flight scheduler bookings yet, since the hot path no-ops for unpublished +shows). Until the seed lands the show is treated as Cuebot-managed (Cuebot keeps +booking it via PG) — strictly safer than flipping the hot path on against unseeded +state. If the seed fails, only that show's cache publish is deferred to the next +refresh; removals (shows that left the managed set) still apply immediately. + +--- + +## Failure modes and drift + +Every failure mode is **safe-direction**: a dropped or delayed signal can only +leave a counter reading *high*, which under-books (too conservative) and +self-heals. A hard cap can only be breached if a counter reads *low*, which only +the recompute-erase hole (closed by the [pending carry-forward](#why-pending-carry-forward-is-required)) +or an unseeded counter (closed by the [blocking bootstrap and managed-flip +seeds](#the-other-backstops-and-seeds)) could cause. + +| Failure | Effect | Recovery | +|---|---|---| +| `acct_release` NOTIFY missed (listener reconnecting) | Store missing a decrement → counter high → under-book | Next recompute (~15 s) overwrites from `SUM(proc)` | +| `acct_limit_change` NOTIFY missed | Store cap stale | Next limit reseed (~5 min) re-reads from PG | +| Scheduler dies between `book` and `proc` INSERT | Booked increment lost on crash (store is in memory) | Bootstrap reseed from `SUM(proc)` on restart | +| `proc` INSERT / RQD launch fails after `book` | `rollback` undoes the increment + pending | Immediate; recompute is a further backstop | +| Recompute snapshot straddles a live booking | Snapshot misses the proc | Carry-forward keeps the booking; never under-counts | +| Cuebot admin cap change | Store stale on that cap until NOTIFY/limit-reseed | `acct_limit_change` NOTIFY (instant) or limit reseed (~5 min) | +| `b_scheduler_managed` toggle mid-flight | Brief window of stale managed-set | Stale-true heals via recompute; stale-false defers to Cuebot (safe); managed-flip seed gates enforcement | +| Cuebot NOTIFY kill-switch off | No live releases/cap-changes → counters high → under-book | Recompute / limit reseed still heal; ops alerted by metric | + +### CueGUI staleness + +CueGUI reads the PG accounting tables unchanged. For scheduler-managed shows +they lag the live store by at most one recompute interval (~15 s) plus any +bookings since the last recompute. For Cuebot-managed shows they are +transactionally exact as before. + +--- + +## The Cuebot kill-switch + +A single property gates whether Cuebot emits the accounting notifications: + +```properties +accounting.notify.enabled=true # default; ${ACCOUNTING_NOTIFY_ENABLED} +``` + +With the flag **off**, Cuebot still deletes procs and updates caps +transactionally but emits no `pg_notify`. The scheduler's store then stops +receiving live releases and cap changes, so its counters only ever grow (reads +high) → it under-books → the recompute and limit-reseed loops heal it within +their intervals. This is the **safe** direction, so flag-off degrades +gracefully to backstop-only operation; it does not over-book. + +Because flag-off is safe, there is **no startup deployment guardrail** that +refuses to run (the old Redis design had one because a disabled Redis publisher +*over*-counted). Instead, when scheduler-managed shows exist and the flag is off, +Cuebot logs a WARN and exposes a `cuebot_accounting_notify_disabled` metric for +ops visibility — utilization will sag (under-booking), but correctness holds. + +The per-show `b_scheduler_managed` toggle remains the live operational rollback: +flip a show back to Cuebot-managed to take it off the scheduler entirely. + +--- + +## The unit invariant: cores, not centicores + +PostgreSQL stores cores as **centicores** (cores × 100; the `int_*cores*` +columns and `proc.int_cores_reserved`). The in-memory store works in +**unmultiplied cores** (1 = 1 core). Conversion happens only at the PG↔store +boundary: + +- The recompute converts `SUM(proc)` centicore sums to cores + (`centicores_to_cores`, via `CoreSize::from_multiplied`). +- The limit reseed converts caps to cores (`centicores_to_cores_cap`), + preserving the `-1` unlimited sentinel. +- Cuebot converts to cores before emitting the NOTIFY payloads. + +GPU fields are **not** multiplied — they pass through verbatim, including their +`-1` sentinel. Inside the store no centicore arithmetic ever happens; the hot +path is unit-clean. + +--- + +## The N=1 assumption and the revisit trigger for N>1 + +The in-memory store is **not shared**. It assumes a single scheduler instance: +exactly one process owns the booked counters, so the booking check and increment +can be a single in-process critical section, and the recompute can overwrite +absolutely without coordinating with any peer. This is the assumption that makes +the whole design correct *and* makes the drift bug class go away. + +This is a deliberate trade. The Redis design existed to allow N>1 schedulers to +share counters, but at this scale N=1 is expected for the foreseeable future and +the only thing N>1 would buy (splitting one show across instances) is not +needed. + +**Revisit trigger:** before ever running more than one scheduler instance that +could book the same show. At that point the in-memory store is no longer a +single source of truth — two processes would each enforce against their own copy +and could jointly over-book a hard cap. Crossing N>1 requires re-introducing a +shared/coordinated counter (a shared store with an atomic check-increment, or a +partitioning scheme that guarantees no two instances ever book the same +subscription/folder/job), plus leader election for the recompute and +limit-reseed loops. None of that is in place today, and the code assumes it is +absent (see `managed_shows.rs` and the entrypoint notes). + +--- + +## Source layout + +| Path | Purpose | +|---|---| +| `accounting/mod.rs` | `AccountingService` facade; `apply_booking` / `confirm_booking` / `rollback_booking`; managed-show short-circuit | +| `accounting/store.rs` | In-memory counters + caps; the locked atomic `book`, `confirm`, `rollback`, `apply_release`, `overwrite_counters`, `set_caps`, `apply_limit_change` | +| `accounting/listener.rs` | `PgListener` on `acct_release` + `acct_limit_change`; payload parsing | +| `accounting/recompute.rs` | ~15 s `SUM(proc)` → PG tables (CueGUI) + store overwrite with pending carry-forward | +| `accounting/limit_reseed.rs` | ~5 min caps → store; the cap-change backstop | +| `accounting/bootstrap.rs` | Blocking startup seed (caps then counters) before dispatch | +| `accounting/managed_shows.rs` | Cached `b_scheduler_managed` set + blocking managed-flip seed | +| `accounting/booking_delta.rs` | Per-booking delta carried through the dispatch pipeline | +| `accounting/dao.rs` | PG queries for the snapshot, baseline keys, and cap tables | +| `accounting/error.rs` | `AccountingError::LimitExceeded` (the one hot-path failure mode) | +| `cuebot/.../service/AccountingNotifier.java` | Emits `pg_notify` for releases and cap changes | +| `cuebot/.../dao/postgres/ProcDaoJdbc.java` | Scheduler-managed branch in `procDestroyed`: DELETE proc + transactional release notify | +| `cuebot/.../dao/postgres/ShowDaoJdbc.java` | `b_scheduler_managed` cache + `setSchedulerManaged` | + +## Glossary + +- **Accounting vertices**: the resource pools a booking touches — + subscription, folder, job, layer, department point. The scheduler **enforces** + only subscription, folder, and job. +- **Booked counters**: the live "how much is currently reserved" `cores`/`gpus` + the cap check reads, as opposed to limit fields (burst, max_cores). +- **Pending delta**: the subset of a counter still in flight — booked in memory + but whose `proc` row may not yet be visible to the recompute snapshot. Carried + forward across a recompute so the absolute overwrite cannot erase it. +- **Recompute**: the ~15 s loop that overwrites booked counters from + `SUM(proc) + pending` and refreshes the PG tables for CueGUI. +- **Limit reseed**: the ~5 min loop that re-reads enforced caps from PG. +- **`acct_release` / `acct_limit_change`**: the two PG NOTIFY channels Cuebot + emits transactionally for releases and cap changes. +- **Scheduler-managed show**: `b_scheduler_managed = true` — dispatch and + hot-path accounting are owned by the Rust scheduler's in-memory store. +- **Cuebot-managed show**: `b_scheduler_managed = false` — Cuebot dispatches and + updates the PG accounting tables transactionally; the scheduler's store is not + consulted. + diff --git a/docs/_docs/developer-guide/scheduler-stress-testing.md b/docs/_docs/developer-guide/scheduler-stress-testing.md index b1400feb23..78bd2082cd 100644 --- a/docs/_docs/developer-guide/scheduler-stress-testing.md +++ b/docs/_docs/developer-guide/scheduler-stress-testing.md @@ -20,37 +20,37 @@ description: > The stress suite (`rust/crates/scheduler/tests/stress_tests.rs`) exercises the [Rust scheduler](/docs/developer-guide/scheduler/)'s full production dispatch -path at scale — `pipeline::run` end to end: Redis accounting bootstrap → +path at scale — `pipeline::run` end to end: in-memory accounting bootstrap → cluster feed → pending-job query → host matching → dispatch (proc insert, host ledger decrement, frame start) — against a deterministic, bulk-seeded farm. It is both a **correctness gate** and a **benchmark harness**: -- **Correctness**: after each phase an audit cross-checks every Redis `acct:*` - hash the run touched against `SUM(proc)` in Postgres (the canonical record — - see the [Redis-Backed Accounting Reference](/docs/developer-guide/redis-accounting/)), +- **Correctness**: after each phase an audit cross-checks the scheduler's + in-memory accounting counters against `SUM(proc)` in Postgres (the canonical + record — see the [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/)), and verifies cap enforcement and ledger invariants. - **Benchmark**: it reports booking throughput (frames/s over the active booking window), host-matching efficiency (wasted attempt %), host-cache hit - ratio, and Redis Lua op counts. + ratio, and booking/rollback op counts. The suite runs two phases in one process: | Phase | Shape | What it proves | |---|---|---| | **drain** | Farm capacity comfortably exceeds demand (default: 1,200 hosts, 6,000 frames) | ≥90% of frames book; throughput measured; accounting stays exact under concurrency, including the force-rollback compensation path | -| **saturation** | Demand vastly exceeds tight subscription bursts and per-job core caps (default: 400 hosts, 3,000 frames, 150-core bursts) | The Redis Lua cap check is the binding constraint: bookings stop exactly at burst, caps are never breached, rejections flow through the hot path | +| **saturation** | Demand vastly exceeds tight subscription bursts and per-job core caps (default: 400 hosts, 3,000 frames, 150-core bursts) | The in-memory cap check is the binding constraint: bookings stop exactly at burst, caps are never breached, rejections flow through the hot path | ### Invariants the audit asserts -1. Every `acct:{sub,folder,job,layer,point}` hash holds exactly - `SUM(proc.int_cores_reserved)/100` cores and `SUM(proc.int_gpus_reserved)` - GPUs for its grouping — the same 5-dimension grouping and centicore→core - conversion the recompute loop uses. The suite pushes the recompute and - limit-reseed loops out to a 1-hour interval, so agreement here proves the - *dispatch hot path alone* (Lua book + force-rollback) kept Redis exact — - reconciliation never got a chance to paper over drift. -2. Jobs with no bookings have no leaked Redis counters. +1. The in-memory counter for every enforced vertex (subscription, folder, job) + the run touched matches `SUM(proc.int_cores_reserved)/100` cores and + `SUM(proc.int_gpus_reserved)` GPUs for its grouping — the same grouping and + centicore→core conversion the recompute loop uses. The suite pushes the + recompute and limit-reseed loops out to a long interval, so agreement here + proves the *dispatch hot path alone* (book + force-rollback) kept the store + exact — reconciliation never got a chance to paper over drift. +2. Jobs with no bookings have no leaked counters. 3. Per-(show, alloc) booked cores never exceed the subscription burst. 4. Per-job booked cores never exceed `job_resource.int_max_cores`. 5. Host ledger: `int_cores - int_cores_idle == SUM(proc)` per host, never negative. @@ -58,6 +58,22 @@ The suite runs two phases in one process: 7. Trigger-maintained `job_stat.int_waiting_count` matches the frame table. 8. After teardown, zero `stress_%` rows remain in any table the suite touches. +### Load-bearing perturbation tests + +Beyond the two throughput phases, the suite drives the failure modes the +in-memory design must survive: + +- **Recompute straddle** — stall the recompute `SUM(proc)` query, fire a booking + into the snapshot-straddle window, and assert the counter never ends below + truth (the pending carry-forward invariant; the one way an absolute overwrite + could under-count and over-book a hard cap). +- **Injected NOTIFY drops** — drop `acct_release` notifications under load and + assert no hard cap is ever exceeded and counters converge to `SUM(proc)` once + load stops (a missed release only ever reads high → under-book → heals). +- **Mid-load managed-flip** — flip a show scheduler-managed during saturation and + assert the blocking caps seed prevents any over-book on the freshly-flipped + show. + ## Running locally ### Prerequisites @@ -75,8 +91,8 @@ The suite runs two phases in one process: done ``` -- A running Docker daemon. The suite starts its own throwaway Redis container - via testcontainers; all accounting state dies with it. +The accounting store is in-process, so no external store (and no Docker daemon +for one) is required — a migrated Postgres is the only dependency. ### Run @@ -117,13 +133,13 @@ runs reflect the code, not the data. frames : 6000 seeded, 5988 dispatched (99.8%), waiting 6000 -> 12 throughput : 975.1 frames/s over a 6.1s booking window (wall 43.3s) matching : 3175 host attempts (41.9% wasted), 39 cluster rounds, host-cache hit 98% -accounting : 7452 redis lua ops, 5988 dispatches (metrics), 24040 booked cores, rejections [...] +accounting : 7452 booking ops, 5988 dispatches (metrics), 24040 booked cores, rejections [...] audit : OK ``` - **throughput** is measured from the first to the last `proc.ts_booked`, so it excludes the post-drain shutdown tail of the feed (the `wall` figure includes it). -- **redis lua ops** above the dispatch count means the compensation path ran: +- **booking ops** above the dispatch count means the compensation path ran: each failed dispatch costs one book plus one force-rollback. The audit passing alongside a surplus is a *positive* signal — rollbacks netted out. - In the saturation phase, expect large `subscription=` rejection counts and @@ -133,8 +149,8 @@ audit : OK All database rows the suite creates are prefixed `stress_`. The suite sweeps that prefix **before** seeding (so leftovers from a crashed earlier run never -skew results) and **after** the run, then asserts zero residue. Redis state -needs no cleanup — the container is destroyed with the test. If a run is +skew results) and **after** the run, then asserts zero residue. The accounting +state is in-process and dies with the test, so it needs no cleanup. If a run is killed hard (e.g. SIGKILL mid-phase), the next run's pre-sweep removes the leftovers. @@ -165,10 +181,10 @@ workflow. ### What fails the job -Only correctness regressions: accounting drift between Redis and Postgres, a -cap breach, booking liveness failures (drain below target, or a saturated farm -producing no Redis rejections), a phase that never converges (hard-timeout), -or test data left behind after cleanup. +Only correctness regressions: accounting drift between the in-memory store and +Postgres, a cap breach, booking liveness failures (drain below target, or a +saturated farm producing no rejections), a phase that never converges +(hard-timeout), or test data left behind after cleanup. ### Launching a manual benchmark run @@ -180,15 +196,16 @@ the run's step summary; the complete log is attached as the ## Scope and limitations - **RQD is not exercised.** The suite runs in `dry_run_mode`: the full booking - path executes (Redis Lua, proc insert, host ledger, frame start) but no gRPC - launch is sent. Frame *completion* and the Cuebot release path are out of - scope — see the [Redis-Backed Accounting Reference](/docs/developer-guide/redis-accounting/) - for how releases are reconciled. + path executes (in-memory book, proc insert, host ledger, frame start) but no + gRPC launch is sent. Frame *completion* and the Cuebot release path are out of + scope — see the [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/) + for how releases are reconciled. (The injected-NOTIFY-drop test simulates the + release-feed behaviour without a live Cuebot.) - **Only scheduler-managed shows** (`show.b_scheduler_managed = true`) are covered; Cuebot-managed accounting is Cuebot's test territory. -- The recompute / limit-reseed loops are intentionally dormant during the run - (see invariant 1); their CAS semantics are covered separately by - `tests/redis_integration.rs` (`--features redis-tests`). +- The recompute / limit-reseed loops are pushed to a long interval for the two + throughput phases (see invariant 1) so the hot path is audited in isolation; + the recompute straddle and managed-flip perturbation tests drive them directly. ## Schema gotchas the suite encodes diff --git a/docs/_docs/developer-guide/scheduler.md b/docs/_docs/developer-guide/scheduler.md index efbdf0b1a4..9c9528e15d 100644 --- a/docs/_docs/developer-guide/scheduler.md +++ b/docs/_docs/developer-guide/scheduler.md @@ -21,7 +21,7 @@ The **Distributed Scheduler** (`rust/crates/scheduler/`) is a standalone Rust se Conceptually, every dispatch is an instance of **multi-dimensional bin packing**: the scheduler must fit a frame (an item with `(cores, memory, gpus)` requirements) into a host (a bin with `(idle_cores, idle_memory, idle_gpus)` capacity). The host cache's B-tree index turns this into an O(log n) lookup, and the [`HostBookingStrategy`](#booking-strategies-bin-packing-heuristics) flags select between classical heuristics (Best-Fit vs. Worst-Fit) on each dimension independently. -Per-show resource accounting (subscription size/burst, folder caps, job/layer caps) is enforced on the hot path through a **Redis-backed accounting layer** shared with Cuebot - see the [Redis Accounting Reference](/docs/developer-guide/redis-accounting/) for the full design. +Per-show resource accounting (subscription burst, folder caps, job caps) is enforced on the hot path through an **in-memory accounting store** inside the scheduler, kept fresh by a PostgreSQL `LISTEN/NOTIFY` feed from Cuebot - see the [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/) for the full design. This document provides technical details for developers, operators, and contributors working with the scheduler's internals. @@ -47,14 +47,13 @@ The scheduler inverts this model: 3. **Intelligent matching** → Find hosts for frames (not frames for hosts) a 2D bin-packing lookup against the host B-tree 4. **Atomic accounting** → Enforce per-show / per-folder / per-job limits via a single - Redis Lua script on the hot path + lock-guarded check-and-increment in an in-memory store on the hot path 5. **Parallel dispatch** → Execute multiple dispatches concurrently **Key Insight**: By caching host state in memory and querying jobs (not hosts), the scheduler dramatically reduces database load. Moving the accounting hot path off -Postgres into Redis removes the remaining lock-contention bottleneck and enables -horizontal scaling across multiple scheduler instances. The host-side placement -becomes a tunable bin-packing heuristic (see +Postgres into an in-process store removes the remaining lock-contention bottleneck. +The host-side placement becomes a tunable bin-packing heuristic (see [Booking Strategies](#booking-strategies-bin-packing-heuristics)) rather than a hard-coded scan. @@ -599,25 +598,28 @@ database: - Prepared statement caching - Transaction support -### 6. Redis-Backed Accounting Subsystem +### 6. In-Memory Accounting Subsystem **Location**: `src/accounting/` The accounting subsystem tracks how much of each resource pool (subscription, folder, -job, layer, department point) is currently booked. Every dispatch decision the -scheduler makes is gated on these counters if a job is already at its `int_max_cores`, -the scheduler must not book another frame against it. +job) is currently booked. Every dispatch decision the scheduler makes is gated on these +counters if a job is already at its `int_max_cores`, the scheduler must not book another +frame against it. Historically these counters lived only in PostgreSQL and were updated transactionally by Cuebot on every booking and release. As the Rust scheduler took over dispatch, the hot path was hammering the same accounting rows Cuebot's `HostReportHandler` writes to, and lock waits on `subscription`, `folder_resource`, and `job_resource` started -limiting throughput. An earlier in-process cache (a `HashMap` inside the -scheduler) solved contention but cannot be shared across multiple scheduler instances. +limiting throughput. A short-lived Redis-backed design moved the hot path off PG locks, +but at a single scheduler instance (N=1) it added cross-process coordination (an +`acct:seq` compare-and-swap on every reseed) without the multi-instance benefit it was +built for, and that coordination spawned a class of accounting-drift bugs. -The current design replaces both with a **Redis-backed accounting layer** that Cuebot -and the Rust scheduler write through on the hot path. PostgreSQL remains the durable -system of record; Redis is the live operational view. +The current design replaces both with an **in-memory `Store`** that is the single source +of truth for booked counters: there is exactly one writer and reader, and the booking +check-and-increment is atomic under one lock. PostgreSQL remains the durable record (the +`proc` rows), and Cuebot feeds live releases and cap changes via `LISTEN/NOTIFY`. #### Per-show ownership @@ -625,11 +627,12 @@ The `show.b_scheduler_managed` boolean (added in migration `V45__show_scheduler_ selects who owns the accounting write path for a given show: - `false` (default): **Cuebot-managed**. Cuebot's dispatcher books and releases against - PG accounting tables transactionally, exactly as before. Redis is not consulted. -- `true`: **Scheduler-managed**. The Rust scheduler books against Redis on the hot - path; Cuebot's release path only deletes the `proc` row and publishes the release - delta to Redis via an `afterCommit` hook. PG accounting tables for this show are - refreshed by the scheduler's 2-min recompute loop. + PG accounting tables transactionally, exactly as before. The scheduler's store is not + consulted. +- `true`: **Scheduler-managed**. The Rust scheduler books against its in-memory store on + the hot path; Cuebot's release path only deletes the `proc` row and emits an + `acct_release` notification. PG accounting tables for this show are refreshed by the + scheduler's recompute loop (so CueGUI stays current). The flag replaces the older `dispatcher.exclusion_list` and `dispatcher.scheduler_manages_resources` properties in `opencue.properties` (both @@ -640,73 +643,82 @@ cueadmin -show -setSchedulerManaged true cueadmin -show -setSchedulerManaged false ``` -#### Hot path: atomic booking Lua +#### Hot path: lock-guarded check-and-increment -A single Lua script runs the per-frame booking against Redis, executing five updates -atomically: +The per-frame booking is an in-process atomic operation in `Store::book`, under a single +lock: ```text -1. Read current state of acct:sub / acct:folder / acct:job / acct:layer / acct:point -2. Check booking would not exceed any limit (size, burst, max_cores, etc.) -3. If OK: 5 × HINCRBY (int_cores, int_gpus) + INCR acct:seq, return {1} -4. If over a limit: return structured failure {0, table_name, current, limit} -5. Then transactionally INSERT proc in Postgres (outside Lua) +1. If cores > 0: check subscription burst, folder int_max_cores, job int_max_cores +2. If gpus > 0: check folder int_max_gpus, job int_max_gpus +3. If OK: increment the three enforced vertices, record the delta as `pending`, return Applied +4. If over a limit: return LimitExceeded { table, current, limit } +5. Then transactionally INSERT proc in Postgres (outside the lock) ``` -On limit-check failure, the rejection is attributed to the responsible table via +Only the three enforced vertices are tracked (subscription, folder, job); layer and +point have no hot-path cap and are not kept in the store. On limit-check failure the +rejection is attributed to the responsible table via `scheduler_accounting_limit_exceeded_total{table=...}` metrics. -If the PG `INSERT proc` fails after the Lua succeeded, the scheduler calls the same -script in **force mode** with negated deltas to undo the Redis-side change one -script, two modes, no separate rollback path. +After the `proc` transaction commits and RQD launches, the dispatcher calls `confirm` +(drops the `pending` portion, keeps the booked increment). If the INSERT or launch +fails, it calls `rollback` (undoes both). Exactly one runs per successful booking. -#### Reseed loops +#### Live feed: PostgreSQL LISTEN/NOTIFY -Three loops keep Redis convergent with PG: +Cuebot emits two notifications, each `pg_notify` in the **same transaction** as the PG +write it describes (delivered iff that write commits no partial-failure window): -- **Booked-counter recompute (every 2 min)**: rebuilds `int_cores` / `int_gpus` for - every scheduler-managed show from `SUM(proc)`. Dual-writes to PG accounting tables - (so CueGUI stays fresh) and to Redis (guarded by an `acct:seq` CAS). -- **Limit reseed (every 5 min)**: copies limit fields (`size`, `burst`, - `int_min_cores`, `int_max_cores`, priorities) from PG to Redis. Catches Cuebot admin - operations that don't go through the `afterCommit` hook. -- **Bootstrap reseed (blocking at startup)**: the booking pipeline does not start - until both reseeds have populated Redis end-to-end. +- `acct_release`: a per-proc release delta on `procDestroyed` for scheduler-managed + shows. The scheduler decrements the three vertices. +- `acct_limit_change`: an enforced cap change (subscription burst, folder/job max + cores/gpus) from a cueadmin operation. The scheduler updates the cap in the store. -Each reseed compares-and-swaps against the `acct:seq` monotonic counter to detect -hot-path mutations between the SQL read and the Redis write, preventing silent loss -of concurrent bookings. +The scheduler listens on a dedicated connection and reconnects on drop; anything missed +during the gap is healed by the backstop loops below. -#### Cuebot integration +#### Backstop loops + +- **Recompute (every 15 s)**: overwrites the store's booked counters from `SUM(proc)`, + carrying each key's in-flight `pending` delta forward so a just-booked frame is never + erased (the one way an absolute overwrite could under-count and over-book). Also + rewrites the PG accounting tables for CueGUI. **No CAS, no retry** the live store is + the primary record. +- **Limit reseed (every 5 min)**: re-reads the enforced caps from PG into the store + the backstop for any missed `acct_limit_change`. +- **Bootstrap reseed (blocking at startup)**: seeds caps then counters from PG before + the pipeline accepts work. The store is the only copy, so this gate is mandatory. -`ProcDaoJdbc.unbookProc` checks `b_scheduler_managed` on every release. For -scheduler-managed shows it only `DELETE`s the proc row transactionally; on -`afterCommit` it publishes the release delta to Redis via -`LettuceAccountingRedisPublisher`. Publish failures are logged at WARN and swallowed -the recompute loop heals the missing decrement on the next cycle. +#### Cuebot integration -A startup guardrail in Cuebot logs a loud WARN and exposes -`cuebot_redis_publish_misconfigured` if any show is scheduler-managed but -`accounting.redis.enabled=false`, since that combination silently over-counts. +`ProcDaoJdbc.procDestroyed` checks `b_scheduler_managed` on every release. For +scheduler-managed shows it `DELETE`s the proc row and issues +`AccountingNotifier.notifyRelease` in the same transaction. A kill-switch property +`accounting.notify.enabled` (default true) disables the notifications; flag-off degrades +to recompute-only, which is the **safe** direction (no decrements → counters read high → +under-book → healed by recompute), so there is no startup guardrail only a WARN and a +`cuebot_accounting_notify_disabled` metric for ops visibility. #### Source files | Path | Purpose | |---|---| -| `accounting/mod.rs` | Module root; orchestrates booking, recompute, limit reseed, bootstrap | -| `accounting/redis_client.rs` | Redis connection management and Lua script wiring | -| `accounting/lua.rs` | Booking + force-rollback Lua sources | -| `accounting/recompute.rs` | 2-min `SUM(proc)` → PG + Redis dual write | -| `accounting/limit_reseed.rs` | 5-min accounting tables → Redis | +| `accounting/mod.rs` | Module root; `apply_booking` / `confirm_booking` / `rollback_booking` facade | +| `accounting/store.rs` | In-memory counters + caps; the locked atomic `book` / `confirm` / `rollback` | +| `accounting/listener.rs` | `PgListener` on `acct_release` + `acct_limit_change` | +| `accounting/recompute.rs` | 15 s `SUM(proc)` → PG tables + store overwrite with pending carry-forward | +| `accounting/limit_reseed.rs` | 5 min caps → store | | `accounting/bootstrap.rs` | Blocking startup reseed | -| `accounting/managed_shows.rs` | Cached lookup of scheduler-managed shows | +| `accounting/managed_shows.rs` | Cached lookup of scheduler-managed shows + managed-flip seed | | `accounting/booking_delta.rs` | Per-booking delta carried through the dispatch pipeline | -| `accounting/dao.rs` | PG accounting-table queries used by the reseeds | -| `accounting/error.rs` | Typed errors for the accounting layer | +| `accounting/dao.rs` | PG queries used by the reseeds | +| `accounting/error.rs` | `AccountingError::LimitExceeded` | -**For the full design** source-of-truth model, schema, `acct:seq` CAS protocol, -failure modes, drift bounds, and operator workflow see the -[Redis Accounting Reference](/docs/developer-guide/redis-accounting/). +**For the full design** source-of-truth model, the pending carry-forward and the +straddle-window race it closes, NOTIFY payload shapes, failure modes, the kill-switch, +and the N=1 assumption see the +[Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/). ### 7. Metrics and Observability @@ -886,26 +898,30 @@ for the full mapping from flag values to packing strategies and the workloads ea The default (Best-Fit cores, Worst-Fit memory) is the right choice for most render farms only flip `memory_saturation` if frame memory estimates are trustworthy. -### Redis Accounting +### Accounting ```yaml accounting: - redis_url: "redis://redis.internal:6379" - recompute_interval_seconds: 120 - limit_reseed_interval_seconds: 300 - redis_pool_size: 20 -``` - -- **`recompute_interval_seconds`** (default 120): how often to rebuild Redis booked - counters from `SUM(proc)`. Lower = tighter drift bound on PG accounting tables - (which CueGUI reads); higher = less DB load. -- **`limit_reseed_interval_seconds`** (default 300): how often to copy limit fields - (size, burst, caps) from PG to Redis. Lower = faster propagation of Cuebot admin - changes; higher = less DB load. -- **`redis_pool_size`** (default 20): connection pool size for the async Redis client. - -The matching Cuebot side requires `accounting.redis.enabled=true` and the same -`accounting.redis.host` / `accounting.redis.port` pointing at the same Redis instance. + recompute_interval: 15s + limit_reseed_interval: 300s + managed_shows_ttl: 30s +``` + +- **`recompute_interval`** (default 15s): how often to overwrite the in-memory booked + counters from `SUM(proc)` (carrying in-flight bookings forward) and refresh the PG + accounting tables CueGUI reads. This is the primary utilization backstop now that + releases arrive live via NOTIFY, so it runs frequently. Lower = tighter convergence; + higher = less DB load. +- **`limit_reseed_interval`** (default 300s): how often to re-read the enforced caps + (subscription burst, folder/job max cores/gpus) from PG into the store. The + `acct_limit_change` NOTIFY propagates cueadmin changes immediately; this is the + backstop for any missed notification. +- **`managed_shows_ttl`** (default 30s): how often the cache of `b_scheduler_managed` + shows is refreshed. + +The accounting store needs only PostgreSQL no Redis or other external store. The +matching Cuebot side emits the live feed when `accounting.notify.enabled=true` (default), +riding the existing PG connection. ## Distributed Operation @@ -947,14 +963,16 @@ to prevent two instances from owning the same clusters. - Database-level conflict resolution - Prevents double-booking even if permits overlap -**Shared Accounting (Redis)**: -- The Redis-backed accounting layer is the foundation that makes N-instance - deployment safe: per-show limits are enforced atomically across all schedulers via - the booking Lua script, and the `acct:seq` CAS guard serialises reseeds. -- Single-instance limitation today: the recompute and limit-reseed loops still need - leader election before > 1 scheduler can run safely; entry points in - `pipeline/entrypoint.rs` carry `// TODO` markers as a pin against rolling this out - without addressing it. +**In-Memory Accounting (N=1)**: +- The accounting store is in-process and **not shared**, so it assumes a single + scheduler instance: exactly one process owns the booked counters, which is what makes + the atomic check-and-increment and the absolute recompute correct. +- This is a deliberate trade for the structural elimination of the accounting-drift bug + class. Running more than one scheduler that could book the same show would let two + processes enforce against separate copies and jointly over-book a hard cap. Crossing + N>1 requires a shared/coordinated counter (or a non-overlapping partitioning scheme) + plus leader election for the recompute and limit-reseed loops none of which is in + place. See the [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/#the-n1-assumption-and-the-revisit-trigger-for-n1). ### Future Architecture (Planned) @@ -1003,8 +1021,7 @@ cargo test -p scheduler cargo test -p scheduler --features smoke-tests --test smoke_tests ``` -**Stress tests** (requires a migrated local Postgres plus a Docker daemon for -the throwaway Redis container; see the +**Stress tests** (requires a migrated local Postgres; see the [Scheduler Stress Testing](/docs/developer-guide/scheduler-stress-testing/) guide for tuning, CI behavior, and how to read the report): ```bash @@ -1040,7 +1057,7 @@ The scheduler: - Communicates with RQD via the same gRPC protocol - Produces the same proc/frame state transitions - Coexists with Cuebot via the per-show `b_scheduler_managed` flag see the - [Redis Accounting Reference](/docs/developer-guide/redis-accounting/) for the + [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/) for the ownership model ### Handing a show to the scheduler @@ -1203,11 +1220,15 @@ metrics::set_custom_gauge("custom_gauge", value); - **HostBookingStrategy**: Per-dimension flag controlling which heuristic (Best-Fit or Worst-Fit) the host cache uses when scanning the B-tree for a candidate. - **Scheduler-managed show**: A show with `b_scheduler_managed = true` dispatch and - hot-path accounting are owned by the Rust scheduler via Redis. + hot-path accounting are owned by the Rust scheduler via its in-memory store. - **Cuebot-managed show**: A show with `b_scheduler_managed = false` legacy behaviour; Cuebot dispatches and updates PG accounting transactionally. -- **`acct:seq`**: Monotonic Redis counter incremented by every mutating accounting - Lua script; the CAS guard for reseed loops. -- **Recompute loop**: 2-min job that rebuilds Redis booked counters from `SUM(proc)`. -- **Limit reseed**: 5-min job that copies limit fields (size/burst/caps) from PG to - Redis. +- **Pending delta**: the portion of a booked counter still in flight (proc row maybe + not yet visible to the recompute snapshot); carried forward across a recompute so the + absolute overwrite cannot erase a just-booked frame. +- **Recompute loop**: 15-second job that overwrites the in-memory booked counters from + `SUM(proc) + pending` and refreshes the PG accounting tables for CueGUI. +- **Limit reseed**: 5-min job that re-reads enforced caps (burst, folder/job max + cores/gpus) from PG into the store. +- **`acct_release` / `acct_limit_change`**: the two PG `LISTEN/NOTIFY` channels Cuebot + emits transactionally for releases and cap changes. diff --git a/docs/_docs/getting-started/deploying-scheduler.md b/docs/_docs/getting-started/deploying-scheduler.md index 8d3f4e8331..ca7ed14d51 100644 --- a/docs/_docs/getting-started/deploying-scheduler.md +++ b/docs/_docs/getting-started/deploying-scheduler.md @@ -37,7 +37,7 @@ To plan your installation of the Distributed Scheduler, consider the following: - **Memory**: Minimum 2GB RAM per scheduler instance (scales with number of hosts cached) - **CPU**: 2-4 cores recommended per instance - **Network**: Low-latency connection to the OpenCue database (same requirements as Cuebot) -- **Database**: PostgreSQL with the same schema as Cuebot (no additional tables required) +- **Database**: PostgreSQL with the same schema as Cuebot (no additional tables required). This is the **only** datastore the scheduler needs no Redis or other external store. Per-show resource accounting is held in memory and kept fresh by a PostgreSQL `LISTEN/NOTIFY` feed from Cuebot (see the [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/)) ## Architecture Overview diff --git a/rust/crates/scheduler/Cargo.toml b/rust/crates/scheduler/Cargo.toml index 77936fb758..1d8e9dc678 100644 --- a/rust/crates/scheduler/Cargo.toml +++ b/rust/crates/scheduler/Cargo.toml @@ -60,15 +60,12 @@ time = { workspace = true } axum = "0.7" tower-http = { version = "0.5", features = ["trace"] } urlencoding = "2.1" -redis = { version = "0.27", features = ["tokio-comp", "script", "connection-manager"] } [features] default = [] smoke-tests = [] -redis-tests = [] # End-to-end booking + accounting stress/benchmark suite (tests/stress_tests.rs). -# Requires a migrated local Postgres (docker compose up -d flyway) and Docker for -# the throwaway Redis container. +# Requires a migrated local Postgres (docker compose up -d flyway). stress-tests = [] [dev-dependencies] @@ -77,5 +74,3 @@ tracing-test = "0.2" serial_test = "3.0" rand = "0.8" proptest = "1.5" -testcontainers = "0.23" -testcontainers-modules = { version = "0.11", features = ["redis"] } diff --git a/rust/crates/scheduler/src/accounting/booking_delta.rs b/rust/crates/scheduler/src/accounting/booking_delta.rs index e09c5284b7..02d1c7cb89 100644 --- a/rust/crates/scheduler/src/accounting/booking_delta.rs +++ b/rust/crates/scheduler/src/accounting/booking_delta.rs @@ -12,107 +12,19 @@ use uuid::Uuid; -/// A single booking-or-release applied atomically to the five Redis accounting hashes. +/// A single booking-or-release applied to the in-memory accounting store. Carries only the +/// three enforced vertices (subscription/folder/job); the legacy Lua incremented layer and +/// point too but never read them, so they are not tracked. /// -/// Field shape matches the Cuebot release publisher (`LettuceAccountingRedisPublisher`), -/// so both sides of the booking lifecycle converge on the same `acct:*` keys. +/// `core_delta`/`gpu_delta` are signed: positive on a booking, negative on a release. +/// Rollback and release are expressed as the store subtracting/adding these directly, so +/// no separate negation helper is needed. #[derive(Debug, Clone)] pub struct BookingDelta { pub show_id: Uuid, pub alloc_id: Uuid, pub folder_id: Uuid, pub job_id: Uuid, - pub layer_id: Uuid, - pub dept_id: Uuid, pub core_delta: i64, pub gpu_delta: i32, } - -impl BookingDelta { - /// Returns the same delta with every numeric component negated. - pub fn negated(&self) -> BookingDelta { - BookingDelta { - show_id: self.show_id, - alloc_id: self.alloc_id, - folder_id: self.folder_id, - job_id: self.job_id, - layer_id: self.layer_id, - dept_id: self.dept_id, - core_delta: -self.core_delta, - gpu_delta: -self.gpu_delta, - } - } - - pub fn sub_key(&self) -> String { - format!("acct:sub:{}:{}", self.show_id, self.alloc_id) - } - - pub fn folder_key(&self) -> String { - format!("acct:folder:{}", self.folder_id) - } - - pub fn job_key(&self) -> String { - format!("acct:job:{}", self.job_id) - } - - pub fn layer_key(&self) -> String { - format!("acct:layer:{}", self.layer_id) - } - - pub fn point_key(&self) -> String { - format!("acct:point:{}:{}", self.dept_id, self.show_id) - } -} - -pub const SEQ_KEY: &str = "acct:seq"; - -#[cfg(test)] -mod tests { - use super::*; - - fn fixture() -> BookingDelta { - BookingDelta { - show_id: Uuid::nil(), - alloc_id: Uuid::nil(), - folder_id: Uuid::nil(), - job_id: Uuid::nil(), - layer_id: Uuid::nil(), - dept_id: Uuid::nil(), - core_delta: 100, - gpu_delta: 1, - } - } - - #[test] - fn negated_inverts_only_numeric_fields() { - let d = fixture(); - let n = d.negated(); - assert_eq!(n.core_delta, -100); - assert_eq!(n.gpu_delta, -1); - assert_eq!(n.show_id, d.show_id); - } - - #[test] - fn keys_match_cuebot_publisher_format() { - // Format must match LettuceAccountingRedisPublisher.evalRelease - both sides - // mutate the same hashes. - let d = fixture(); - assert_eq!( - d.sub_key(), - "acct:sub:00000000-0000-0000-0000-000000000000:00000000-0000-0000-0000-000000000000" - ); - assert_eq!( - d.folder_key(), - "acct:folder:00000000-0000-0000-0000-000000000000" - ); - assert_eq!(d.job_key(), "acct:job:00000000-0000-0000-0000-000000000000"); - assert_eq!( - d.layer_key(), - "acct:layer:00000000-0000-0000-0000-000000000000" - ); - assert_eq!( - d.point_key(), - "acct:point:00000000-0000-0000-0000-000000000000:00000000-0000-0000-0000-000000000000" - ); - } -} diff --git a/rust/crates/scheduler/src/accounting/bootstrap.rs b/rust/crates/scheduler/src/accounting/bootstrap.rs index daa19baca5..71c0cc8901 100644 --- a/rust/crates/scheduler/src/accounting/bootstrap.rs +++ b/rust/crates/scheduler/src/accounting/bootstrap.rs @@ -10,10 +10,6 @@ // or implied. See the License for the specific language governing permissions and limitations under // the License. -// TODO: multi-scheduler race - first-mover-wins reseed semantics deferred (design §5). -// Today's single-scheduler model means there is no concurrent bootstrap; when leader -// election lands, only the leader should call run_blocking_reseed at startup. - use miette::Result; use tracing::info; @@ -21,18 +17,18 @@ use crate::accounting::limit_reseed; use crate::accounting::recompute; use crate::accounting::AccountingService; -/// Blocking startup reseed. Populates Redis from PG before the scheduler accepts work: -/// limit fields (subscription burst, folder/job/point caps) first, then booked counters -/// (`int_cores`/`int_gpus`) from `SUM(proc)`. Idempotent - `HSET` overwrites, and the -/// limit-fields and booked-counter-fields are disjoint so a partial Redis state from a -/// previous run is safely replaced. +/// Blocking startup seed. Populates the in-memory store from PG before the scheduler +/// accepts work: the enforced caps (subscription burst, folder/job max cores+gpus) first, +/// then booked counters from `SUM(proc)`. The store is the only copy of this state, so +/// this gate is mandatory - dispatching before it completes would book every hard cap wide +/// open against empty counters. /// /// Must complete successfully before `recompute::spawn_loop` / `limit_reseed::spawn_loop` -/// are spawned and before any dispatcher accepts a host report - see design §4.3 row 5. +/// / `listener::spawn_loop` are spawned and before any dispatcher accepts a host report. pub async fn run_blocking_reseed(service: &AccountingService) -> Result<()> { - info!("Bootstrap reseed: starting (limits, then booked counters)"); + info!("Bootstrap seed: starting (caps, then booked counters)"); limit_reseed::reseed_once(service).await?; - recompute::reseed_redis_once(service).await?; - info!("Bootstrap reseed: complete"); + recompute::reseed_store_once(service).await?; + info!("Bootstrap seed: complete"); Ok(()) } diff --git a/rust/crates/scheduler/src/accounting/dao.rs b/rust/crates/scheduler/src/accounting/dao.rs index e16c1f7d75..413930a11e 100644 --- a/rust/crates/scheduler/src/accounting/dao.rs +++ b/rust/crates/scheduler/src/accounting/dao.rs @@ -19,58 +19,50 @@ use uuid::Uuid; use crate::dao::helpers::parse_uuid; use crate::pgpool::connection_pool; -/// Single PG-side DAO for the Redis-backed accounting module. Owns: +/// Single PG-side DAO for the in-memory accounting module. Owns: /// - the managed-shows lookup that drives the in-process cache, -/// - the unified SUM(proc) snapshot used by the booked-counter recompute, -/// - per-table limit-field reads used by the limit reseed (subscription burst/size, -/// folder/job caps, point min/max). +/// - the `SUM(proc)` snapshot used by the booked-counter recompute, +/// - per-table reads of the enforced caps used by the limit reseed (subscription burst, +/// folder/job max cores+gpus). pub struct AccountingDao { connection_pool: Arc>, } -/// One row from the unified SUM(proc) snapshot used by the recompute loop. +/// One row from the `SUM(proc)` snapshot used by the recompute loop, grouped by the three +/// enforced vertices (subscription via show+alloc, folder, job). #[derive(Debug, Clone)] pub struct BookedSnapshotRow { pub show_id: Uuid, pub alloc_id: Uuid, pub folder_id: Uuid, pub job_id: Uuid, - pub layer_id: Uuid, - pub dept_id: Uuid, pub cores: i64, pub gpus: i64, } -/// The full universe of enumerable accounting keys for scheduler-managed shows, -/// used by the booked-counter recompute to seed zero baselines. A key present here -/// but absent from the `SUM(proc)` snapshot has no procs, so its booked counter must -/// be reset to 0 - otherwise a counter that drifted stale-high and then drained to -/// zero procs would never converge (the snapshot only returns keys that still have -/// procs). Layer keys are intentionally absent: layers have no limit table to -/// enumerate from, and the booking Lua never reads the layer counter (it is -/// `HINCRBY`-only), so residual layer drift is cosmetic. See `recompute.rs`. +/// The full universe of enumerable accounting keys for scheduler-managed shows, used by +/// the recompute to seed zero baselines. A key present here but absent from the +/// `SUM(proc)` snapshot has no procs, so its booked counter must reset to 0 - otherwise a +/// counter that drifted stale-high and then drained to zero procs would never converge +/// (the snapshot only returns keys that still have procs). #[derive(Debug, Clone, Default)] pub struct BaselineKeys { pub subs: Vec<(Uuid, Uuid)>, pub folders: Vec, pub jobs: Vec, - pub points: Vec<(Uuid, Uuid)>, } #[derive(Debug, Clone)] pub struct SubscriptionLimitsRow { pub show_id: Uuid, pub alloc_id: Uuid, - pub size: i64, pub burst: i64, } #[derive(Debug, Clone)] pub struct FolderLimitsRow { pub folder_id: Uuid, - pub min_cores: i64, pub max_cores: i64, - pub min_gpus: i64, pub max_gpus: i64, } @@ -79,37 +71,22 @@ pub struct JobLimitsRow { pub job_id: Uuid, pub max_cores: i64, pub max_gpus: i64, - pub priority: i64, -} - -#[derive(Debug, Clone)] -pub struct PointLimitsRow { - pub dept_id: Uuid, - pub show_id: Uuid, - pub min_cores: i64, - pub min_gpus: i64, } static QUERY_MANAGED_SHOW_IDS: &str = r#" SELECT pk_show FROM show WHERE b_scheduler_managed = true "#; -/// One snapshot row per (show, alloc, folder, job, layer, dept) tuple of bookings. -/// Filtered to scheduler-managed shows; excludes local procs. Joins through host to -/// derive `pk_alloc` per the same path Cuebot uses. +/// One snapshot row per (show, alloc, folder, job) tuple of bookings. Filtered to +/// scheduler-managed shows; excludes local procs. Joins through host to derive `pk_alloc` +/// per the same path Cuebot uses. /// -/// Why `b_local=false`: the Redis hot path is naturally non-local-only - the Rust -/// scheduler never books local procs, and Cuebot's `procDestroyed` skips Redis -/// publishing on the local branch. Recompute must match that to avoid creating -/// drift the hot path can't repair. Consequence: for a scheduler-managed show with -/// locals, `acct:layer:{layer}.int_cores` under-counts vs `layer_resource.int_cores` -/// in PG (Cuegui's view still includes locals via the existing PG recompute that -/// doesn't filter). A layer "full" with locals could still accept non-local -/// bookings via the Lua cap check - acceptable per design §4.4 because local -/// dispatch is rare and intentionally accounted for via `job_resource.int_local_cores` -/// rather than the show-wide caps. -static QUERY_BOOKED_BY_5DIM: &str = r#" - SELECT j.pk_show, h.pk_alloc, j.pk_folder, p.pk_job, p.pk_layer, j.pk_dept, +/// Why `b_local=false`: the scheduler never books local procs, and Cuebot's `procDestroyed` +/// skips the release NOTIFY on the local branch, so recompute must match to avoid creating +/// drift the hot path can't repair. Local dispatch is accounted for separately via +/// `job_resource.int_local_cores`, not the show-wide caps the scheduler enforces. +static QUERY_BOOKED: &str = r#" + SELECT j.pk_show, h.pk_alloc, j.pk_folder, p.pk_job, COALESCE(SUM(p.int_cores_reserved), 0)::bigint as cores, COALESCE(SUM(p.int_gpus_reserved), 0)::bigint as gpus FROM proc p @@ -117,40 +94,42 @@ static QUERY_BOOKED_BY_5DIM: &str = r#" JOIN job j ON j.pk_job = p.pk_job AND j.str_state <> 'FINISHED' JOIN show s ON s.pk_show = j.pk_show AND s.b_scheduler_managed = true WHERE p.b_local = false - GROUP BY j.pk_show, h.pk_alloc, j.pk_folder, p.pk_job, p.pk_layer, j.pk_dept + GROUP BY j.pk_show, h.pk_alloc, j.pk_folder, p.pk_job +"#; + +/// Same shape as `QUERY_BOOKED` but scoped to one show, for the managed-flip booked-counter +/// seed. Not gated on `b_scheduler_managed` (the caller already knows the show is flipping). +static QUERY_BOOKED_FOR_SHOW: &str = r#" + SELECT h.pk_alloc, j.pk_folder, p.pk_job, + COALESCE(SUM(p.int_cores_reserved), 0)::bigint as cores, + COALESCE(SUM(p.int_gpus_reserved), 0)::bigint as gpus + FROM proc p + JOIN host h ON h.pk_host = p.pk_host + JOIN job j ON j.pk_job = p.pk_job AND j.str_state <> 'FINISHED' + WHERE p.b_local = false AND j.pk_show = $1 + GROUP BY h.pk_alloc, j.pk_folder, p.pk_job "#; static QUERY_SUBSCRIPTION_LIMITS: &str = r#" - SELECT s.pk_show, s.pk_alloc, s.int_size, s.int_burst + SELECT s.pk_show, s.pk_alloc, s.int_burst FROM subscription s JOIN show sh ON sh.pk_show = s.pk_show AND sh.b_scheduler_managed = true "#; static QUERY_FOLDER_LIMITS: &str = r#" - SELECT fr.pk_folder, - fr.int_min_cores, fr.int_max_cores, - fr.int_min_gpus, fr.int_max_gpus + SELECT fr.pk_folder, fr.int_max_cores, fr.int_max_gpus FROM folder_resource fr JOIN folder f ON f.pk_folder = fr.pk_folder JOIN show s ON s.pk_show = f.pk_show AND s.b_scheduler_managed = true "#; static QUERY_JOB_LIMITS: &str = r#" - SELECT jr.pk_job, jr.int_max_cores, jr.int_max_gpus, jr.int_priority + SELECT jr.pk_job, jr.int_max_cores, jr.int_max_gpus FROM job_resource jr JOIN job j ON j.pk_job = jr.pk_job AND j.str_state <> 'FINISHED' JOIN show s ON s.pk_show = j.pk_show AND s.b_scheduler_managed = true "#; -// `point` has `int_min_cores`/`int_min_gpus` but no `int_max_*` columns - points enforce a -// floor, not a ceiling. The design doc §2.3 lists `int_max_cores` in the `acct:point` -// HASH for symmetry, but the schema doesn't back it; we don't populate or read it. -static QUERY_POINT_LIMITS: &str = r#" - SELECT pt.pk_dept, pt.pk_show, pt.int_min_cores, pt.int_min_gpus - FROM point pt - JOIN show s ON s.pk_show = pt.pk_show AND s.b_scheduler_managed = true -"#; - impl AccountingDao { pub async fn new() -> Result { let pool = connection_pool().await.into_diagnostic()?; @@ -179,12 +158,10 @@ impl AccountingDao { pk_alloc: String, pk_folder: String, pk_job: String, - pk_layer: String, - pk_dept: String, cores: i64, gpus: i64, } - let rows: Vec = sqlx::query_as(QUERY_BOOKED_BY_5DIM) + let rows: Vec = sqlx::query_as(QUERY_BOOKED) .fetch_all(self.connection_pool.as_ref()) .await .into_diagnostic() @@ -197,32 +174,60 @@ impl AccountingDao { alloc_id: parse_uuid(&r.pk_alloc), folder_id: parse_uuid(&r.pk_folder), job_id: parse_uuid(&r.pk_job), - layer_id: parse_uuid(&r.pk_layer), - dept_id: parse_uuid(&r.pk_dept), cores: r.cores, gpus: r.gpus, }) .collect()) } - /// Enumerate the full set of sub/folder/job/point keys for scheduler-managed - /// shows, by projecting the key tuples out of the four limit queries. Reusing the - /// limit queries (rather than dedicated key-only SELECTs) guarantees this key - /// universe is grain-consistent with both the limit reseed and the booked snapshot - /// (same `b_scheduler_managed` / `str_state <> 'FINISHED'` filters). Layer keys are - /// not enumerable from a limit table, so they are absent here by design. + /// Booked snapshot for a single show (managed-flip seed). `show_id` fills the row's + /// `show_id` since the query keys it by parameter rather than selecting `pk_show`. + pub async fn query_booked_snapshot_for_show( + &self, + show_id: Uuid, + ) -> Result> { + #[derive(sqlx::FromRow)] + struct Row { + pk_alloc: String, + pk_folder: String, + pk_job: String, + cores: i64, + gpus: i64, + } + let rows: Vec = sqlx::query_as(QUERY_BOOKED_FOR_SHOW) + .bind(show_id.to_string()) + .fetch_all(self.connection_pool.as_ref()) + .await + .into_diagnostic() + .wrap_err("Failed to query booked snapshot for show")?; + Ok(rows + .into_iter() + .map(|r| BookedSnapshotRow { + show_id, + alloc_id: parse_uuid(&r.pk_alloc), + folder_id: parse_uuid(&r.pk_folder), + job_id: parse_uuid(&r.pk_job), + cores: r.cores, + gpus: r.gpus, + }) + .collect()) + } + + /// Enumerate the full set of sub/folder/job keys for scheduler-managed shows, by + /// projecting the key tuples out of the three limit queries. Reusing the limit queries + /// (rather than dedicated key-only SELECTs) guarantees this key universe is + /// grain-consistent with both the limit reseed and the booked snapshot (same + /// `b_scheduler_managed` / `str_state <> 'FINISHED'` filters). pub async fn query_booked_baseline_keys(&self) -> Result { - let (subs, folders, jobs, points) = tokio::try_join!( + let (subs, folders, jobs) = tokio::try_join!( self.query_subscription_limits(), self.query_folder_limits(), self.query_job_limits(), - self.query_point_limits(), )?; Ok(BaselineKeys { subs: subs.iter().map(|r| (r.show_id, r.alloc_id)).collect(), folders: folders.iter().map(|r| r.folder_id).collect(), jobs: jobs.iter().map(|r| r.job_id).collect(), - points: points.iter().map(|r| (r.dept_id, r.show_id)).collect(), }) } @@ -231,7 +236,6 @@ impl AccountingDao { struct Row { pk_show: String, pk_alloc: String, - int_size: i64, int_burst: i64, } let rows: Vec = sqlx::query_as(QUERY_SUBSCRIPTION_LIMITS) @@ -244,20 +248,17 @@ impl AccountingDao { .map(|r| SubscriptionLimitsRow { show_id: parse_uuid(&r.pk_show), alloc_id: parse_uuid(&r.pk_alloc), - size: r.int_size, burst: r.int_burst, }) .collect()) } pub async fn query_folder_limits(&self) -> Result> { - // folder_resource columns are INT (i32); widen to i64 for Redis storage. + // folder_resource columns are INT (i32); widen to i64 for store arithmetic. #[derive(sqlx::FromRow)] struct Row { pk_folder: String, - int_min_cores: i32, int_max_cores: i32, - int_min_gpus: i32, int_max_gpus: i32, } let rows: Vec = sqlx::query_as(QUERY_FOLDER_LIMITS) @@ -269,22 +270,19 @@ impl AccountingDao { .into_iter() .map(|r| FolderLimitsRow { folder_id: parse_uuid(&r.pk_folder), - min_cores: i64::from(r.int_min_cores), max_cores: i64::from(r.int_max_cores), - min_gpus: i64::from(r.int_min_gpus), max_gpus: i64::from(r.int_max_gpus), }) .collect()) } pub async fn query_job_limits(&self) -> Result> { - // job_resource columns are INT (i32); widen to i64 for Redis storage. + // job_resource columns are INT (i32); widen to i64 for store arithmetic. #[derive(sqlx::FromRow)] struct Row { pk_job: String, int_max_cores: i32, int_max_gpus: i32, - int_priority: i32, } let rows: Vec = sqlx::query_as(QUERY_JOB_LIMITS) .fetch_all(self.connection_pool.as_ref()) @@ -297,32 +295,6 @@ impl AccountingDao { job_id: parse_uuid(&r.pk_job), max_cores: i64::from(r.int_max_cores), max_gpus: i64::from(r.int_max_gpus), - priority: i64::from(r.int_priority), - }) - .collect()) - } - - pub async fn query_point_limits(&self) -> Result> { - // point columns are INT (i32); widen to i64 for Redis storage. - #[derive(sqlx::FromRow)] - struct Row { - pk_dept: String, - pk_show: String, - int_min_cores: i32, - int_min_gpus: i32, - } - let rows: Vec = sqlx::query_as(QUERY_POINT_LIMITS) - .fetch_all(self.connection_pool.as_ref()) - .await - .into_diagnostic() - .wrap_err("Failed to query point limits")?; - Ok(rows - .into_iter() - .map(|r| PointLimitsRow { - dept_id: parse_uuid(&r.pk_dept), - show_id: parse_uuid(&r.pk_show), - min_cores: i64::from(r.int_min_cores), - min_gpus: i64::from(r.int_min_gpus), }) .collect()) } diff --git a/rust/crates/scheduler/src/accounting/error.rs b/rust/crates/scheduler/src/accounting/error.rs index 42af9ef342..b28cf70c28 100644 --- a/rust/crates/scheduler/src/accounting/error.rs +++ b/rust/crates/scheduler/src/accounting/error.rs @@ -12,18 +12,12 @@ use thiserror::Error; -/// Errors from the Redis-backed accounting service. +/// Error from the in-memory accounting service. /// -/// The booking hot path (`apply_booking`) can fail in three meaningful ways: -/// - `LimitExceeded`: the Lua check rejected the booking because a hard cap -/// (subscription burst, folder/job `int_max_cores`, or folder/job -/// `int_max_gpus`) would be exceeded. Carries the offending table, the current -/// counter value, and the limit - used to build user-facing error messages. -/// - `Unavailable`: Redis is unreachable or in a state where bookings can't safely -/// proceed (empty after a restart, before bootstrap reseed has run). The dispatcher -/// maps this to an idle-cycle equivalent - design §4.3 row 5. -/// - `Unexpected`: anything else (Lua syntax error, malformed return, etc.). Surfaced -/// for diagnostics; same dispatch consequence as `Unavailable`. +/// The booking hot path (`apply_booking`) has exactly one failure mode now that the store +/// is in process and cannot be unreachable: `LimitExceeded`, when a hard cap (subscription +/// burst, folder/job `int_max_cores`/`int_max_gpus`) would be exceeded. Carries the +/// offending table, the current counter value, and the limit for user-facing messages. #[derive(Error, Debug)] pub enum AccountingError { #[error("limit exceeded on {table}: current={current} limit={limit}")] @@ -32,30 +26,4 @@ pub enum AccountingError { current: i64, limit: i64, }, - - #[error("redis unavailable: {0}")] - Unavailable(String), - - /// Raised when a CAS-guarded reseed exhausts its retry budget. The periodic - /// reseed loops downgrade this to a warn-log (hot-path writes keep Redis fresh, - /// per design §2.4), but the bootstrap reseed surfaces it as a startup gate so a - /// scheduler never begins booking against an unseeded Redis. Carries the number of - /// attempts made (`cas_max_retries + 1`) for diagnostics. - #[error( - "CAS contention exceeded retry budget after {attempts} attempts; reseed cycle skipped" - )] - CasContentionExceeded { attempts: u32 }, - - #[error("accounting redis error: {0}")] - Unexpected(String), -} - -impl From for AccountingError { - fn from(err: redis::RedisError) -> Self { - if err.is_connection_refusal() || err.is_io_error() || err.is_timeout() { - AccountingError::Unavailable(err.to_string()) - } else { - AccountingError::Unexpected(err.to_string()) - } - } } diff --git a/rust/crates/scheduler/src/accounting/limit_reseed.rs b/rust/crates/scheduler/src/accounting/limit_reseed.rs index 7397726f2d..0d83d09355 100644 --- a/rust/crates/scheduler/src/accounting/limit_reseed.rs +++ b/rust/crates/scheduler/src/accounting/limit_reseed.rs @@ -10,29 +10,31 @@ // or implied. See the License for the specific language governing permissions and limitations under // the License. -//! Limit-field reseed loop. Every `CONFIG.accounting.limit_reseed_interval`, -//! reads limit fields from PG (subscription burst/size, folder/job caps, point min/max) -//! and writes them to Redis with unconditional `HSET`s. `HSET` overwrites only the -//! specified fields, leaving booked counters (`int_cores`/`int_gpus`) untouched, so this -//! needs no `acct:seq` CAS guard (limit fields are written only here, never by the -//! booking hot path) - which also stops force-rollback churn from starving the reseed and -//! leaving a freshly-managed show's subscription `burst` unseeded (== 0 == "reject all"). +//! Limit-field reseed loop - the cap-change backstop. Every +//! `CONFIG.accounting.limit_reseed_interval`, reads the enforced caps from PG +//! (subscription burst, folder/job `int_max_cores`/`int_max_gpus`) and writes them into +//! the in-memory store. The live `acct_limit_change` NOTIFY propagates cueadmin changes +//! immediately; this loop heals any missed notification within one interval. +//! +//! Only the caps the booking check reads are seeded (subscription burst, folder/job max +//! cores+gpus). Size, min-cores, priority and point caps are not enforced by the +//! scheduler, so they live only in PG (CueGUI reads them there, unchanged). use std::panic::AssertUnwindSafe; use std::sync::Arc; use futures::FutureExt; -use miette::{IntoDiagnostic, Result, WrapErr}; +use miette::Result; use tokio::time; use tracing::{error, info, warn}; +use uuid::Uuid; use crate::accounting::dao::{ - AccountingDao, FolderLimitsRow, JobLimitsRow, PointLimitsRow, SubscriptionLimitsRow, + AccountingDao, FolderLimitsRow, JobLimitsRow, SubscriptionLimitsRow, }; -use crate::accounting::redis_client::{RedisAccounting, ReseedOp}; +use crate::accounting::store::{centicores_to_cores, centicores_to_cores_cap, Store}; use crate::accounting::AccountingService; use crate::config::CONFIG; -use crate::models::CoreSize; pub fn spawn_loop(service: Arc) { tokio::spawn(async move { @@ -55,43 +57,27 @@ pub fn spawn_loop(service: Arc) { }); } -/// One reseed pass: snapshot all four limit tables, flatten to ops, write them with -/// unconditional `HSET`s (no `acct:seq` CAS). Shared by the periodic loop, the bootstrap, -/// and the synchronous seed performed when a show first becomes scheduler-managed -/// (`ManagedShowsCache`). -/// -/// No CAS guard because limit fields are written only here and are disjoint from the -/// booked counters the hot path mutates - so this can never clobber a booking, and a -/// concurrent booking can never clobber it. The previous CAS-guarded variant could be -/// starved into skipping by force-rollback churn bumping `acct:seq`; an unconditional -/// write always lands. See `RedisAccounting::reseed_unconditional`. -pub async fn reseed_limits(redis: &RedisAccounting, dao: &AccountingDao) -> Result<()> { - let (subs, folders, jobs, points) = tokio::try_join!( +/// One reseed pass: snapshot the limit tables and write the enforced caps into the store. +/// Shared by the periodic loop, the bootstrap, and the synchronous seed performed when a +/// show first becomes scheduler-managed (`ManagedShowsCache`). +pub async fn reseed_limits(store: &Store, dao: &AccountingDao) -> Result<()> { + let (subs, folders, jobs) = tokio::try_join!( dao.query_subscription_limits(), dao.query_folder_limits(), dao.query_job_limits(), - dao.query_point_limits(), )?; - let ops: Vec = subscription_ops(&subs) - .chain(folder_ops(&folders)) - .chain(job_ops(&jobs)) - .chain(point_ops(&points)) - .collect(); - - redis - .reseed_unconditional(&ops) - .await - .into_diagnostic() - .wrap_err("HSET reseed for limit fields failed")?; + store.set_caps( + subs.iter().map(sub_cap), + folders.iter().map(folder_cap), + jobs.iter().map(job_cap), + ); info!( - "Limit reseed applied: {} ops (subs={} folders={} jobs={} points={})", - ops.len(), + "Limit reseed applied: subs={} folders={} jobs={}", subs.len(), folders.len(), jobs.len(), - points.len(), ); Ok(()) } @@ -99,162 +85,56 @@ pub async fn reseed_limits(redis: &RedisAccounting, dao: &AccountingDao) -> Resu /// Thin wrapper over [`reseed_limits`] taking the full service. Used by the periodic loop /// and the bootstrap. pub async fn reseed_once(service: &AccountingService) -> Result<()> { - reseed_limits(service.redis(), service.dao()).await + reseed_limits(service.store(), service.dao()).await } -/// Flatten subscription limit rows into per-field `ReseedOp`s (`size`, `burst`). -/// PG centicores → Redis cores via `CoreSize::from_multiplied`; the type carries -/// the unit through the conversion. Subscription caps never use the `-1` sentinel. -fn subscription_ops(rows: &[SubscriptionLimitsRow]) -> impl Iterator + '_ { - rows.iter().flat_map(|r| { - let key = format!("acct:sub:{}:{}", r.show_id, r.alloc_id); - [ - ReseedOp { - key: key.clone(), - field: "size", - value: i64::from(CoreSize::from_multiplied(r.size).value()), - }, - ReseedOp { - key, - field: "burst", - value: i64::from(CoreSize::from_multiplied(r.burst).value()), - }, - ] - }) +/// `(show, alloc, burst_cores)`. Burst is PG centicores → cores; never the `-1` sentinel. +fn sub_cap(r: &SubscriptionLimitsRow) -> (Uuid, Uuid, i64) { + (r.show_id, r.alloc_id, centicores_to_cores(r.burst)) } -/// Flatten folder limit rows into the four cap fields per row. `int_max_cores` uses -/// `from_multiplied_cap` to preserve the `-1` "unlimited" sentinel; GPU fields -/// pass through unconverted. -fn folder_ops(rows: &[FolderLimitsRow]) -> impl Iterator + '_ { - rows.iter().flat_map(|r| { - let key = format!("acct:folder:{}", r.folder_id); - [ - ReseedOp { - key: key.clone(), - field: "int_min_cores", - value: i64::from(CoreSize::from_multiplied(r.min_cores).value()), - }, - ReseedOp { - key: key.clone(), - field: "int_max_cores", - value: i64::from(CoreSize::from_multiplied_cap(r.max_cores).value()), - }, - ReseedOp { - key: key.clone(), - field: "int_min_gpus", - value: r.min_gpus, - }, - ReseedOp { - key, - field: "int_max_gpus", - value: r.max_gpus, - }, - ] - }) +/// `(folder, max_cores, max_gpus)`. Cores preserve the `-1` unlimited sentinel; GPUs pass +/// through unconverted (their `-1` sentinel survives a verbatim copy). +fn folder_cap(r: &FolderLimitsRow) -> (Uuid, i64, i64) { + (r.folder_id, centicores_to_cores_cap(r.max_cores), r.max_gpus) } -/// Flatten job limit rows into the three cap fields per row. `int_max_cores` -/// uses `from_multiplied_cap` for the `-1` sentinel; `int_max_gpus` and -/// `int_priority` pass through unconverted. -fn job_ops(rows: &[JobLimitsRow]) -> impl Iterator + '_ { - rows.iter().flat_map(|r| { - let key = format!("acct:job:{}", r.job_id); - [ - ReseedOp { - key: key.clone(), - field: "int_max_cores", - value: i64::from(CoreSize::from_multiplied_cap(r.max_cores).value()), - }, - ReseedOp { - key: key.clone(), - field: "int_max_gpus", - value: r.max_gpus, - }, - ReseedOp { - key, - field: "int_priority", - value: r.priority, - }, - ] - }) -} - -/// Flatten point limit rows into the two floor fields per row. (Point has no -/// `int_max_*` columns in the schema; only minimums are surfaced.) `int_min_cores` -/// is a floor, never negative, so `from_multiplied` is sufficient. -fn point_ops(rows: &[PointLimitsRow]) -> impl Iterator + '_ { - rows.iter().flat_map(|r| { - let key = format!("acct:point:{}:{}", r.dept_id, r.show_id); - [ - ReseedOp { - key: key.clone(), - field: "int_min_cores", - value: i64::from(CoreSize::from_multiplied(r.min_cores).value()), - }, - ReseedOp { - key, - field: "int_min_gpus", - value: r.min_gpus, - }, - ] - }) +/// `(job, max_cores, max_gpus)`. Same conventions as [`folder_cap`]. +fn job_cap(r: &JobLimitsRow) -> (Uuid, i64, i64) { + (r.job_id, centicores_to_cores_cap(r.max_cores), r.max_gpus) } #[cfg(test)] mod tests { use super::*; - use uuid::Uuid; #[test] - fn subscription_emits_two_fields_in_cores() { - // PG centicores 900/1000 -> Redis cores 9/10. - let out: Vec = subscription_ops(&[SubscriptionLimitsRow { + fn sub_burst_converts_centicores_to_cores() { + let r = SubscriptionLimitsRow { show_id: Uuid::nil(), alloc_id: Uuid::nil(), - size: 900, burst: 1000, - }]) - .collect(); - assert_eq!(out.len(), 2); - assert!(out.iter().any(|o| o.field == "size" && o.value == 9)); - assert!(out.iter().any(|o| o.field == "burst" && o.value == 10)); + }; + assert_eq!(sub_cap(&r), (Uuid::nil(), Uuid::nil(), 10)); } #[test] - fn folder_converts_cores_passes_gpus_and_preserves_unlimited() { - // min_cores: 0 -> 0; max_cores: -1 (unlimited sentinel) -> -1; GPUs unchanged. - let out: Vec = folder_ops(&[FolderLimitsRow { + fn folder_preserves_unlimited_and_passes_gpus() { + let r = FolderLimitsRow { folder_id: Uuid::nil(), - min_cores: 0, max_cores: -1, - min_gpus: 0, max_gpus: -1, - }]) - .collect(); - assert_eq!(out.len(), 4); - let by_field: std::collections::HashMap<_, _> = - out.iter().map(|o| (o.field, o.value)).collect(); - assert_eq!(by_field["int_min_cores"], 0); - assert_eq!(by_field["int_max_cores"], -1); - assert_eq!(by_field["int_min_gpus"], 0); - assert_eq!(by_field["int_max_gpus"], -1); + }; + assert_eq!(folder_cap(&r), (Uuid::nil(), -1, -1)); } #[test] - fn folder_converts_positive_cap_to_cores() { - let out: Vec = folder_ops(&[FolderLimitsRow { - folder_id: Uuid::nil(), - min_cores: 500, + fn job_converts_positive_cap_to_cores() { + let r = JobLimitsRow { + job_id: Uuid::nil(), max_cores: 2000, - min_gpus: 0, max_gpus: 4, - }]) - .collect(); - let by_field: std::collections::HashMap<_, _> = - out.iter().map(|o| (o.field, o.value)).collect(); - assert_eq!(by_field["int_min_cores"], 5); - assert_eq!(by_field["int_max_cores"], 20); - assert_eq!(by_field["int_max_gpus"], 4); // GPUs unconverted. + }; + assert_eq!(job_cap(&r), (Uuid::nil(), 20, 4)); } } diff --git a/rust/crates/scheduler/src/accounting/listener.rs b/rust/crates/scheduler/src/accounting/listener.rs new file mode 100644 index 0000000000..ab4035f47a --- /dev/null +++ b/rust/crates/scheduler/src/accounting/listener.rs @@ -0,0 +1,239 @@ +// Copyright Contributors to the OpenCue Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under +// the License. + +//! PostgreSQL `LISTEN/NOTIFY` listener: the live, cross-process feed that keeps the +//! in-memory [`Store`] fresh between recompute passes. See +//! `docs/_docs/developer-guide/scheduler-accounting.md`. +//! +//! Cuebot emits two channels, each in the same transaction as the PG write it describes +//! (so a notification is delivered iff that write commits): +//! - `acct_release`: a per-proc release delta on `unbookProc` for scheduler-managed +//! shows. Cores/gpus are signed deltas (negative for a release). +//! - `acct_limit_change`: an enforced cap change from a cueadmin operation. +//! +//! Both are best-effort optimisations: a dropped notification (listener reconnecting) +//! only leaves the store reading high → under-book → healed by the next recompute / +//! limit reseed. Nothing here can over-book a hard cap. + +use std::sync::Arc; +use std::time::Duration; + +use serde::Deserialize; +use sqlx::postgres::PgListener; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +use crate::accounting::booking_delta::BookingDelta; +use crate::accounting::store::{LimitChange, Store}; +use crate::config::CONFIG; + +const RELEASE_CHANNEL: &str = "acct_release"; +const LIMIT_CHANGE_CHANNEL: &str = "acct_limit_change"; +const RECONNECT_BACKOFF: Duration = Duration::from_secs(5); + +/// Release delta payload on `acct_release`. Cores/gpus are signed deltas to apply +/// directly to the store (negative for a release). Cuebot also includes `layer`/`dept` +/// for symmetry/debuggability; serde ignores those extra fields (the store only enforces +/// subscription/folder/job). +#[derive(Debug, Deserialize)] +struct ReleasePayload { + show: Uuid, + alloc: Uuid, + folder: Uuid, + job: Uuid, + cores: i64, + gpus: i32, +} + +/// Cap change payload on `acct_limit_change`. Values are in cores (`-1` = unlimited), +/// GPUs pass through. Exactly one optional field is set per message. +#[derive(Debug, Deserialize)] +#[serde(tag = "vertex")] +enum LimitChangePayload { + #[serde(rename = "sub")] + Sub { + show: Uuid, + alloc: Uuid, + burst: i64, + }, + #[serde(rename = "folder")] + Folder { + id: Uuid, + #[serde(default)] + max_cores: Option, + #[serde(default)] + max_gpus: Option, + }, + #[serde(rename = "job")] + Job { + id: Uuid, + #[serde(default)] + max_cores: Option, + #[serde(default)] + max_gpus: Option, + }, +} + +/// Spawns the listen loop. Reconnects with a fixed backoff on any failure; the recompute +/// and limit-reseed loops are the correctness backstop for whatever it misses. +pub fn spawn_loop(store: Arc) { + tokio::spawn(async move { + loop { + if let Err(err) = run(&store).await { + warn!( + "Accounting NOTIFY listener disconnected: {err}; reconnecting in {:?} \ + (recompute heals the gap)", + RECONNECT_BACKOFF + ); + } + tokio::time::sleep(RECONNECT_BACKOFF).await; + } + }); +} + +async fn run(store: &Store) -> Result<(), sqlx::Error> { + let mut listener = PgListener::connect(&CONFIG.database.connection_url()).await?; + listener + .listen_all([RELEASE_CHANNEL, LIMIT_CHANGE_CHANNEL]) + .await?; + info!("Accounting NOTIFY listener connected ({RELEASE_CHANNEL}, {LIMIT_CHANGE_CHANNEL})"); + loop { + // `recv` errors when the connection drops; bubble up so `spawn_loop` reconnects. + let notification = listener.recv().await?; + match notification.channel() { + RELEASE_CHANNEL => handle_release(store, notification.payload()), + LIMIT_CHANGE_CHANNEL => handle_limit_change(store, notification.payload()), + other => debug!("Ignoring NOTIFY on unexpected channel {other}"), + } + } +} + +fn handle_release(store: &Store, payload: &str) { + match serde_json::from_str::(payload) { + Ok(p) => store.apply_release(&BookingDelta { + show_id: p.show, + alloc_id: p.alloc, + folder_id: p.folder, + job_id: p.job, + core_delta: p.cores, + gpu_delta: p.gpus, + }), + Err(err) => warn!("Dropping malformed acct_release payload ({err}): {payload}"), + } +} + +fn handle_limit_change(store: &Store, payload: &str) { + let parsed = match serde_json::from_str::(payload) { + Ok(p) => p, + Err(err) => { + warn!("Dropping malformed acct_limit_change payload ({err}): {payload}"); + return; + } + }; + for change in limit_changes(parsed) { + store.apply_limit_change(&change); + } +} + +/// Expand a payload into the concrete cap changes it carries (a folder/job message may +/// set cores, gpus, or both). +fn limit_changes(p: LimitChangePayload) -> Vec { + match p { + LimitChangePayload::Sub { + show, + alloc, + burst, + } => vec![LimitChange::SubBurst { + show_id: show, + alloc_id: alloc, + burst, + }], + LimitChangePayload::Folder { + id, + max_cores, + max_gpus, + } => max_cores + .map(|c| LimitChange::FolderMaxCores { + folder_id: id, + max_cores: c, + }) + .into_iter() + .chain(max_gpus.map(|g| LimitChange::FolderMaxGpus { + folder_id: id, + max_gpus: g, + })) + .collect(), + LimitChangePayload::Job { + id, + max_cores, + max_gpus, + } => max_cores + .map(|c| LimitChange::JobMaxCores { + job_id: id, + max_cores: c, + }) + .into_iter() + .chain(max_gpus.map(|g| LimitChange::JobMaxGpus { + job_id: id, + max_gpus: g, + })) + .collect(), + } +} + +// Surface a parse failure loudly in tests if the wire contract drifts from Cuebot's +// `AccountingNotifier`. +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_release_payload() { + let json = r#"{"show":"00000000-0000-0000-0000-000000000001", + "alloc":"00000000-0000-0000-0000-000000000002", + "folder":"00000000-0000-0000-0000-000000000003", + "job":"00000000-0000-0000-0000-000000000004", + "layer":"00000000-0000-0000-0000-000000000005", + "dept":"00000000-0000-0000-0000-000000000006", + "cores":-10,"gpus":-1}"#; + let p: ReleasePayload = serde_json::from_str(json).expect("release payload parses"); + assert_eq!(p.cores, -10); + assert_eq!(p.gpus, -1); + assert_eq!(p.job, Uuid::parse_str("00000000-0000-0000-0000-000000000004").unwrap()); + } + + #[test] + fn parses_sub_burst_change() { + let json = r#"{"vertex":"sub","show":"00000000-0000-0000-0000-000000000001", + "alloc":"00000000-0000-0000-0000-000000000002","burst":200}"#; + let changes = limit_changes(serde_json::from_str(json).unwrap()); + assert!(matches!(changes[..], [LimitChange::SubBurst { burst: 200, .. }])); + } + + #[test] + fn parses_job_max_cores_change_preserving_unlimited() { + let json = r#"{"vertex":"job","id":"00000000-0000-0000-0000-000000000004","max_cores":-1}"#; + let changes = limit_changes(serde_json::from_str(json).unwrap()); + assert!(matches!( + changes[..], + [LimitChange::JobMaxCores { max_cores: -1, .. }] + )); + } + + #[test] + fn folder_change_with_both_fields_expands_to_two() { + let json = r#"{"vertex":"folder","id":"00000000-0000-0000-0000-000000000003", + "max_cores":20,"max_gpus":4}"#; + let changes = limit_changes(serde_json::from_str(json).unwrap()); + assert_eq!(changes.len(), 2); + } +} diff --git a/rust/crates/scheduler/src/accounting/lua.rs b/rust/crates/scheduler/src/accounting/lua.rs deleted file mode 100644 index 2e7c808b88..0000000000 --- a/rust/crates/scheduler/src/accounting/lua.rs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Contributors to the OpenCue Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under -// the License. - -// TODO: per-booking idempotency token if duplicate-booking rate becomes material (design §5). - -/// Hot-path booking script. KEYS / ARGV layout: -/// -/// ```text -/// KEYS[1] = acct:sub:{show_id}:{alloc_id} -/// KEYS[2] = acct:folder:{folder_id} -/// KEYS[3] = acct:job:{job_id} -/// KEYS[4] = acct:layer:{layer_id} -/// KEYS[5] = acct:point:{dept_id}:{show_id} -/// KEYS[6] = acct:seq -/// ARGV[1] = core_delta (signed int as string) -/// ARGV[2] = gpu_delta (signed int as string) -/// ARGV[3] = force ("0" to enforce limits, "1" to bypass) -/// ``` -/// -/// Return shape: -/// - `{1, new_seq}` on success -/// - `{0, table_name, current_value, limit}` on limit-exceeded (force=0 only) -/// -/// Cap semantics: -/// - **Subscription burst** is enforced unconditionally. `int_burst=0` means -/// "reject all bookings", matching Cuebot's `s.int_cores + ? > s.int_burst` -/// check in `SubscriptionDaoJdbc.IS_SHOW_OVER_BURST`. The bootstrap reseed -/// populates burst before the scheduler accepts work, so unconfigured -/// subscriptions cannot dispatch. -/// - **Folder / job `int_max_cores` / `int_max_gpus`** retain the `> 0` guard. -/// Cuebot's convention uses `-1` (the schema default for `folder_resource. -/// int_max_cores` and `int_max_gpus`) as the "unlimited" sentinel, and 0 is -/// not a meaningful configured value for these caps in practice. Core caps -/// are checked only when `core_delta > 0`; GPU caps only when `gpu_delta > 0`. -/// Mirrors `DispatchQuery.FIND_JOBS_BY_SHOW_PRIORITY_MODE`'s -/// `job_resource.int_gpus + layer.int_gpus_min < job_resource.int_max_gpus` -/// predicate that lived in PG before accounting moved to Redis. -/// -/// Units: every numeric field this script reads and writes - `int_cores`, `size`, -/// `burst`, `int_max_cores` - is in **cores** (not centicores). Conversion from PG's -/// centicore storage happens at the limit-reseed and booked-counter-recompute -/// boundaries; the Cuebot release publisher and the Rust booking-delta builder -/// likewise pass cores into this script. See design §0 unit invariant. -pub const BOOK_OR_FORCE: &str = r#" -local core_d = tonumber(ARGV[1]) -local gpu_d = tonumber(ARGV[2]) -local force = ARGV[3] == "1" - -if not force then - if core_d > 0 then - local cur_sub = tonumber(redis.call('HGET', KEYS[1], 'int_cores') or "0") - local sub_burst = tonumber(redis.call('HGET', KEYS[1], 'burst') or "0") - if (cur_sub + core_d) > sub_burst then - return {0, "subscription", cur_sub, sub_burst} - end - - local cur_folder = tonumber(redis.call('HGET', KEYS[2], 'int_cores') or "0") - local folder_max = tonumber(redis.call('HGET', KEYS[2], 'int_max_cores') or "0") - if folder_max > 0 and (cur_folder + core_d) > folder_max then - return {0, "folder", cur_folder, folder_max} - end - - local cur_job = tonumber(redis.call('HGET', KEYS[3], 'int_cores') or "0") - local job_max = tonumber(redis.call('HGET', KEYS[3], 'int_max_cores') or "0") - if job_max > 0 and (cur_job + core_d) > job_max then - return {0, "job", cur_job, job_max} - end - end - - if gpu_d > 0 then - local cur_folder_gpu = tonumber(redis.call('HGET', KEYS[2], 'int_gpus') or "0") - local folder_gpu_max = tonumber(redis.call('HGET', KEYS[2], 'int_max_gpus') or "0") - if folder_gpu_max > 0 and (cur_folder_gpu + gpu_d) > folder_gpu_max then - return {0, "folder_gpus", cur_folder_gpu, folder_gpu_max} - end - - local cur_job_gpu = tonumber(redis.call('HGET', KEYS[3], 'int_gpus') or "0") - local job_gpu_max = tonumber(redis.call('HGET', KEYS[3], 'int_max_gpus') or "0") - if job_gpu_max > 0 and (cur_job_gpu + gpu_d) > job_gpu_max then - return {0, "job_gpus", cur_job_gpu, job_gpu_max} - end - end -end - -redis.call('HINCRBY', KEYS[1], 'int_cores', core_d) -redis.call('HINCRBY', KEYS[1], 'int_gpus', gpu_d) -redis.call('HINCRBY', KEYS[2], 'int_cores', core_d) -redis.call('HINCRBY', KEYS[2], 'int_gpus', gpu_d) -redis.call('HINCRBY', KEYS[3], 'int_cores', core_d) -redis.call('HINCRBY', KEYS[3], 'int_gpus', gpu_d) -redis.call('HINCRBY', KEYS[4], 'int_cores', core_d) -redis.call('HINCRBY', KEYS[4], 'int_gpus', gpu_d) -redis.call('HINCRBY', KEYS[5], 'int_cores', core_d) -redis.call('HINCRBY', KEYS[5], 'int_gpus', gpu_d) --- `acct:seq` is bumped on every mutation, including force-mode rollbacks (design --- §2.4 - all mutations bump seq so concurrent reseed CAS attempts notice). Trade-off: --- during a wave of force-rollbacks (e.g. flaky RQD launch failures) the reseed CAS --- budget can be exhausted, causing the recompute cycle to skip. Acceptable per --- design - hot-path writes are keeping Redis fresh; recompute is reconciliation, --- not primary sync. -return {1, redis.call('INCR', KEYS[6])} -"#; - -/// Reseed write under the `acct:seq` CAS guard. Used by both the recompute loop -/// and the limit reseed loop. ARGV-encoded ops (rather than KEYS) because Redis -/// Cluster is out of scope (single-node per design §2.4) and ARGV avoids the -/// 8000-key EVALSHA limit when reseeding thousands of shows in one shot. -/// -/// ```text -/// KEYS[1] = acct:seq -/// ARGV[1] = seq_before (string-encoded i64) -/// ARGV[2] = n_ops (string-encoded i64) -/// For i in 1..=n_ops: -/// ARGV[2 + 3*(i-1) + 1] = key -/// ARGV[2 + 3*(i-1) + 2] = field -/// ARGV[2 + 3*(i-1) + 3] = value -/// ``` -/// -/// Returns `1` on success, `0` on CAS miss (caller recomputes snapshot and retries). -/// Does NOT bump `acct:seq` - reseed is reconciliation, not mutation; bumping would -/// invalidate any concurrent CAS attempts. -pub const RESEED_CAS: &str = r#" -local cur = redis.call('GET', KEYS[1]) or "0" -if cur ~= ARGV[1] then return 0 end - -local n = tonumber(ARGV[2]) -for i = 0, n - 1 do - local base = 3 + i * 3 - redis.call('HSET', ARGV[base], ARGV[base + 1], ARGV[base + 2]) -end -return 1 -"#; diff --git a/rust/crates/scheduler/src/accounting/managed_shows.rs b/rust/crates/scheduler/src/accounting/managed_shows.rs index 55aabc71dd..d4b1eefb44 100644 --- a/rust/crates/scheduler/src/accounting/managed_shows.rs +++ b/rust/crates/scheduler/src/accounting/managed_shows.rs @@ -19,20 +19,21 @@ use tokio::time; use tracing::{debug, error, warn}; use uuid::Uuid; -use crate::accounting::dao::AccountingDao; +use crate::accounting::dao::{AccountingDao, BaselineKeys}; use crate::accounting::limit_reseed; -use crate::accounting::redis_client::RedisAccounting; +use crate::accounting::recompute; +use crate::accounting::store::Store; use crate::config::CONFIG; /// In-process cache of show ids where `b_scheduler_managed = true`. Refreshed by a /// dedicated `tokio::spawn` loop every `CONFIG.accounting.managed_shows_ttl` /// (default 30 s). /// -/// - Stale-true (show was managed, now isn't): scheduler keeps writing Redis for up to -/// one TTL after the flip; orphan writes are reseeded away within 5 min. -/// - Stale-false (show is now managed, cache still says no): scheduler dispatches without -/// Redis writes. Silent over-count until next refresh + next recompute heal (≤2 min -/// after refresh). Acceptable per design §4.3.1. +/// - Stale-true (show was managed, now isn't): scheduler keeps booking it against the +/// store for up to one TTL after the flip; the next recompute reconciles from `proc`. +/// - Stale-false (show is now managed, cache still says no): scheduler treats it as +/// Cuebot-managed (no store enforcement) until the next refresh seeds its caps. Cuebot +/// keeps booking it via PG in the meantime, so no decision is made against stale state. pub struct ManagedShowsCache { inner: Arc>>, } @@ -56,8 +57,9 @@ impl ManagedShowsCache { /// shutdown ever becomes a requirement (multi-scheduler rollout, integration /// tests that recreate the service), this needs a `CancellationToken` or a /// stored handle. - // TODO: cancellation handle when multi-init/graceful-shutdown lands (design §5). - pub fn start_refresh_loop(self: &Arc, dao: Arc, redis: RedisAccounting) { + // Single-scheduler (N=1) assumed; if multi-init/graceful-shutdown ever lands this needs + // a cancellation handle. + pub fn start_refresh_loop(self: &Arc, dao: Arc, store: Arc) { let inner = self.inner.clone(); tokio::spawn(async move { let mut interval = time::interval(CONFIG.accounting.managed_shows_ttl); @@ -71,12 +73,12 @@ impl ManagedShowsCache { let new_set: HashSet = ids.into_iter().collect(); // Shows that became scheduler-managed since the last refresh. - // Their accounting limits (notably subscription `burst`) may - // not be in Redis yet - bootstrap only seeded shows managed at + // Their enforced caps (notably subscription `burst`) may not be + // in the store yet - bootstrap only seeded shows managed at // startup, and the periodic limit reseed runs on a slow cadence. // Publishing them into the cache now would flip the booking hot // path to enforce against an unseeded burst (== 0 == "reject - // all"). So seed limits FIRST, then publish. + // all"). So seed caps FIRST, then publish (the managed-flip gate). let added: Vec = { let lock = inner.read().unwrap_or_else(|p| p.into_inner()); new_set @@ -86,22 +88,41 @@ impl ManagedShowsCache { .collect() }; if !added.is_empty() { - if let Err(err) = limit_reseed::reseed_limits(&redis, &dao).await { + // Managed-flip blocking gate: seed caps AND booked counters + // for the newly-managed shows BEFORE publishing them into the + // cache. A flipped show may already have live Cuebot procs, so + // the hot path must enforce against real usage from the first + // booking, not against unseeded 0s (== full burst free == + // over-book). The booked seed is a one-shot absolute set per + // show (no epoch bump), so it does not race the recompute + // driver's begin/overwrite sequencing. + let seed = async { + limit_reseed::reseed_limits(&store, &dao).await?; + for show in &added { + let rows = + dao.query_booked_snapshot_for_show(*show).await?; + store.seed_show_booked(&recompute::snapshot_to_counters( + &rows, + &BaselineKeys::default(), + )); + } + Ok::<(), miette::Report>(()) + } + .await; + if let Err(err) = seed { // Defer publishing only the *additions* this tick: a - // newly-managed show that is not yet in the cache - // dispatches without Redis enforcement (silent - // over-count, healed by the next recompute) - strictly - // safer than enforcing against an unseeded burst. - // Retried on the next tick. + // newly-managed show that is not yet in the cache is + // treated as Cuebot-managed (Cuebot still books it via PG + // until the flip lands) - strictly safer than enforcing + // against unseeded state. Retried next tick. // - // Removals still apply: a show that is no longer - // scheduler-managed must drop out of the cache - // regardless of the reseed outcome, otherwise - // apply_booking keeps enforcing Redis for it - // indefinitely (until reseed eventually succeeds). + // Removals still apply: a show no longer scheduler-managed + // must drop out of the cache regardless of the seed + // outcome, otherwise apply_booking keeps enforcing the + // store for it indefinitely. warn!( - "Limit seed for newly-managed show(s) {:?} failed; \ - deferring their cache publish to next tick: {err}", + "Seed for newly-managed show(s) {:?} failed; deferring \ + their cache publish to next tick: {err}", added ); let added_set: HashSet = added.iter().copied().collect(); @@ -112,7 +133,8 @@ impl ManagedShowsCache { return; } debug!( - "Seeded limits for {} newly-managed show(s) before publishing", + "Seeded caps + booked counters for {} newly-managed \ + show(s) before publishing", added.len() ); } diff --git a/rust/crates/scheduler/src/accounting/mod.rs b/rust/crates/scheduler/src/accounting/mod.rs index 12805552bb..337b52e662 100644 --- a/rust/crates/scheduler/src/accounting/mod.rs +++ b/rust/crates/scheduler/src/accounting/mod.rs @@ -10,52 +10,51 @@ // or implied. See the License for the specific language governing permissions and limitations under // the License. -//! Redis-backed accounting service for the Rust scheduler. +//! In-memory accounting service for the Rust scheduler. //! -//! Replaces the in-process `ResourceAccountingService` with a shared store that scales -//! horizontally across N scheduler instances. See the Redis-Backed Accounting Reference -//! at `docs/_docs/developer-guide/redis-accounting.md` for architecture; key invariants: +//! Holds the per-show resource counters in process - a single source of truth that makes +//! the accounting-drift bug class structurally impossible. See the Scheduler Accounting +//! Reference at `docs/_docs/developer-guide/scheduler-accounting.md` for architecture; +//! key invariants: //! -//! - Both Cuebot (release path, see `LettuceAccountingRedisPublisher`) and the Rust -//! scheduler (booking path, here) mutate the same `acct:*` hashes via Lua scripts -//! that bump `acct:seq` atomically. -//! - PG accounting tables stay durable via Cuebot's transactional UPDATEs (for -//! Cuebot-managed shows) or the Rust 2-min recompute loop (for scheduler-managed -//! shows). `proc` remains the canonical record of bookings. -//! - All reseeds (booted counters + limits + bootstrap) use the `acct:seq` CAS guard -//! to avoid clobbering concurrent hot-path writes. +//! - The booking hot path checks subscription burst + folder/job caps and increments the +//! counters atomically under one lock (`Store`). `proc` remains the canonical record. +//! - Releases arrive live via Cuebot's PG `acct_release` NOTIFY (`listener`); cap changes +//! via `acct_limit_change`. Both are optimisations over the periodic backstops. +//! - The recompute (`SUM(proc)` → counters, carrying pending in-flight bookings forward) +//! and the limit reseed keep the store convergent and are the recovery mechanism. +//! - Single scheduler instance (N=1) is assumed; the in-memory store is not shared. pub mod booking_delta; pub mod bootstrap; pub mod dao; pub mod error; pub mod limit_reseed; -pub mod lua; +pub mod listener; pub mod managed_shows; pub mod recompute; -pub mod redis_client; +pub mod store; use std::sync::Arc; use miette::Result; use tokio::sync::OnceCell; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; use uuid::Uuid; use crate::accounting::booking_delta::BookingDelta; use crate::accounting::dao::AccountingDao; use crate::accounting::error::AccountingError; use crate::accounting::managed_shows::ManagedShowsCache; -use crate::accounting::redis_client::{BookMode, BookOutcome, RedisAccounting}; -use crate::config::CONFIG; +use crate::accounting::store::{BookOutcome, Store}; static ACCOUNTING_SERVICE: OnceCell> = OnceCell::const_new(); /// Returns the process-wide `AccountingService`, initializing on first call. /// -/// Initialization connects to Redis and populates the managed-shows cache; the caller -/// is responsible for running `bootstrap::run_blocking_reseed` before the scheduler -/// accepts work, and for spawning the recompute + limit-reseed loops afterwards. +/// Initialization builds the in-memory store and the managed-shows cache; the caller is +/// responsible for running `bootstrap::run_blocking_reseed` before the scheduler accepts +/// work, and for spawning the recompute, limit-reseed, and NOTIFY-listener loops after. pub async fn accounting_service() -> Result> { ACCOUNTING_SERVICE .get_or_try_init(|| async { @@ -66,41 +65,46 @@ pub async fn accounting_service() -> Result> { .cloned() } +/// Result of [`AccountingService::apply_booking`]. Carries the delta so the caller can +/// `confirm`/`rollback` without re-deriving managed status (which can flip mid-dispatch). +pub enum Booking { + /// Show is scheduler-managed and the delta was applied to the store. + Applied(BookingDelta), + /// Show is Cuebot-managed; nothing was applied. Confirm/rollback are no-ops. + NotManaged, +} + /// High-level facade that the dispatcher hot path and the background loops both consume. pub struct AccountingService { - redis: RedisAccounting, + store: Arc, dao: Arc, managed_shows: Arc, } impl AccountingService { - /// Connects to Redis, populates the managed-shows cache from PG, and returns a - /// ready-to-use service. Callers must run `bootstrap::run_blocking_reseed` against - /// the returned service before the scheduler accepts work. + /// Builds the in-memory store, populates the managed-shows cache from PG, and starts + /// the cache refresh loop. Callers must run `bootstrap::run_blocking_reseed` before + /// the scheduler accepts work. pub async fn init() -> Result { - let redis = RedisAccounting::connect(&CONFIG.accounting.redis) - .await - .map_err(|e| miette::miette!("Failed to connect to redis: {e}"))?; + let store = Arc::new(Store::new()); let dao = Arc::new(AccountingDao::new().await?); let managed_shows = ManagedShowsCache::populate(&dao).await?; - // The refresh loop also seeds limits for shows that become managed after startup, - // before publishing them into the cache - so it needs a Redis handle. - managed_shows.start_refresh_loop(dao.clone(), redis.clone()); + // The refresh loop seeds caps for shows that become managed after startup, before + // publishing them into the cache - so it needs a store handle. + managed_shows.start_refresh_loop(dao.clone(), store.clone()); info!( - "AccountingService initialized: redis={}:{} managed_shows={}", - CONFIG.accounting.redis.host, - CONFIG.accounting.redis.port, + "AccountingService initialized (in-memory store): managed_shows={}", managed_shows.snapshot().len(), ); Ok(Self { - redis, + store, dao, managed_shows, }) } - pub fn redis(&self) -> &RedisAccounting { - &self.redis + pub fn store(&self) -> &Arc { + &self.store } pub fn dao(&self) -> &Arc { @@ -111,82 +115,58 @@ impl AccountingService { &self.managed_shows } - /// Hot-path booking. Atomically checks subscription burst, folder/job caps and - /// applies the delta to all five `acct:*` hashes. For shows not currently - /// `b_scheduler_managed = true`, this is a no-op (Cuebot owns the show's accounting). - pub async fn apply_booking(&self, delta: &BookingDelta) -> Result<(), AccountingError> { + /// Hot-path booking. Atomically checks subscription burst, folder/job caps and applies + /// the delta to the counters. For shows not currently `b_scheduler_managed = true` this + /// is a no-op (Cuebot owns the show's accounting), reported as `Booking::NotManaged`. + pub fn apply_booking(&self, delta: &BookingDelta) -> Result { if !self.managed_shows.contains(&delta.show_id) { debug!( "apply_booking skipped: show {} is not scheduler-managed", delta.show_id ); - return Ok(()); + return Ok(Booking::NotManaged); } - match self.redis.book(delta, BookMode::Enforce).await? { - BookOutcome::Applied { new_seq: _ } => Ok(()), + match self.store.book(delta) { + BookOutcome::Applied => Ok(Booking::Applied(delta.clone())), BookOutcome::LimitExceeded { table, current, limit, } => Err(AccountingError::LimitExceeded { - table, + table: table.to_string(), current, limit, }), } } - /// Force-applies a (typically negated) delta, bypassing all caps. Used by the - /// dispatcher on DB or RQD failure to roll back a booking that was already counted - /// in Redis. Cannot return `LimitExceeded`. On Redis failure we log and swallow - - /// the next recompute (≤2 min) heals Redis from `proc`. + /// Booking fully succeeded (proc committed + RQD launched): drop the pending portion, + /// keep the booked increment. No-op when nothing was booked. /// - /// Intentionally does NOT consult `managed_shows`: if `apply_booking` wrote the - /// hashes (the only path that reaches this rollback), we must undo it, even if - /// the show was flipped to Cuebot-managed between the booking and the failure. - /// Otherwise the orphan booking persists in Redis - the recompute filters by - /// `b_scheduler_managed=true` in SQL and won't include the show. - pub async fn apply_force_rollback(&self, delta: &BookingDelta) { - match self.redis.book(delta, BookMode::Force).await { - Ok(_) => {} - Err(err) => { - warn!( - "apply_force_rollback failed for show={} job={}: {err}; \ - recompute will heal Redis from proc", - delta.show_id, delta.job_id - ); - } + /// Intentionally does NOT re-check managed status: if `apply_booking` applied the + /// delta we must settle it even if the show flipped to Cuebot-managed in between. + pub fn confirm_booking(&self, booking: &Booking) { + if let Booking::Applied(delta) = booking { + self.store.confirm(delta); } } - /// Cheap pre-check that would filter doomed host candidates before host selection. - /// Returns `Ok(true)` when the requested cores fit under the subscription burst, - /// `Ok(false)` when they don't. Conservatively returns `true` for non-managed shows - /// and on transient Redis errors. - /// - /// **Currently unwired** because `host_cache::CheckOut.validation` is a sync `Fn` - /// and calling Redis from inside it requires either making the validation async - /// (host_cache actor refactor) or precomputing a per-layer (show, alloc) → bookable - /// map before CheckOut. See the comment at `pipeline/matcher.rs::process_layer`. - #[allow(dead_code)] - pub async fn subscription_can_book( - &self, - show_id: Uuid, - alloc_id: Uuid, - cores_requested: i64, - ) -> bool { - if !self.managed_shows.contains(&show_id) { - return true; - } - match self.redis.read_sub_counters(show_id, alloc_id).await { - Ok((booked, burst)) => burst <= 0 || (booked + cores_requested) <= burst, - Err(err) => { - debug!( - "subscription_can_book HGET failed for show={show_id} alloc={alloc_id}: \ - {err}; allowing through, Lua will decide", - ); - true - } + /// Booking failed before launch: undo the counter increment and the pending delta. + /// No-op when nothing was booked. Like `confirm_booking`, ignores managed status. + pub fn rollback_booking(&self, booking: &Booking) { + if let Booking::Applied(delta) = booking { + self.store.rollback(delta); } } + + /// Live booked cores for a job (E-PVM placement snapshot in the matcher). 0 if unseen + /// or the show is not managed. + pub fn job_cores_in_use(&self, job_id: Uuid) -> i64 { + self.store.job_cores_in_use(job_id) + } + + /// `(booked_cores, burst)` for a subscription (matcher over-burst pre-check), in cores. + pub fn sub_counters(&self, show_id: Uuid, alloc_id: Uuid) -> (i64, i64) { + self.store.sub_counters(show_id, alloc_id) + } } diff --git a/rust/crates/scheduler/src/accounting/recompute.rs b/rust/crates/scheduler/src/accounting/recompute.rs index 453ff8f1ae..42b2ca5fc0 100644 --- a/rust/crates/scheduler/src/accounting/recompute.rs +++ b/rust/crates/scheduler/src/accounting/recompute.rs @@ -10,38 +10,35 @@ // or implied. See the License for the specific language governing permissions and limitations under // the License. -//! Booked-counter recompute loop. Every `CONFIG.accounting.recompute_interval`: +//! Booked-counter recompute loop - the correctness backstop. Every +//! `CONFIG.accounting.recompute_interval`: //! -//! 1. PG side (durable, unconditional): the four existing `RECOMPUTE_*_FROM_PROC` -//! UPDATEs in `ResourceAccountingDao::recompute_all_from_proc` are run -//! concurrently and committed transactionally. These keep the PG accounting -//! tables (Cuegui's view) within ~2 min of `proc` for scheduler-managed shows. -//! 2. Redis side (CAS-guarded): a single unified `SUM(proc)` snapshot -//! keyed by (show, alloc, folder, job, layer, dept) is converted to `HSET` ops -//! on `int_cores`/`int_gpus` fields of the five `acct:*` hashes. Sent in one -//! `RESEED_CAS` Lua call; on CAS miss the snapshot is recomputed and retried -//! up to `CONFIG.accounting.cas_max_retries` times. On budget exhaustion the -//! cycle is skipped (hot-path writes are keeping Redis fresh, per §2.4). -//! -//! PG writes are independent of Redis writes - even if Redis CAS keeps missing, -//! PG converges. They are decoupled stores by design §2.1. - +//! 1. PG side (durable, unconditional): the existing `RECOMPUTE_*_FROM_PROC` UPDATEs keep +//! the PG accounting tables (CueGUI's view) within one interval of `proc` for +//! scheduler-managed shows. +//! 2. Store side: a single `SUM(proc)` snapshot keyed by (show, alloc, folder, job) is +//! overlaid on a zero-baseline and written absolutely into the in-memory store, with +//! each key's in-flight `pending` delta carried forward. The carry-forward is what +//! keeps the overwrite from erasing a booking whose `proc` row is not yet visible to +//! the snapshot - the only way an absolute overwrite could under-count and over-book a +//! hard cap. No retry / CAS: the live store is the primary record, this only reconciles. + +use std::collections::HashMap; use std::panic::AssertUnwindSafe; use std::sync::Arc; use futures::FutureExt; -use miette::{IntoDiagnostic, Result, WrapErr}; +use miette::Result; use tokio::time; use tracing::{debug, error, info, warn}; +use uuid::Uuid; use crate::accounting::dao::{BaselineKeys, BookedSnapshotRow}; -use crate::accounting::error::AccountingError; -use crate::accounting::redis_client::ReseedOp; +use crate::accounting::store::{centicores_to_cores, CounterSnapshot}; use crate::accounting::AccountingService; use crate::config::CONFIG; use crate::dao::ResourceAccountingDao; use crate::metrics; -use crate::models::CoreSize; pub fn spawn_loop(service: Arc) { tokio::spawn(async move { @@ -94,202 +91,105 @@ pub fn spawn_loop(service: Arc) { }); } -/// One pass: PG recompute (unconditional) + Redis reseed (CAS-guarded). +/// One pass: PG recompute (unconditional, for CueGUI) + store reseed. async fn run_once(service: &AccountingService, pg_dao: &Arc) -> Result<()> { debug!("Recompute cycle: starting"); - // PG side: durable, scoped to scheduler-managed shows. Empty list is a no-op - // inside the DAO so we never widen to all shows and clobber Cuebot's accounting. - let managed_ids: Vec = service.managed_shows().snapshot().into_iter().collect(); + // PG side: durable, scoped to scheduler-managed shows. Empty list is a no-op inside + // the DAO so we never widen to all shows and clobber Cuebot's accounting. + let managed_ids: Vec = service.managed_shows().snapshot().into_iter().collect(); if managed_ids.is_empty() { debug!("PG recompute skipped: no scheduler-managed shows"); } else { if let Err(err) = pg_dao.recompute_all_from_proc(&managed_ids).await { - warn!( - "PG recompute (layer/job/folder/point) failed (Redis reseed will still run): {err}" - ); + warn!("PG recompute (layer/job/folder/point) failed (store reseed still runs): {err}"); } if let Err(err) = pg_dao.recompute_subscription_table(&managed_ids).await { - warn!("PG subscription recompute failed (Redis reseed will still run): {err}"); + warn!("PG subscription recompute failed (store reseed still runs): {err}"); } } - // Redis side: CAS-guarded. - reseed_redis_once(service).await + reseed_store_once(service).await } -/// CAS-guarded reseed of the booked-counter fields (`int_cores`/`int_gpus`) on the -/// five `acct:*` hashes from a fresh `SUM(proc)` snapshot. Used by both the recompute -/// loop and the bootstrap. On CAS-budget exhaustion this returns -/// `AccountingError::CasContentionExceeded`: the bootstrap caller treats it as a fatal -/// startup gate, while the periodic loop downgrades it to a warn-log (per §2.4). +/// Overwrite the store's booked counters from a fresh `SUM(proc)` snapshot, carrying each +/// key's pending in-flight delta forward (handled inside `Store::overwrite_counters`). /// -/// The snapshot is overlaid on a zero-baseline of every enumerable sub/folder/job/point -/// key (see `query_booked_baseline_keys`), so a key whose counter drifted stale-high and -/// then drained to zero procs is reset to 0 rather than being left untouched - the -/// `SUM(proc)` snapshot alone only returns keys that still have procs. Both the snapshot -/// and the baseline are re-fetched per CAS attempt, matching `limit_reseed::reseed_once`. -pub async fn reseed_redis_once(service: &AccountingService) -> Result<()> { - let max_retries = CONFIG.accounting.cas_max_retries; - for attempt in 0..=max_retries { - let seq_before = service.redis().get_seq().await.into_diagnostic()?; - let (rows, baseline) = tokio::try_join!( - service.dao().query_booked_snapshot(), - service.dao().query_booked_baseline_keys(), - )?; - let ops = booked_ops_from_snapshot(&rows, &baseline); - debug!( - "Recompute reseed attempt {}/{}: {} rows -> {} ops at seq={}", - attempt + 1, - max_retries + 1, - rows.len(), - ops.len(), - seq_before - ); - let applied = service - .redis() - .reseed_cas(seq_before, &ops) - .await - .into_diagnostic() - .wrap_err("RESEED_CAS for booked counters failed")?; - if applied { - info!( - "Recompute reseed applied: {} ops, seq={}", - ops.len(), - seq_before - ); - return Ok(()); - } - warn!( - "Recompute reseed CAS miss (attempt {}/{}); resnapshot and retry", - attempt + 1, - max_retries + 1 - ); - } - // CAS budget exhausted. Return an error so the bootstrap caller (which uses `?` as a - // startup gate) refuses to begin booking against an unseeded Redis. The periodic - // recompute loop catches this and downgrades it to a warn-log instead - there, - // hot-path writes are keeping Redis fresh per design §2.4, so a skipped cycle is fine. - Err(AccountingError::CasContentionExceeded { - attempts: max_retries + 1, - }) - .into_diagnostic() +/// The snapshot is overlaid on a zero-baseline of every enumerable sub/folder/job key, so +/// a key whose counter drifted stale-high and then drained to zero procs is reset to 0 +/// (the `SUM(proc)` snapshot alone only returns keys that still have procs). Used by both +/// the recompute loop and the bootstrap. +pub async fn reseed_store_once(service: &AccountingService) -> Result<()> { + // Bump the epoch BEFORE reading the snapshot so a `confirm` that races this read lands + // in a settled bucket the overwrite will not clear (closes the straddle hole). + let epoch = service.store().begin_recompute(); + let (rows, baseline) = tokio::try_join!( + service.dao().query_booked_snapshot(), + service.dao().query_booked_baseline_keys(), + )?; + let snapshot = snapshot_to_counters(&rows, &baseline); + let n = snapshot.sub.len() + snapshot.folder.len() + snapshot.job.len(); + service.store().overwrite_counters(&snapshot, epoch); + info!("Recompute reseed applied: {} keys from {} proc rows", n, rows.len()); + Ok(()) } -/// Aggregate one `SUM(proc)` snapshot into HSET ops, one set of ops per unique key. -/// -/// The SQL groups by `(show, alloc, folder, job, layer, dept)` - a finer granularity -/// than any of the Redis keys. A folder with several jobs, or a job whose procs span -/// multiple allocations, produces several snapshot rows that all map to the same -/// `acct:folder:{folder}` (or `acct:job:{job}`, etc.) key. Because `RESEED_CAS` does -/// `HSET` (overwrite) rather than `HINCRBY`, emitting one op per row would let later -/// rows clobber earlier ones - the final value would be whichever row sorted last, -/// not the sum across rows. Aggregate first, emit once per unique key. +/// Aggregate one `SUM(proc)` snapshot into per-vertex core/gpu totals (in cores), overlaid +/// on the zero-baseline. The SQL groups finer than the vertices (by layer/dept too), so +/// several rows fold into the same sub/folder/job key - aggregate before converting. /// -/// `baseline` seeds a zero entry for every enumerable sub/folder/job/point key before -/// the proc sums are folded in. A key in the baseline but absent from `rows` has no -/// procs, so it emits `int_cores=0`/`int_gpus=0` - this is what lets recompute converge -/// a counter that drifted stale-high and then drained to zero procs (without it, such a -/// key would simply be missing from the snapshot and never corrected). Layers are not -/// in the baseline (no limit table to enumerate them and the booking Lua never reads the -/// layer counter), so the layer map stays purely proc-driven - residual layer drift is -/// cosmetic by design. -fn booked_ops_from_snapshot(rows: &[BookedSnapshotRow], baseline: &BaselineKeys) -> Vec { - use std::collections::HashMap; - - let mut sub_totals: HashMap<(uuid::Uuid, uuid::Uuid), (i64, i64)> = HashMap::new(); - let mut folder_totals: HashMap = HashMap::new(); - let mut job_totals: HashMap = HashMap::new(); - let mut layer_totals: HashMap = HashMap::new(); - let mut point_totals: HashMap<(uuid::Uuid, uuid::Uuid), (i64, i64)> = HashMap::new(); - - // Zero-baseline first: every enumerable key gets a (0, 0) entry so keys with no - // procs still emit a resetting HSET. The proc fold below adds on top of these. +/// Layer and point are intentionally absent: the booking check never reads them, so they +/// are not tracked in the store (the legacy Lua incremented them but never enforced them). +pub(crate) fn snapshot_to_counters( + rows: &[BookedSnapshotRow], + baseline: &BaselineKeys, +) -> CounterSnapshot { + let mut sub: HashMap<(Uuid, Uuid), (i64, i64)> = HashMap::new(); + let mut folder: HashMap = HashMap::new(); + let mut job: HashMap = HashMap::new(); + + // Zero-baseline first: every enumerable key gets a (0, 0) entry so drained keys still + // reset. Centicore sums are folded on top below, then converted once at the end. for &k in &baseline.subs { - sub_totals.entry(k).or_default(); + sub.entry(k).or_default(); } for &k in &baseline.folders { - folder_totals.entry(k).or_default(); + folder.entry(k).or_default(); } for &k in &baseline.jobs { - job_totals.entry(k).or_default(); - } - for &k in &baseline.points { - point_totals.entry(k).or_default(); + job.entry(k).or_default(); } + // Accumulate centicores; convert to cores after summing so truncation happens once. for r in rows { - let s = sub_totals.entry((r.show_id, r.alloc_id)).or_default(); + let s = sub.entry((r.show_id, r.alloc_id)).or_default(); s.0 += r.cores; s.1 += r.gpus; - let f = folder_totals.entry(r.folder_id).or_default(); + let f = folder.entry(r.folder_id).or_default(); f.0 += r.cores; f.1 += r.gpus; - let j = job_totals.entry(r.job_id).or_default(); + let j = job.entry(r.job_id).or_default(); j.0 += r.cores; j.1 += r.gpus; - let l = layer_totals.entry(r.layer_id).or_default(); - l.0 += r.cores; - l.1 += r.gpus; - let p = point_totals.entry((r.dept_id, r.show_id)).or_default(); - p.0 += r.cores; - p.1 += r.gpus; } - let total_keys = sub_totals.len() - + folder_totals.len() - + job_totals.len() - + layer_totals.len() - + point_totals.len(); - let mut ops = Vec::with_capacity(total_keys * 2); - - fn push_pair(ops: &mut Vec, key: String, cores_centi: i64, gpus: i64) { - // PG centicores → Redis cores via the typed conversion. Booked sums are - // non-negative, so the non-cap variant is correct here. - ops.push(ReseedOp { - key: key.clone(), - field: "int_cores", - value: i64::from(CoreSize::from_multiplied(cores_centi).value()), - }); - ops.push(ReseedOp { - key, - field: "int_gpus", - value: gpus, - }); + CounterSnapshot { + sub: to_cores(sub), + folder: to_cores(folder), + job: to_cores(job), } +} - for ((show_id, alloc_id), (cores, gpus)) in sub_totals { - push_pair( - &mut ops, - format!("acct:sub:{}:{}", show_id, alloc_id), - cores, - gpus, - ); - } - for (folder_id, (cores, gpus)) in folder_totals { - push_pair(&mut ops, format!("acct:folder:{}", folder_id), cores, gpus); - } - for (job_id, (cores, gpus)) in job_totals { - push_pair(&mut ops, format!("acct:job:{}", job_id), cores, gpus); - } - for (layer_id, (cores, gpus)) in layer_totals { - push_pair(&mut ops, format!("acct:layer:{}", layer_id), cores, gpus); - } - for ((dept_id, show_id), (cores, gpus)) in point_totals { - push_pair( - &mut ops, - format!("acct:point:{}:{}", dept_id, show_id), - cores, - gpus, - ); - } - ops +/// Convert each key's accumulated centicore total to cores (GPUs pass through). +fn to_cores(m: HashMap) -> HashMap { + m.into_iter() + .map(|(k, (cores, gpus))| (k, (centicores_to_cores(cores), gpus))) + .collect() } #[cfg(test)] mod tests { use super::*; - use uuid::Uuid; fn fixture_row() -> BookedSnapshotRow { // PG-shaped: `cores` is centicores per SUM(proc.int_cores_reserved). 4200 = 42 cores. @@ -298,260 +198,58 @@ mod tests { alloc_id: Uuid::nil(), folder_id: Uuid::nil(), job_id: Uuid::nil(), - layer_id: Uuid::nil(), - dept_id: Uuid::nil(), cores: 4200, gpus: 3, } } - fn find_op<'a>(ops: &'a [ReseedOp], key: &str, field: &str) -> &'a ReseedOp { - ops.iter() - .find(|o| o.key == key && o.field == field) - .unwrap_or_else(|| panic!("no op for key={key} field={field}")) - } - - fn count_ops_for_key(ops: &[ReseedOp], key: &str) -> usize { - ops.iter().filter(|o| o.key == key).count() - } - fn empty_baseline() -> BaselineKeys { BaselineKeys::default() } #[test] - fn snapshot_single_row_expands_to_ten_ops_in_cores() { - let ops = booked_ops_from_snapshot(&[fixture_row()], &empty_baseline()); - // 5 unique keys × 2 fields (int_cores, int_gpus). - assert_eq!(ops.len(), 10); - let cores_ops: Vec<_> = ops.iter().filter(|o| o.field == "int_cores").collect(); - let gpus_ops: Vec<_> = ops.iter().filter(|o| o.field == "int_gpus").collect(); - assert_eq!(cores_ops.len(), 5); - assert_eq!(gpus_ops.len(), 5); - // PG centicores 4200 -> Redis cores 42. - assert!(cores_ops.iter().all(|o| o.value == 42)); - // GPUs pass through unconverted. - assert!(gpus_ops.iter().all(|o| o.value == 3)); + fn single_row_converts_centicores_to_cores() { + let snap = snapshot_to_counters(&[fixture_row()], &empty_baseline()); + assert_eq!(snap.sub[&(Uuid::nil(), Uuid::nil())], (42, 3)); + assert_eq!(snap.folder[&Uuid::nil()], (42, 3)); + assert_eq!(snap.job[&Uuid::nil()], (42, 3)); } + /// Two jobs in the same folder/sub: the coarse keys SUM across rows (not last-write). #[test] - fn snapshot_keys_match_publisher_format() { - let ops = booked_ops_from_snapshot(&[fixture_row()], &empty_baseline()); - let keys: std::collections::HashSet<&str> = ops.iter().map(|o| o.key.as_str()).collect(); - assert!(keys.contains( - "acct:sub:00000000-0000-0000-0000-000000000000:00000000-0000-0000-0000-000000000000" - )); - assert!(keys.contains("acct:folder:00000000-0000-0000-0000-000000000000")); - assert!(keys.contains("acct:job:00000000-0000-0000-0000-000000000000")); - assert!(keys.contains("acct:layer:00000000-0000-0000-0000-000000000000")); - assert!(keys.contains( - "acct:point:00000000-0000-0000-0000-000000000000:00000000-0000-0000-0000-000000000000" - )); - } - - /// Two jobs in the same folder, same sub, same point. The coarse counters must - /// SUM across the per-job snapshot rows, not pick "last write wins." - /// Snapshot `cores` are PG centicores; assertions are in Redis cores. - #[test] - fn snapshot_sums_coarse_keys_across_per_job_rows() { + fn sums_coarse_keys_across_per_job_rows() { let show = Uuid::new_v4(); let alloc = Uuid::new_v4(); let folder = Uuid::new_v4(); - let dept = Uuid::new_v4(); let row_a = BookedSnapshotRow { show_id: show, alloc_id: alloc, folder_id: folder, job_id: Uuid::new_v4(), - layer_id: Uuid::new_v4(), - dept_id: dept, cores: 1000, // 10 cores gpus: 1, }; let row_b = BookedSnapshotRow { - show_id: show, - alloc_id: alloc, - folder_id: folder, - job_id: Uuid::new_v4(), - layer_id: Uuid::new_v4(), - dept_id: dept, cores: 2500, // 25 cores gpus: 2, + job_id: Uuid::new_v4(), + ..row_a.clone() }; - - let ops = booked_ops_from_snapshot(&[row_a, row_b], &empty_baseline()); - - // Centicores summed (3500), then /100 -> 35 cores. - let sub_key = format!("acct:sub:{}:{}", show, alloc); - assert_eq!(find_op(&ops, &sub_key, "int_cores").value, 35); - assert_eq!(find_op(&ops, &sub_key, "int_gpus").value, 3); - assert_eq!(count_ops_for_key(&ops, &sub_key), 2); - - let folder_key = format!("acct:folder:{}", folder); - assert_eq!(find_op(&ops, &folder_key, "int_cores").value, 35); - assert_eq!(find_op(&ops, &folder_key, "int_gpus").value, 3); - assert_eq!(count_ops_for_key(&ops, &folder_key), 2); - - let point_key = format!("acct:point:{}:{}", dept, show); - assert_eq!(find_op(&ops, &point_key, "int_cores").value, 35); - assert_eq!(find_op(&ops, &point_key, "int_gpus").value, 3); - assert_eq!(count_ops_for_key(&ops, &point_key), 2); - } - - /// One job whose procs span two allocations. The job and layer counters must sum - /// across the two snapshot rows, while the two sub counters are independent. - /// Snapshot `cores` are PG centicores; assertions are in Redis cores. - #[test] - fn snapshot_sums_job_and_layer_across_allocations() { - let show = Uuid::new_v4(); - let folder = Uuid::new_v4(); - let dept = Uuid::new_v4(); - let job = Uuid::new_v4(); - let layer = Uuid::new_v4(); - let alloc_a = Uuid::new_v4(); - let alloc_b = Uuid::new_v4(); - let rows = [ - BookedSnapshotRow { - show_id: show, - alloc_id: alloc_a, - folder_id: folder, - job_id: job, - layer_id: layer, - dept_id: dept, - cores: 1000, // 10 cores - gpus: 0, - }, - BookedSnapshotRow { - show_id: show, - alloc_id: alloc_b, - folder_id: folder, - job_id: job, - layer_id: layer, - dept_id: dept, - cores: 700, // 7 cores - gpus: 0, - }, - ]; - - let ops = booked_ops_from_snapshot(&rows, &empty_baseline()); - - let job_key = format!("acct:job:{}", job); - assert_eq!(find_op(&ops, &job_key, "int_cores").value, 17); - assert_eq!(count_ops_for_key(&ops, &job_key), 2); - - let layer_key = format!("acct:layer:{}", layer); - assert_eq!(find_op(&ops, &layer_key, "int_cores").value, 17); - assert_eq!(count_ops_for_key(&ops, &layer_key), 2); - - // Sub counters stay per-allocation. - let sub_a = format!("acct:sub:{}:{}", show, alloc_a); - let sub_b = format!("acct:sub:{}:{}", show, alloc_b); - assert_eq!(find_op(&ops, &sub_a, "int_cores").value, 10); - assert_eq!(find_op(&ops, &sub_b, "int_cores").value, 7); + let snap = snapshot_to_counters(&[row_a, row_b], &empty_baseline()); + // 3500 centicores summed then /100 -> 35 cores; gpus 3. + assert_eq!(snap.sub[&(show, alloc)], (35, 3)); + assert_eq!(snap.folder[&folder], (35, 3)); } - /// A baseline key with no matching proc row (its procs drained to zero) must emit a - /// resetting `int_cores=0`/`int_gpus=0` pair so recompute can converge a stale-high - /// counter back to truth. This is the core of the zero-convergence fix. + /// A baseline key with no proc row (drained to zero) must reset to 0. #[test] - fn baseline_key_absent_from_snapshot_emits_zero_pair() { - let show = Uuid::new_v4(); - let alloc = Uuid::new_v4(); - let folder = Uuid::new_v4(); + fn baseline_key_absent_from_snapshot_resets_to_zero() { let job = Uuid::new_v4(); - let dept = Uuid::new_v4(); let baseline = BaselineKeys { - subs: vec![(show, alloc)], - folders: vec![folder], jobs: vec![job], - points: vec![(dept, show)], + ..Default::default() }; - - // No proc rows at all: every baseline key drained to zero. - let ops = booked_ops_from_snapshot(&[], &baseline); - - // 4 enumerable keys × 2 fields; layers have no baseline so none appear. - assert_eq!(ops.len(), 8); - for key in [ - format!("acct:sub:{}:{}", show, alloc), - format!("acct:folder:{}", folder), - format!("acct:job:{}", job), - format!("acct:point:{}:{}", dept, show), - ] { - assert_eq!(find_op(&ops, &key, "int_cores").value, 0); - assert_eq!(find_op(&ops, &key, "int_gpus").value, 0); - } - } - - /// A baseline key that also appears in the snapshot is not double-counted: it emits - /// one pair carrying the proc sum, not the seeded zero plus the sum. - #[test] - fn baseline_key_present_in_snapshot_uses_proc_sum_once() { - let row = fixture_row(); // all-nil keys, 4200 centicores -> 42 cores, 3 gpus. - let baseline = BaselineKeys { - subs: vec![(Uuid::nil(), Uuid::nil())], - folders: vec![Uuid::nil()], - jobs: vec![Uuid::nil()], - points: vec![(Uuid::nil(), Uuid::nil())], - }; - - let ops = booked_ops_from_snapshot(&[row], &baseline); - - // Still one pair per key (no zero/sum duplication): 5 keys × 2 fields. - assert_eq!(ops.len(), 10); - let job_key = "acct:job:00000000-0000-0000-0000-000000000000"; - assert_eq!(count_ops_for_key(&ops, job_key), 2); - assert_eq!(find_op(&ops, job_key, "int_cores").value, 42); - assert_eq!(find_op(&ops, job_key, "int_gpus").value, 3); - } - - /// Mixed: one baseline job has procs (keep its sum), another drained to zero (reset). - /// The layer that exists only in the snapshot is still emitted from proc data. - #[test] - fn baseline_resets_drained_key_while_keeping_active_one() { - let show = Uuid::new_v4(); - let alloc = Uuid::new_v4(); - let folder = Uuid::new_v4(); - let dept = Uuid::new_v4(); - let active_job = Uuid::new_v4(); - let drained_job = Uuid::new_v4(); - let active_layer = Uuid::new_v4(); - - let baseline = BaselineKeys { - subs: vec![(show, alloc)], - folders: vec![folder], - jobs: vec![active_job, drained_job], - points: vec![(dept, show)], - }; - let rows = [BookedSnapshotRow { - show_id: show, - alloc_id: alloc, - folder_id: folder, - job_id: active_job, - layer_id: active_layer, - dept_id: dept, - cores: 500, // 5 cores - gpus: 1, - }]; - - let ops = booked_ops_from_snapshot(&rows, &baseline); - - let active_key = format!("acct:job:{}", active_job); - assert_eq!(find_op(&ops, &active_key, "int_cores").value, 5); - assert_eq!(find_op(&ops, &active_key, "int_gpus").value, 1); - - let drained_key = format!("acct:job:{}", drained_job); - assert_eq!(find_op(&ops, &drained_key, "int_cores").value, 0); - assert_eq!(find_op(&ops, &drained_key, "int_gpus").value, 0); - - // Layer is proc-driven only; the active layer is present, no zero-baseline layers. - let layer_key = format!("acct:layer:{}", active_layer); - assert_eq!(find_op(&ops, &layer_key, "int_cores").value, 5); - assert_eq!( - ops.iter() - .filter(|o| o.key.starts_with("acct:layer:")) - .count(), - 2 - ); + let snap = snapshot_to_counters(&[], &baseline); + assert_eq!(snap.job[&job], (0, 0)); } } diff --git a/rust/crates/scheduler/src/accounting/redis_client.rs b/rust/crates/scheduler/src/accounting/redis_client.rs deleted file mode 100644 index 2daa1bfb03..0000000000 --- a/rust/crates/scheduler/src/accounting/redis_client.rs +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright Contributors to the OpenCue Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under -// the License. - -// TODO: replace single-node URL with Sentinel/Cluster topology when Redis HA lands (design §5). -// TODO: detect Redis FLUSHALL/restart and re-trigger bootstrap reseed without scheduler restart -// (design §4.3 row 5). Today, on connection failure the booking returns Unavailable and -// the dispatcher idles; if Redis is restarted empty, the next booking will populate -// acct:* hashes from zero - limit fields will be missing until the next limit reseed (5 min). - -use redis::{aio::ConnectionManager, AsyncCommands, Script}; - -use crate::accounting::booking_delta::{BookingDelta, SEQ_KEY}; -use crate::accounting::error::AccountingError; -use crate::accounting::lua::{BOOK_OR_FORCE, RESEED_CAS}; -use crate::config::RedisConfig; - -/// One value to write during a reseed: HSET `key` `field` `value`. -#[derive(Debug, Clone)] -pub struct ReseedOp { - pub key: String, - pub field: &'static str, - pub value: i64, -} - -/// Outcome of a `BOOK_OR_FORCE` Lua call. -pub enum BookOutcome { - /// Booking accepted; `acct:seq` was bumped to `new_seq` (currently consumed for - /// diagnostics by callers that want to log it; the dispatcher path ignores it). - Applied { - #[allow(dead_code)] - new_seq: i64, - }, - /// Booking rejected because a hard cap would be exceeded. - LimitExceeded { - table: String, - current: i64, - limit: i64, - }, -} - -/// Whether to enforce subscription/folder/job caps in the booking Lua. -#[derive(Debug, Clone, Copy)] -pub enum BookMode { - /// Hot-path booking: enforce caps; reject the booking if any would be exceeded. - Enforce, - /// Force mode: bypass caps. Used for compensation rollbacks (negated deltas) and - /// any other path that must not be rejected by a stale cap. - Force, -} - -impl BookMode { - fn as_arg(self) -> &'static str { - match self { - BookMode::Enforce => "0", - BookMode::Force => "1", - } - } -} - -/// Low-level Redis client for accounting. Holds two `redis::Script` handles which manage -/// their own SHA cache and reload on `NOSCRIPT` automatically (see `Script::invoke_async`). -#[derive(Clone)] -pub struct RedisAccounting { - conn: ConnectionManager, - book_script: Script, - reseed_script: Script, -} - -impl RedisAccounting { - pub async fn connect(cfg: &RedisConfig) -> Result { - let client = redis::Client::open(cfg.url()) - .map_err(|e| AccountingError::Unavailable(format!("invalid redis url: {e}")))?; - let conn = ConnectionManager::new(client).await?; - Ok(Self { - conn, - book_script: Script::new(BOOK_OR_FORCE), - reseed_script: Script::new(RESEED_CAS), - }) - } - - /// Runs the booking Lua script over the five hashes referenced by `delta`. - pub async fn book( - &self, - delta: &BookingDelta, - mode: BookMode, - ) -> Result { - let mut conn = self.conn.clone(); - let value: redis::Value = self - .book_script - .key(delta.sub_key()) - .key(delta.folder_key()) - .key(delta.job_key()) - .key(delta.layer_key()) - .key(delta.point_key()) - .key(SEQ_KEY) - .arg(delta.core_delta.to_string()) - .arg(delta.gpu_delta.to_string()) - .arg(mode.as_arg()) - .invoke_async(&mut conn) - .await?; - parse_book_outcome(value) - } - - /// Issues a CAS reseed. `seq_before` is the snapshot caller observed; `ops` is the - /// list of HSETs to apply atomically. Returns `Ok(true)` on success, `Ok(false)` on - /// CAS miss (caller should resnapshot and retry). - pub async fn reseed_cas( - &self, - seq_before: i64, - ops: &[ReseedOp], - ) -> Result { - let mut conn = self.conn.clone(); - let mut invocation = self.reseed_script.prepare_invoke(); - invocation.key(SEQ_KEY); - invocation.arg(seq_before.to_string()); - invocation.arg(ops.len().to_string()); - for op in ops { - invocation.arg(op.key.as_str()); - invocation.arg(op.field); - invocation.arg(op.value.to_string()); - } - let result: i64 = invocation.invoke_async(&mut conn).await?; - Ok(result == 1) - } - - /// Applies `ops` as plain `HSET`s in a single pipeline, with no `acct:seq` CAS guard. - /// - /// Used by the limit reseed. Unlike the booked-counter recompute (which overwrites the - /// `int_cores`/`int_gpus` fields the hot path also increments, and so must CAS against - /// `acct:seq` to avoid clobbering a concurrent booking), limit fields (`burst`, `size`, - /// `int_max_cores`, `int_max_gpus`, `int_min_*`, `int_priority`) are written ONLY here - - /// the hot-path booking Lua never mutates them. They therefore cannot race a booking, so - /// the CAS guard is unnecessary and, worse, lets a wave of force-rollbacks bumping - /// `acct:seq` (e.g. flaky RQD launches) starve the reseed into skipping its cycle - which - /// leaves a freshly-managed show's subscription `burst` unseeded (== 0 == "reject all" in - /// the booking Lua). An unconditional write always lands. - /// - /// Does not bump `acct:seq` (`HSET` only), so it never disturbs a concurrent - /// `reseed_cas`. Per-field `HSET` is not all-or-nothing, but limit fields are mutually - /// independent and the next cycle (or the synchronous seed on managed-flip) completes any - /// partial write. - pub async fn reseed_unconditional(&self, ops: &[ReseedOp]) -> Result<(), AccountingError> { - if ops.is_empty() { - return Ok(()); - } - let mut conn = self.conn.clone(); - let mut pipe = redis::pipe(); - for op in ops { - pipe.cmd("HSET") - .arg(op.key.as_str()) - .arg(op.field) - .arg(op.value) - .ignore(); - } - let _: () = pipe.query_async(&mut conn).await?; - Ok(()) - } - - /// Reads the current `acct:seq` value (defaults to 0 if missing). - pub async fn get_seq(&self) -> Result { - let mut conn = self.conn.clone(); - let v: Option = conn.get(SEQ_KEY).await?; - Ok(v.unwrap_or(0)) - } - - /// Reads the subscription hash's booked cores + burst in one round-trip from - /// `acct:sub:{show_id}:{alloc_id}` (fields `int_cores`, `burst`, both in - /// cores — the centicore→core conversion happens once at the reseed write - /// boundary, see the `lua.rs` unit invariant). Missing keys/fields are - /// treated as `(0, 0)`. Non-authoritative: - /// the dispatcher's Lua `BOOK_OR_FORCE` call remains the source of truth for - /// the booking decision; this is a snapshot suitable for optimistic pre-filters - /// and scoring inputs. - pub async fn read_sub_counters( - &self, - show_id: uuid::Uuid, - alloc_id: uuid::Uuid, - ) -> Result<(i64, i64), AccountingError> { - let mut conn = self.conn.clone(); - let key = format!("acct:sub:{}:{}", show_id, alloc_id); - let values: Vec> = conn.hget(&key, &["int_cores", "burst"]).await?; - let booked = values.first().copied().flatten().unwrap_or(0); - let burst = values.get(1).copied().flatten().unwrap_or(0); - Ok((booked, burst)) - } - - /// Reads `acct:job:{job_id}` `int_cores` (live booked cores, in cores — see - /// the `lua.rs` unit invariant; Redis accounting counters are never stored - /// in centicores). Returns 0 when the key/field is missing. Used by the E-PVM placement - /// snapshot in `MatchingService::process_layer` (design Branch 2a). - pub async fn read_job_cores_in_use(&self, job_id: uuid::Uuid) -> Result { - let mut conn = self.conn.clone(); - let key = format!("acct:job:{}", job_id); - let v: Option = conn.hget(&key, "int_cores").await?; - Ok(v.unwrap_or(0)) - } -} - -/// Parses the raw `redis::Value` returned by the `BOOK_OR_FORCE` Lua script (see -/// `accounting::lua`) into a typed `BookOutcome`. -/// -/// # Wire format -/// -/// Lua's `return {...}` produces a multi-bulk reply, which `redis-rs` decodes as -/// `Value::Array`. Two shapes are possible: -/// -/// ```text -/// Applied (status=1): Array[Int(1), Int(new_seq)] -/// LimitExceeded (status=0): Array[Int(0), Str(table_name), Int(current), Int(limit)] -/// ``` -/// -/// `redis::Value` element types: -/// - `Int(i64)` — Lua numbers (`return {1, ...}`, `INCR`-returned counters). -/// - `BulkString(Vec)` on RESP2 / `SimpleString(String)` on RESP3 — Lua string -/// literals (`"subscription"`, `"folder"`, `"job"`). Both forms must be accepted -/// because the connection's protocol version is not pinned here. -/// -/// # Error handling -/// -/// Anything outside the two shapes above means the Lua source and this parser are -/// out of sync — a programmer error, not a transient infra fault. We surface it as -/// `AccountingError::Unexpected`, which the dispatcher routes to the new -/// `DispatchVirtualProcError::AccountingUnexpected` arm so ops can alert on it -/// separately from `Unavailable`. Each match arm carries the position it failed at -/// (status / new_seq / table / current / limit) so a `{other:?}` log line is enough -/// to localize the protocol drift. -fn parse_book_outcome(value: redis::Value) -> Result { - use redis::Value; - - // Top-level: must be the multi-bulk reply produced by Lua's `return {...}`. - let array = match value { - Value::Array(items) => items, - other => { - return Err(AccountingError::Unexpected(format!( - "expected Lua array, got {other:?}" - ))) - } - }; - - // array[0] = status discriminator. 1 = booking applied, 0 = cap rejected. - let status = match array.first() { - Some(Value::Int(n)) => *n, - other => { - return Err(AccountingError::Unexpected(format!( - "expected int status as first element, got {other:?}" - ))) - } - }; - - if status == 1 { - // Applied shape: [Int(1), Int(new_seq)]. new_seq is `INCR acct:seq`. - let new_seq = match array.get(1) { - Some(Value::Int(n)) => *n, - other => { - return Err(AccountingError::Unexpected(format!( - "expected int new_seq, got {other:?}" - ))) - } - }; - Ok(BookOutcome::Applied { new_seq }) - } else { - // LimitExceeded shape: [Int(0), Str(table), Int(current), Int(limit)]. - // `table` is one of "subscription" / "folder" / "job" / "folder_gpus" / - // "job_gpus" — see lua.rs cap checks. Accept both BulkString (RESP2) and - // SimpleString (RESP3) since either is valid for a Lua string literal - // across redis-rs versions. - let table = match array.get(1) { - Some(Value::BulkString(s)) => String::from_utf8_lossy(s).into_owned(), - Some(Value::SimpleString(s)) => s.clone(), - other => { - return Err(AccountingError::Unexpected(format!( - "expected table name string, got {other:?}" - ))) - } - }; - // `current` is the pre-increment counter value the Lua observed via HGET; - // useful for log lines / `LimitExceeded` error messages so operators see - // exactly where the cap fired. - let current = match array.get(2) { - Some(Value::Int(n)) => *n, - other => { - return Err(AccountingError::Unexpected(format!( - "expected int current, got {other:?}" - ))) - } - }; - // `limit` is the cap that was exceeded (subscription burst, folder/job - // int_max_cores, or folder/job int_max_gpus). Same source: HGET inside - // the Lua. - let limit = match array.get(3) { - Some(Value::Int(n)) => *n, - other => { - return Err(AccountingError::Unexpected(format!( - "expected int limit, got {other:?}" - ))) - } - }; - Ok(BookOutcome::LimitExceeded { - table, - current, - limit, - }) - } -} diff --git a/rust/crates/scheduler/src/accounting/store.rs b/rust/crates/scheduler/src/accounting/store.rs new file mode 100644 index 0000000000..561b66f753 --- /dev/null +++ b/rust/crates/scheduler/src/accounting/store.rs @@ -0,0 +1,786 @@ +// Copyright Contributors to the OpenCue Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under +// the License. + +//! In-process accounting store. Single source of truth for the scheduler's booking +//! decisions, replacing the Redis-backed counters. See +//! `docs/_docs/developer-guide/scheduler-accounting.md` for the full design. +//! +//! Only the three enforced vertices are tracked: subscription (`burst`), folder +//! (`int_max_cores`/`int_max_gpus`) and job (`int_max_cores`/`int_max_gpus`). The +//! booking enforcement this replaces incremented layer and point counters too, but +//! never read them, so they are not kept here. +//! +//! Concurrency: one `Mutex` guards the whole state. Every critical section is pure +//! in-memory arithmetic (no I/O, no `.await`), so contention is negligible at this +//! scale and a single lock keeps the three-vertex check-and-increment trivially atomic. +//! +//! ## Pending carry-forward (the hard-cap invariant) +//! +//! The recompute reconciles booked counters by absolute-overwrite from a `SUM(proc)` +//! snapshot, which is read OUTSIDE the lock and is therefore stale: it can miss a proc +//! that committed after the read. To stop the overwrite from erasing such a booking (the +//! only way to under-count → over-book a hard cap), each booking is carried as "pending" +//! until a recompute whose snapshot provably includes its `proc` row has run: +//! +//! - `book` adds the delta to the live counter and to the **in-flight** bucket (proc not +//! yet committed → in no snapshot yet → always carried). +//! - `confirm` (proc committed + RQD launched) moves the delta from in-flight to a +//! **settled** bucket, double-buffered by recompute-epoch parity. +//! - `rollback` (dispatch failed) removes the delta from the live counter and in-flight. +//! - The recompute bumps the epoch under the lock *before* its snapshot read, then on +//! overwrite sets `counter = snapshot + in-flight + settled[both buckets]` and clears +//! only the settled bucket from *before* this epoch — confirms that raced the snapshot +//! read land in the other bucket and survive. Double-counting a booking that is in both +//! the snapshot and a settled bucket is harmless (over-count → under-book → safe). + +use std::collections::HashMap; +use std::sync::Mutex; + +use uuid::Uuid; + +use crate::accounting::booking_delta::BookingDelta; +use crate::models::CoreSize; + +/// `-1` is the "unlimited" sentinel on folder/job `int_max_cores`/`int_max_gpus`; +/// the `> 0` guard in [`over_cap`] treats any non-positive cap as unbounded. +const UNLIMITED: i64 = -1; + +/// Booked counters for one accounting vertex. +/// +/// `cores`/`gpus` is the live booked total the cap check reads. The other fields track +/// the portion still in flight so the recompute's absolute overwrite cannot erase a +/// booking whose `proc` row is not yet in its (stale) snapshot. See the module docs. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +struct Counter { + cores: i64, + gpus: i64, + /// Booked, proc INSERT not yet confirmed committed. In no snapshot → always carried. + inflight_cores: i64, + inflight_gpus: i64, + /// Confirmed bookings not yet provably in a recompute snapshot, double-buffered by + /// recompute-epoch parity. Index `e % 2` is written by confirms during epoch `e`. + settled_cores: [i64; 2], + settled_gpus: [i64; 2], +} + +impl Counter { + /// Pending delta to carry across an overwrite: in-flight (uncommitted) bookings plus + /// the settled bucket `keep` — the one holding confirms that raced *this* pass's + /// snapshot read. The other settled bucket holds confirms from before the snapshot read, + /// which are already in the snapshot, so it is NOT carried (and is cleared afterward). + fn carried_cores(&self, keep: usize) -> i64 { + self.inflight_cores + self.settled_cores[keep] + } + fn carried_gpus(&self, keep: usize) -> i64 { + self.inflight_gpus + self.settled_gpus[keep] + } + + /// `book`: add to the live total and the in-flight bucket. + fn add_booking(&mut self, dc: i64, dg: i64) { + self.cores += dc; + self.gpus += dg; + self.inflight_cores += dc; + self.inflight_gpus += dg; + } + + /// `confirm`: move from in-flight to the current epoch's settled bucket (live unchanged). + fn settle(&mut self, dc: i64, dg: i64, bucket: usize) { + self.inflight_cores -= dc; + self.inflight_gpus -= dg; + self.settled_cores[bucket] += dc; + self.settled_gpus[bucket] += dg; + } + + /// `rollback`: undo a `book` (live total and in-flight). + fn remove_booking(&mut self, dc: i64, dg: i64) { + self.cores -= dc; + self.gpus -= dg; + self.inflight_cores -= dc; + self.inflight_gpus -= dg; + } +} + +/// Folder/job cap pair, in cores. `-1` means unlimited (see [`UNLIMITED`]). +#[derive(Default, Clone, Copy, Debug)] +struct MaxCap { + max_cores: i64, + max_gpus: i64, +} + +#[derive(Default)] +struct Inner { + sub: HashMap<(Uuid, Uuid), Counter>, + folder: HashMap, + job: HashMap, + /// Subscription burst caps, in cores. Missing == 0 == "reject all" (matches the + /// Cuebot `IS_SHOW_OVER_BURST` convention and fails closed before the bootstrap seed). + sub_burst: HashMap<(Uuid, Uuid), i64>, + folder_caps: HashMap, + job_caps: HashMap, + /// Monotonic recompute epoch. Bumped under the lock at the start of each recompute so + /// `confirm` tags the correct settled bucket relative to the in-flight snapshot read. + epoch: u64, +} + +/// Outcome of a [`Store::book`] call. +pub enum BookOutcome { + /// Booking accepted; counters incremented and the delta recorded as pending. + Applied, + /// Rejected because a hard cap would be exceeded. `table` is the vertex label the + /// rejection is attributed to (`subscription`/`folder`/`job`/`folder_gpus`/`job_gpus`). + LimitExceeded { + table: &'static str, + current: i64, + limit: i64, + }, +} + +/// A single enforced cap change, delivered live by the PG `acct_limit_change` listener +/// or applied in bulk by the limit reseed. Values are in cores (GPUs pass through). +#[derive(Debug, Clone)] +pub enum LimitChange { + SubBurst { + show_id: Uuid, + alloc_id: Uuid, + burst: i64, + }, + FolderMaxCores { + folder_id: Uuid, + max_cores: i64, + }, + FolderMaxGpus { + folder_id: Uuid, + max_gpus: i64, + }, + JobMaxCores { + job_id: Uuid, + max_cores: i64, + }, + JobMaxGpus { + job_id: Uuid, + max_gpus: i64, + }, +} + +/// Aggregated `SUM(proc)` totals for one recompute pass, already converted to cores and +/// overlaid on the zero-baseline (every enumerable key present, drained keys carrying 0). +#[derive(Default, Debug)] +pub struct CounterSnapshot { + pub sub: HashMap<(Uuid, Uuid), (i64, i64)>, + pub folder: HashMap, + pub job: HashMap, +} + +/// Process-wide in-memory accounting state. +#[derive(Default)] +pub struct Store { + inner: Mutex, +} + +/// `cur + delta` exceeds `cap` only when `cap` is a real (positive) ceiling. A cap of +/// `0` on a subscription burst means "reject all"; on a folder/job max it means unset → +/// treated as unlimited, matching the legacy `> 0` guard. `enforce_zero` selects which +/// convention applies. +fn over_cap(cur: i64, delta: i64, cap: i64, enforce_zero: bool) -> bool { + if enforce_zero { + cur + delta > cap + } else { + cap > 0 && cur + delta > cap + } +} + +impl Store { + pub fn new() -> Self { + Self::default() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner.lock().unwrap_or_else(|p| p.into_inner()) + } + + /// Hot-path booking: atomically check subscription burst and folder/job core/GPU + /// caps, and on success increment all three vertices and record the delta as in-flight. + pub fn book(&self, delta: &BookingDelta) -> BookOutcome { + let dc = delta.core_delta; + let dg = i64::from(delta.gpu_delta); + let mut inner = self.lock(); + + if dc > 0 { + let cur_sub = inner + .sub + .get(&(delta.show_id, delta.alloc_id)) + .map_or(0, |c| c.cores); + let burst = inner + .sub_burst + .get(&(delta.show_id, delta.alloc_id)) + .copied() + .unwrap_or(0); + // Subscription burst enforces 0 as "reject all". + if over_cap(cur_sub, dc, burst, true) { + return BookOutcome::LimitExceeded { + table: "subscription", + current: cur_sub, + limit: burst, + }; + } + + let cur_folder = inner.folder.get(&delta.folder_id).map_or(0, |c| c.cores); + let folder_max = inner.folder_caps.get(&delta.folder_id).map_or(0, |c| c.max_cores); + if over_cap(cur_folder, dc, folder_max, false) { + return BookOutcome::LimitExceeded { + table: "folder", + current: cur_folder, + limit: folder_max, + }; + } + + let cur_job = inner.job.get(&delta.job_id).map_or(0, |c| c.cores); + let job_max = inner.job_caps.get(&delta.job_id).map_or(0, |c| c.max_cores); + if over_cap(cur_job, dc, job_max, false) { + return BookOutcome::LimitExceeded { + table: "job", + current: cur_job, + limit: job_max, + }; + } + } + + if dg > 0 { + let cur_folder_gpu = inner.folder.get(&delta.folder_id).map_or(0, |c| c.gpus); + let folder_gpu_max = inner.folder_caps.get(&delta.folder_id).map_or(0, |c| c.max_gpus); + if over_cap(cur_folder_gpu, dg, folder_gpu_max, false) { + return BookOutcome::LimitExceeded { + table: "folder_gpus", + current: cur_folder_gpu, + limit: folder_gpu_max, + }; + } + + let cur_job_gpu = inner.job.get(&delta.job_id).map_or(0, |c| c.gpus); + let job_gpu_max = inner.job_caps.get(&delta.job_id).map_or(0, |c| c.max_gpus); + if over_cap(cur_job_gpu, dg, job_gpu_max, false) { + return BookOutcome::LimitExceeded { + table: "job_gpus", + current: cur_job_gpu, + limit: job_gpu_max, + }; + } + } + + inner.sub.entry((delta.show_id, delta.alloc_id)).or_default().add_booking(dc, dg); + inner.folder.entry(delta.folder_id).or_default().add_booking(dc, dg); + inner.job.entry(delta.job_id).or_default().add_booking(dc, dg); + BookOutcome::Applied + } + + /// Booking settled (proc committed + RQD launched): move the delta from in-flight to + /// the current epoch's settled bucket. The live counter is unchanged. Exactly one of + /// `confirm`/`rollback` runs per `book`. + pub fn confirm(&self, delta: &BookingDelta) { + let dc = delta.core_delta; + let dg = i64::from(delta.gpu_delta); + let mut inner = self.lock(); + let bucket = (inner.epoch % 2) as usize; + inner.sub.entry((delta.show_id, delta.alloc_id)).or_default().settle(dc, dg, bucket); + inner.folder.entry(delta.folder_id).or_default().settle(dc, dg, bucket); + inner.job.entry(delta.job_id).or_default().settle(dc, dg, bucket); + } + + /// Booking failed before launch: undo the live increment and the in-flight delta. + pub fn rollback(&self, delta: &BookingDelta) { + let dc = delta.core_delta; + let dg = i64::from(delta.gpu_delta); + let mut inner = self.lock(); + inner.sub.entry((delta.show_id, delta.alloc_id)).or_default().remove_booking(dc, dg); + inner.folder.entry(delta.folder_id).or_default().remove_booking(dc, dg); + inner.job.entry(delta.job_id).or_default().remove_booking(dc, dg); + } + + /// Apply a release delta (negative cores/gpus) from the Cuebot `acct_release` NOTIFY. + /// Unconditional and pending-free: releases are for long-settled bookings. + pub fn apply_release(&self, delta: &BookingDelta) { + let dc = delta.core_delta; + let dg = i64::from(delta.gpu_delta); + let mut inner = self.lock(); + if let Some(c) = inner.sub.get_mut(&(delta.show_id, delta.alloc_id)) { + c.cores += dc; + c.gpus += dg; + } + if let Some(c) = inner.folder.get_mut(&delta.folder_id) { + c.cores += dc; + c.gpus += dg; + } + if let Some(c) = inner.job.get_mut(&delta.job_id) { + c.cores += dc; + c.gpus += dg; + } + } + + /// Begin a recompute pass: bump the epoch under the lock and return the pre-bump value. + /// Must be called BEFORE reading the `SUM(proc)` snapshot, and the returned epoch passed + /// to [`Store::overwrite_counters`]. This is what lets a `confirm` racing the snapshot + /// read land in a settled bucket the overwrite will not clear. + pub fn begin_recompute(&self) -> u64 { + let mut inner = self.lock(); + let g = inner.epoch; + inner.epoch = inner.epoch.wrapping_add(1); + g + } + + /// Recompute backstop: overwrite every booked total with `snapshot + carried pending`, + /// then clear the settled bucket that pre-dates this pass's snapshot read (`epoch % 2`). + /// + /// `snapshot` already carries the zero-baseline (drained keys present with `0`). A + /// confirm that raced the snapshot read tagged the *other* bucket (the epoch was bumped + /// before the read), so it is carried, not cleared — closing the straddle hole. Keys + /// absent from `snapshot` (e.g. FINISHED jobs no longer enumerable) keep their value; + /// only their stale settled bucket is cleared. + pub fn overwrite_counters(&self, snapshot: &CounterSnapshot, epoch: u64) { + let clear = (epoch % 2) as usize; + let keep = 1 - clear; + let mut inner = self.lock(); + for (&k, &(cores, gpus)) in &snapshot.sub { + let c = inner.sub.entry(k).or_default(); + c.cores = cores + c.carried_cores(keep); + c.gpus = gpus + c.carried_gpus(keep); + } + for (&k, &(cores, gpus)) in &snapshot.folder { + let c = inner.folder.entry(k).or_default(); + c.cores = cores + c.carried_cores(keep); + c.gpus = gpus + c.carried_gpus(keep); + } + for (&k, &(cores, gpus)) in &snapshot.job { + let c = inner.job.entry(k).or_default(); + c.cores = cores + c.carried_cores(keep); + c.gpus = gpus + c.carried_gpus(keep); + } + // Clear the pre-snapshot settled bucket across all keys (confirms older than this + // pass's snapshot read are provably reflected in the snapshot now). + for c in inner.sub.values_mut() { + c.settled_cores[clear] = 0; + c.settled_gpus[clear] = 0; + } + for c in inner.folder.values_mut() { + c.settled_cores[clear] = 0; + c.settled_gpus[clear] = 0; + } + for c in inner.job.values_mut() { + c.settled_cores[clear] = 0; + c.settled_gpus[clear] = 0; + } + } + + /// One-shot absolute seed of a show's booked counters when it is flipped to + /// scheduler-managed, BEFORE it enters the managed-shows cache. At this point the show + /// has no scheduler bookings (the hot path no-ops for unpublished shows), so there is no + /// in-flight/settled pending and no concurrent booking on its keys. Unlike the recompute + /// overwrite this does NOT bump the epoch or touch the settled buckets, so it never + /// interferes with the single recompute driver's begin/overwrite sequencing. Setting the + /// live counter directly is what closes the managed-flip over-book window: the first + /// booking after publish enforces against real usage, not against 0 (= full burst free). + pub fn seed_show_booked(&self, snapshot: &CounterSnapshot) { + let mut inner = self.lock(); + for (&k, &(cores, gpus)) in &snapshot.sub { + let c = inner.sub.entry(k).or_default(); + c.cores = cores; + c.gpus = gpus; + } + for (&k, &(cores, gpus)) in &snapshot.folder { + let c = inner.folder.entry(k).or_default(); + c.cores = cores; + c.gpus = gpus; + } + for (&k, &(cores, gpus)) in &snapshot.job { + let c = inner.job.entry(k).or_default(); + c.cores = cores; + c.gpus = gpus; + } + } + + /// Bulk-set caps from the limit reseed (PG → store). Values are in cores; `-1` + /// unlimited sentinels are preserved by the caller. + pub fn set_caps( + &self, + subs: impl IntoIterator, + folders: impl IntoIterator, + jobs: impl IntoIterator, + ) { + let mut inner = self.lock(); + for (show_id, alloc_id, burst) in subs { + inner.sub_burst.insert((show_id, alloc_id), burst); + } + for (folder_id, max_cores, max_gpus) in folders { + inner.folder_caps.insert( + folder_id, + MaxCap { + max_cores, + max_gpus, + }, + ); + } + for (job_id, max_cores, max_gpus) in jobs { + inner.job_caps.insert( + job_id, + MaxCap { + max_cores, + max_gpus, + }, + ); + } + } + + /// Apply a single live cap change from the `acct_limit_change` listener. A single-field + /// change on an unseen vertex defaults the other dimension to `UNLIMITED` rather than 0 + /// so it cannot accidentally under-enforce before the next full limit reseed seeds both. + pub fn apply_limit_change(&self, change: &LimitChange) { + let mut inner = self.lock(); + match *change { + LimitChange::SubBurst { + show_id, + alloc_id, + burst, + } => { + inner.sub_burst.insert((show_id, alloc_id), burst); + } + LimitChange::FolderMaxCores { + folder_id, + max_cores, + } => { + inner.folder_caps.entry(folder_id).or_insert(MaxCap::unlimited()).max_cores = + max_cores; + } + LimitChange::FolderMaxGpus { + folder_id, + max_gpus, + } => { + inner.folder_caps.entry(folder_id).or_insert(MaxCap::unlimited()).max_gpus = + max_gpus; + } + LimitChange::JobMaxCores { job_id, max_cores } => { + inner.job_caps.entry(job_id).or_insert(MaxCap::unlimited()).max_cores = max_cores; + } + LimitChange::JobMaxGpus { job_id, max_gpus } => { + inner.job_caps.entry(job_id).or_insert(MaxCap::unlimited()).max_gpus = max_gpus; + } + } + } + + /// Live booked cores for a job (E-PVM placement snapshot in the matcher). 0 if unseen. + pub fn job_cores_in_use(&self, job_id: Uuid) -> i64 { + self.lock().job.get(&job_id).map_or(0, |c| c.cores) + } + + /// `(booked_cores, burst)` for a subscription (matcher over-burst pre-check). Both in + /// cores; missing entries read as 0. + pub fn sub_counters(&self, show_id: Uuid, alloc_id: Uuid) -> (i64, i64) { + let inner = self.lock(); + let booked = inner.sub.get(&(show_id, alloc_id)).map_or(0, |c| c.cores); + let burst = inner.sub_burst.get(&(show_id, alloc_id)).copied().unwrap_or(0); + (booked, burst) + } + + /// Snapshot of booked `(cores, gpus)` per vertex, for the stress-test accounting audit. + #[cfg(feature = "stress-tests")] + pub fn audit_snapshot(&self) -> AuditSnapshot { + let inner = self.lock(); + let booked = |c: &Counter| (c.cores, c.gpus); + AuditSnapshot { + sub: inner.sub.iter().map(|(&k, c)| (k, booked(c))).collect(), + folder: inner.folder.iter().map(|(&k, c)| (k, booked(c))).collect(), + job: inner.job.iter().map(|(&k, c)| (k, booked(c))).collect(), + } + } +} + +impl MaxCap { + fn unlimited() -> Self { + MaxCap { + max_cores: UNLIMITED, + max_gpus: UNLIMITED, + } + } +} + +/// Convert a PG centicore total to cores. Booked sums are non-negative. +pub fn centicores_to_cores(centicores: i64) -> i64 { + i64::from(CoreSize::from_multiplied(centicores).value()) +} + +/// Convert a PG centicore cap to cores, preserving the `-1` unlimited sentinel. +pub fn centicores_to_cores_cap(centicores: i64) -> i64 { + if centicores < 0 { + UNLIMITED + } else { + i64::from(CoreSize::from_multiplied_cap(centicores).value()) + } +} + +/// Booked `(cores, gpus)` per vertex key, returned by [`Store::audit_snapshot`]. +#[cfg(feature = "stress-tests")] +#[derive(Default, Debug)] +pub struct AuditSnapshot { + pub sub: HashMap<(Uuid, Uuid), (i64, i64)>, + pub folder: HashMap, + pub job: HashMap, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn delta(show: Uuid, alloc: Uuid, folder: Uuid, job: Uuid, cores: i64, gpus: i32) -> BookingDelta { + BookingDelta { + show_id: show, + alloc_id: alloc, + folder_id: folder, + job_id: job, + core_delta: cores, + gpu_delta: gpus, + } + } + + fn applied(o: BookOutcome) -> bool { + matches!(o, BookOutcome::Applied) + } + + fn ids() -> (Uuid, Uuid, Uuid, Uuid) { + (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()) + } + + #[test] + fn book_rejects_when_unseeded_burst_is_zero() { + // Missing burst == 0 == reject all (fail closed before any seed). + let store = Store::new(); + let d = delta(Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), 1, 0); + assert!(matches!( + store.book(&d), + BookOutcome::LimitExceeded { table: "subscription", .. } + )); + } + + #[test] + fn book_enforces_job_hard_cap_atomically() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 1000)], [(folder, -1, -1)], [(job, 10, -1)]); + let d = delta(show, alloc, folder, job, 6, 0); + assert!(applied(store.book(&d))); // 6 <= 10 + // Second booking of 6 would reach 12 > 10 -> rejected. No partial state. + assert!(matches!( + store.book(&d), + BookOutcome::LimitExceeded { table: "job", current: 6, limit: 10 } + )); + assert_eq!(store.job_cores_in_use(job), 6); + } + + #[test] + fn unlimited_sentinel_never_rejects() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 1_000_000)], [(folder, -1, -1)], [(job, -1, -1)]); + let d = delta(show, alloc, folder, job, 500, 4); + assert!(applied(store.book(&d))); + } + + /// Recompute that runs entirely after a confirm reconciles the counter to `SUM(proc)`. + #[test] + fn confirm_then_recompute_keeps_booked() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + let d = delta(show, alloc, folder, job, 10, 0); + assert!(applied(store.book(&d))); + store.confirm(&d); + let epoch = store.begin_recompute(); + let snap = CounterSnapshot { + job: [(job, (10, 0))].into_iter().collect(), + ..Default::default() + }; + store.overwrite_counters(&snap, epoch); + assert_eq!(store.job_cores_in_use(job), 10); + } + + #[test] + fn rollback_undoes_book() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + let d = delta(show, alloc, folder, job, 10, 0); + assert!(applied(store.book(&d))); + store.rollback(&d); + assert_eq!(store.job_cores_in_use(job), 0); + let epoch = store.begin_recompute(); + let snap = CounterSnapshot { + job: [(job, (0, 0))].into_iter().collect(), + ..Default::default() + }; + store.overwrite_counters(&snap, epoch); + assert_eq!(store.job_cores_in_use(job), 0); + } + + /// In-flight booking (booked, not yet confirmed) absent from the snapshot must survive. + #[test] + fn recompute_carries_forward_in_flight_booking() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, 20, -1)]); + let d = delta(show, alloc, folder, job, 8, 0); + assert!(applied(store.book(&d))); // booked, still in-flight (not confirmed) + + let epoch = store.begin_recompute(); + let snap = CounterSnapshot { + sub: [((show, alloc), (0, 0))].into_iter().collect(), + folder: [(folder, (0, 0))].into_iter().collect(), + job: [(job, (0, 0))].into_iter().collect(), + }; + store.overwrite_counters(&snap, epoch); + assert_eq!(store.job_cores_in_use(job), 8); + } + + /// THE STRADDLE HOLE (M1): a recompute reads its snapshot, THEN a booking commits and + /// is confirmed, THEN the overwrite lands. The confirm tagged the post-`begin_recompute` + /// bucket, so the overwrite must NOT erase it. Without epoch double-buffering this + /// over-books a hard cap. + #[test] + fn recompute_does_not_erase_booking_confirmed_after_snapshot_read() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, 20, -1)]); + let d = delta(show, alloc, folder, job, 8, 0); + assert!(applied(store.book(&d))); + + // Recompute begins (epoch bumped) and reads a snapshot that does NOT yet see the proc. + let epoch = store.begin_recompute(); + // Proc commits + dispatch succeeds AFTER the snapshot read -> confirm now. + store.confirm(&d); + // Overwrite lands. The confirm tagged the other settled bucket, so 8 survives. + let snap = CounterSnapshot { + sub: [((show, alloc), (0, 0))].into_iter().collect(), + folder: [(folder, (0, 0))].into_iter().collect(), + job: [(job, (0, 0))].into_iter().collect(), + }; + store.overwrite_counters(&snap, epoch); + assert_eq!(store.job_cores_in_use(job), 8, "booking confirmed after the snapshot read was erased"); + + // The following recompute (proc now visible) reconciles cleanly to the true value. + let epoch2 = store.begin_recompute(); + let snap2 = CounterSnapshot { + sub: [((show, alloc), (8 * 100 / 100, 0))].into_iter().collect(), + folder: [(folder, (8, 0))].into_iter().collect(), + job: [(job, (8, 0))].into_iter().collect(), + }; + store.overwrite_counters(&snap2, epoch2); + assert_eq!(store.job_cores_in_use(job), 8); + } + + #[test] + fn release_decrements_unconditionally() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + let d = delta(show, alloc, folder, job, 10, 0); + assert!(applied(store.book(&d))); + store.confirm(&d); + // Cuebot sends a release as a negative delta; the listener applies it as-is. + store.apply_release(&delta(show, alloc, folder, job, -10, 0)); + assert_eq!(store.job_cores_in_use(job), 0); + } + + /// A dropped release NOTIFY leaves the counter reading high (safe: under-book); the next + /// recompute heals it down to `SUM(proc)`. Must never strand a hard cap as over-booked. + #[test] + fn missed_release_heals_via_recompute() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + let d = delta(show, alloc, folder, job, 10, 0); + assert!(applied(store.book(&d))); + store.confirm(&d); + // Settle the booking through one full recompute so it leaves the pending buckets. + let e1 = store.begin_recompute(); + store.overwrite_counters( + &CounterSnapshot { + sub: [((show, alloc), (10, 0))].into_iter().collect(), + folder: [(folder, (10, 0))].into_iter().collect(), + job: [(job, (10, 0))].into_iter().collect(), + }, + e1, + ); + let e2 = store.begin_recompute(); + store.overwrite_counters( + &CounterSnapshot { + sub: [((show, alloc), (10, 0))].into_iter().collect(), + folder: [(folder, (10, 0))].into_iter().collect(), + job: [(job, (10, 0))].into_iter().collect(), + }, + e2, + ); + // Frame completed and Cuebot deleted the proc, but the release NOTIFY was dropped: + // the store still reads 10 (high → under-book, never over-book). + assert_eq!(store.job_cores_in_use(job), 10); + // Recompute sees the drained proc (SUM = 0) and heals the counter down. + let e3 = store.begin_recompute(); + store.overwrite_counters( + &CounterSnapshot { + sub: [((show, alloc), (0, 0))].into_iter().collect(), + folder: [(folder, (0, 0))].into_iter().collect(), + job: [(job, (0, 0))].into_iter().collect(), + }, + e3, + ); + assert_eq!(store.job_cores_in_use(job), 0); + } + + /// Managed-flip seed: a show flipped to managed already has live Cuebot procs. After + /// seeding the booked counters from `SUM(proc)`, the first scheduler booking enforces + /// against real usage, not against 0. Without the seed it would over-book the burst. + #[test] + fn managed_flip_seed_prevents_overbook() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + // Cuebot already has 90 cores booked on this (show, alloc) at flip time. + let seed = CounterSnapshot { + sub: [((show, alloc), (90, 0))].into_iter().collect(), + folder: [(folder, (90, 0))].into_iter().collect(), + job: [(job, (90, 0))].into_iter().collect(), + }; + store.seed_show_booked(&seed); + assert_eq!(store.sub_counters(show, alloc), (90, 100)); + // A 20-core booking would reach 110 > 100 burst -> must reject (not over-book). + assert!(matches!( + store.book(&delta(show, alloc, folder, job, 20, 0)), + BookOutcome::LimitExceeded { table: "subscription", current: 90, limit: 100 } + )); + // A 10-core booking fits exactly at the burst. + assert!(applied(store.book(&delta(show, alloc, folder, job, 10, 0)))); + } + + #[test] + fn live_limit_change_updates_cap() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, 50, -1)]); + let d = delta(show, alloc, folder, job, 40, 0); + assert!(applied(store.book(&d))); // 40 <= 50 + // Operator lowers the hard cap to 30 live; further bookings must reject. + store.apply_limit_change(&LimitChange::JobMaxCores { + job_id: job, + max_cores: 30, + }); + let d2 = delta(show, alloc, folder, job, 1, 0); + assert!(matches!( + store.book(&d2), + BookOutcome::LimitExceeded { table: "job", .. } + )); + } +} diff --git a/rust/crates/scheduler/src/cluster.rs b/rust/crates/scheduler/src/cluster.rs index 61f1017a2b..da366bf725 100644 --- a/rust/crates/scheduler/src/cluster.rs +++ b/rust/crates/scheduler/src/cluster.rs @@ -313,7 +313,7 @@ impl ClusterFeed { match cluster.ttype.as_str() { // Each alloc tag becomes its own cluster. Carry pk_alloc // through Tag so the matcher can snapshot the - // (show, alloc) subscription burst from Redis before + // (show, alloc) subscription burst from the accounting store before // host checkout (see `MatchingService::process_layer`). "ALLOC" => { let alloc_id = cluster.alloc_id.as_deref().map(parse_uuid); diff --git a/rust/crates/scheduler/src/cluster_key.rs b/rust/crates/scheduler/src/cluster_key.rs index 9b8ec73ca4..33bda36e40 100644 --- a/rust/crates/scheduler/src/cluster_key.rs +++ b/rust/crates/scheduler/src/cluster_key.rs @@ -38,7 +38,7 @@ pub struct Tag { /// `TagType::Alloc` cluster tag from the database. Populated by /// `cluster.rs::load_clusters` on the `"ALLOC"` arm and consumed by /// `MatchingService::process_layer` to read the per-(show, alloc) - /// subscription burst snapshot from Redis before host checkout. + /// subscription burst snapshot from the accounting store before host checkout. /// /// `None` for non-alloc tags (manual / hostname / hardware) and for /// CLI-built tags where the str_tag → pk_alloc mapping isn't resolved diff --git a/rust/crates/scheduler/src/config/mod.rs b/rust/crates/scheduler/src/config/mod.rs index 0422d32a8e..bc8343fd4f 100644 --- a/rust/crates/scheduler/src/config/mod.rs +++ b/rust/crates/scheduler/src/config/mod.rs @@ -49,61 +49,31 @@ pub struct Config { #[derive(Debug, Deserialize, Clone)] #[serde(default)] pub struct AccountingConfig { - pub redis: RedisConfig, - /// Cadence at which booked counters are reseeded from `SUM(proc)` to both - /// the PG accounting tables and Redis (under the `acct:seq` CAS guard). + /// Cadence at which booked counters are reconciled from `SUM(proc)` into the in-memory + /// store (and the PG accounting tables for CueGUI). This is the primary utilization + /// backstop for releases now that the live `acct_release` NOTIFY feeds the store + /// between ticks, so it runs tighter than the legacy Redis cadence. #[serde(with = "humantime_serde")] pub recompute_interval: Duration, - /// Cadence at which limit fields (subscription burst, folder/job/point caps) - /// are reseeded from PG accounting tables to Redis. + /// Cadence at which enforced caps (subscription burst, folder/job max cores+gpus) are + /// reseeded from PG into the store. Backstop for any missed `acct_limit_change` NOTIFY. #[serde(with = "humantime_serde")] pub limit_reseed_interval: Duration, /// TTL of the in-process `b_scheduler_managed=true` show-id cache. #[serde(with = "humantime_serde")] pub managed_shows_ttl: Duration, - /// Maximum CAS retries per reseed cycle before giving up and waiting for the - /// next cycle (per design §2.4). - pub cas_max_retries: u32, } impl Default for AccountingConfig { fn default() -> Self { Self { - redis: RedisConfig::default(), - recompute_interval: Duration::from_secs(120), + recompute_interval: Duration::from_secs(15), limit_reseed_interval: Duration::from_secs(300), managed_shows_ttl: Duration::from_secs(30), - cas_max_retries: 3, } } } -#[derive(Debug, Deserialize, Clone)] -#[serde(default)] -pub struct RedisConfig { - pub enabled: bool, - pub host: String, - pub port: u16, - pub pool_size: u32, -} - -impl Default for RedisConfig { - fn default() -> Self { - Self { - enabled: false, - host: "localhost".to_string(), - port: 6379, - pool_size: 20, - } - } -} - -impl RedisConfig { - pub fn url(&self) -> String { - format!("redis://{}:{}/", self.host, self.port) - } -} - #[derive(Debug, Deserialize, Clone)] #[serde(default)] pub struct LoggingConfig { diff --git a/rust/crates/scheduler/src/dao/job_dao.rs b/rust/crates/scheduler/src/dao/job_dao.rs index d1a087efbc..f4082c8019 100644 --- a/rust/crates/scheduler/src/dao/job_dao.rs +++ b/rust/crates/scheduler/src/dao/job_dao.rs @@ -88,7 +88,7 @@ static QUERY_PENDING_BY_SHOW_FACILITY_TAG: &str = r#" -- so we sum it directly. Each CTE is scoped to this show ($1) via i_proc_pkshow and -- mirrors the recompute aggregation (so the value equals a fresher copy of the PG -- column), and joins on indexed pk_host / pk_job. Gating on stale PG would otherwise --- (a) over-fetch jobs for caps that are full in Redis -> wasted rejections, and worse +-- (a) over-fetch jobs for caps that are full in the accounting store -> wasted rejections, and worse -- (b) FALSE-EXCLUDE: a frame completes and frees burst live, but the lagged PG column -- stays high for up to a cycle, dropping the show/folder/job from the fetch and starving -- its (esp. low-priority) jobs until the next recompute. diff --git a/rust/crates/scheduler/src/metrics/mod.rs b/rust/crates/scheduler/src/metrics/mod.rs index a7f9a07f1f..425a7533b9 100644 --- a/rust/crates/scheduler/src/metrics/mod.rs +++ b/rust/crates/scheduler/src/metrics/mod.rs @@ -68,13 +68,13 @@ lazy_static! { // Accounting metrics from accounting/mod.rs + dispatcher/actor.rs // // Labeled by the table whose cap was hit (subscription / folder / job). Tracks - // dispatch attempts that paid for a Redis Lua round-trip only to be rejected by - // the Lua cap check. Used to decide whether the pre-CheckOut pre-check + // dispatch attempts that reached the accounting check only to be rejected by + // it. Used to decide whether the pre-CheckOut pre-check // optimization described in `pipeline/matcher.rs::process_layer` is worth // implementing. pub static ref ACCOUNTING_LIMIT_EXCEEDED_TOTAL: CounterVec = register_counter_vec!( "scheduler_accounting_limit_exceeded_total", - "Dispatch attempts rejected by the Redis Lua cap check, labeled by table", + "Dispatch attempts rejected by the accounting cap check, labeled by table", &["table"] ) .expect("Failed to register accounting_limit_exceeded_total counter"); diff --git a/rust/crates/scheduler/src/models/core_size.rs b/rust/crates/scheduler/src/models/core_size.rs index 473eb1d2e9..f7f357f7ca 100644 --- a/rust/crates/scheduler/src/models/core_size.rs +++ b/rust/crates/scheduler/src/models/core_size.rs @@ -73,8 +73,8 @@ impl CoreSize { /// Like `from_multiplied` but preserves negative sentinels. Cap fields /// (`folder_resource.int_max_cores`, `job_resource.int_max_cores`) use `-1` as the /// "unlimited" sentinel; the unguarded division would truncate that to 0. The Lua - /// script's `> 0` guard handles either, but preserving `-1` faithfully keeps - /// `redis-cli` output meaningful for operators. + /// script's `> 0` guard handles either, but preserving `-1` faithfully keeps cap + /// output meaningful for operators. pub fn from_multiplied_cap(centicores: impl Into) -> CoreSize { let c = centicores.into(); if c < 0 { diff --git a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs index a98fabe11f..f82e47743d 100644 --- a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs +++ b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs @@ -255,7 +255,7 @@ impl RqdDispatcherService { // caller-supplied `job_cores_in_use` so each frame's reservation is // clamped against the job's *remaining* cap, mirroring Cuebot's // VirtualProc.build. Without this, a single threadable frame reserves a - // whole fat host (e.g. 32 cores) and the Lua rejects it forever against + // whole fat host (e.g. 32 cores) and the accounting check rejects it forever against // a small job cap (e.g. 16), wedging the job at zero booked. let mut job_cores_booked: i32 = 0; @@ -275,7 +275,7 @@ impl RqdDispatcherService { // all-but-N), so a raw `remaining < frame.min_cores.value()` // comparison would slip past the check (e.g. `0 < 0` is false) // when a capped job has `remaining == 0`, then build a host-sized - // reservation that the Lua rejects forever. Non-threadable frames + // reservation that the accounting check rejects forever. Non-threadable frames // always reserve exactly one core regardless of the sentinel. let frame_min_cores = if frame.threadable { Self::calculate_cores_requested(frame.min_cores, last_host_version.total_cores) @@ -285,7 +285,7 @@ impl RqdDispatcherService { }; // If the job can't fit even one minimum-sized frame, it's at its // cap: stop the layer here rather than reserving a sub-minimum - // proc or busy-looping on guaranteed Lua rejections. + // proc or busy-looping on guaranteed accounting rejections. if remaining < frame_min_cores { debug!( "({dispatch_id}) Job {} at core cap (max={}, in_use={}, booked_here={}); \ @@ -439,34 +439,6 @@ impl RqdDispatcherService { break; } - DispatchVirtualProcError::AccountingUnavailable(msg) => { - // Transient: Redis is unreachable. Per design §4.3 the scheduler - // stops dispatching until Redis recovers - break the per-host - // loop and let the matcher's outer loop handle the next host - // (which will also fail until Redis is back). - warn!( - "({dispatch_id}) Accounting unavailable for frame {}: {}", - frame_str, msg - ); - last_error = Some(DispatchError::Failure(miette!( - "Accounting unavailable: {msg}" - ))); - break; - } - DispatchVirtualProcError::AccountingUnexpected(msg) => { - // Programmer/protocol error from the accounting layer - should - // not happen in steady state. Logged at error level so ops can - // alert on it separately from the transient unavailable case. - // Break the per-host loop and surface as a dispatch failure. - error!( - "({dispatch_id}) Accounting internal error for frame {}: {}", - frame_str, msg - ); - last_error = Some(DispatchError::Failure(miette!( - "Accounting internal error: {msg}" - ))); - break; - } } } } @@ -539,24 +511,22 @@ impl RqdDispatcherService { ) -> Result<(Host, CoreSize), DispatchVirtualProcError> { trace!("({dispatch_id}) Built virtual proc {}", virtual_proc); // `cores_reserved` is centicores (CoreSizeWithMultiplier); convert to cores for - // both the Redis booking delta (Redis = cores, design §0 unit invariant) and the + // both the booking delta (store = cores, design §0 unit invariant) and the // per-cluster allocation_capacity comparison below (which is already CoreSize). let cores_reserved: CoreSize = virtual_proc.cores_reserved.into(); - // Build the booking delta once - used by Redis apply and by any compensation rollback. + // Build the booking delta once - used by the booking apply and by any compensation rollback. let delta = BookingDelta { show_id: virtual_proc.show_id, alloc_id: virtual_proc.alloc_id, folder_id: virtual_proc.folder_id, job_id: virtual_proc.job_id, - layer_id: virtual_proc.layer_id, - dept_id: virtual_proc.dept_id, core_delta: i64::from(cores_reserved.value()), gpu_delta: virtual_proc.gpus_reserved as i32, }; // Per-cluster host accounting check - this dispatcher iteration may have already - // consumed cores from the allocation above what Redis knows about. + // consumed cores from the allocation above what the store knows about. if cores_reserved > allocation_capacity { return Err(DispatchVirtualProcError::AllocationOverBurst( DispatchError::AllocationOverBurst(host.alloc_name.clone()), @@ -564,17 +534,18 @@ impl RqdDispatcherService { } let new_allocation_capacity = allocation_capacity - virtual_proc.cores_reserved.into(); - // Redis Lua first (design §2.1) - atomic check + 5×HINCRBY + INCR seq. Limit - // exceeded here means a subscription/folder/job cap would be breached; we bail - // before opening a transaction. - let booking_response = self.accounting.apply_booking(&delta).await; - booking_response.map_err(|err| match err { + // In-memory accounting check + increment first - atomic across subscription / + // folder / job under one lock. Limit exceeded here means one of those hard caps + // would be breached; we bail before opening a transaction. `Booking` records + // whether the delta was applied (managed show) so confirm/rollback can no-op for + // Cuebot-managed shows and survive a mid-dispatch managed flip. + let booking = self.accounting.apply_booking(&delta).map_err(|err| match err { AccountingError::LimitExceeded { table, current, limit, } => { - // Tracks "wasted" Redis Lua calls — see + // Tracks "wasted" booking attempts — see // `pipeline/matcher.rs::process_layer` for the trade-off this metric // exists to measure. metrics::increment_accounting_limit_exceeded(&table); @@ -584,30 +555,21 @@ impl RqdDispatcherService { )), ) } - AccountingError::Unavailable(msg) => { - DispatchVirtualProcError::AccountingUnavailable(msg) - } - AccountingError::Unexpected(msg) => DispatchVirtualProcError::AccountingUnexpected(msg), - AccountingError::CasContentionExceeded { .. } => { - DispatchVirtualProcError::AccountingUnexpected( - "CAS contention exceeded".to_string(), - ) - } })?; - // From this point on, any failure before successful RQD launch must force-roll back - // the Redis booking we just applied. Closure captures `self.accounting` and `delta` - // so the rollback sites stay terse. + // From this point on, any failure before successful RQD launch must roll back the + // booking we just applied. Closure captures `self.accounting` and `booking` so the + // rollback sites stay terse. let accounting = &self.accounting; - let redis_rollback_on_error = async || { - accounting.apply_force_rollback(&delta.negated()).await; + let rollback_on_error = async || { + accounting.rollback_booking(&booking); }; // Begin a per-proc transaction for DB updates let mut proc_transaction = match begin_transaction().await { Ok(tx) => tx, Err(e) => { - redis_rollback_on_error().await; + rollback_on_error().await; return Err(DispatchVirtualProcError::FailedToStartOnDb( DispatchError::DbFailure(e), )); @@ -623,12 +585,12 @@ impl RqdDispatcherService { Ok(true) => {} Ok(false) => { let _ = proc_transaction.rollback().await; - redis_rollback_on_error().await; + rollback_on_error().await; return Err(DispatchVirtualProcError::LayerLimitReached); } Err(e) => { let _ = proc_transaction.rollback().await; - redis_rollback_on_error().await; + rollback_on_error().await; return Err(DispatchVirtualProcError::FailedToStartOnDb( DispatchError::DbFailure(e), )); @@ -650,14 +612,14 @@ impl RqdDispatcherService { } Err(err) => { let _ = proc_transaction.rollback().await; - redis_rollback_on_error().await; + rollback_on_error().await; return Err(err); } }; // Commit BEFORE the RQD call to release job_stat/layer_stat row locks immediately if let Err(e) = proc_transaction.commit().await { - redis_rollback_on_error().await; + rollback_on_error().await; return Err(DispatchVirtualProcError::FailedToStartOnDb( DispatchError::DbFailure(e), )); @@ -666,9 +628,9 @@ impl RqdDispatcherService { // When running on dry_run_mode, just log the outcome if !self.dry_run_mode { if let Err(err) = self.launch_on_rqd(&virtual_proc, &host, true).await { - // RQD launch failed after DB commit. Force-roll the Redis booking and - // compensate the DB writes (delete proc, restore host, clear frame). - redis_rollback_on_error().await; + // RQD launch failed after DB commit. Roll back the booking and compensate + // the DB writes (delete proc, restore host, clear frame). + rollback_on_error().await; self.compensate_failed_dispatch(dispatch_id, &virtual_proc, &host.name) .await; @@ -696,6 +658,10 @@ impl RqdDispatcherService { updated_host.idle_gpu_memory = ByteSize::kb(updated_resources.gpu_mem_idle as u64); updated_host.last_updated = updated_resources.last_updated; + // Dispatch fully succeeded (proc committed + RQD launched): settle the booking so + // its pending delta stops being carried forward by the recompute. + self.accounting.confirm_booking(&booking); + Ok((updated_host, new_allocation_capacity)) } @@ -869,11 +835,10 @@ impl RqdDispatcherService { /// host report reconciles them. This is acceptable because a failed proc delete /// will surface as an error log and the host report cycle corrects the drift. /// - /// **Redis accounting:** `dispatch_virtual_proc` already issued - /// `apply_force_rollback` for the booking delta before invoking this DB compensation, - /// so the Redis hashes are consistent with `proc` once both rollbacks complete. If - /// the force-rollback itself failed (logged as a warning by `apply_force_rollback`), - /// the next 2-min recompute cycle reseeds Redis from `SUM(proc)`. + /// **In-memory accounting:** `dispatch_virtual_proc` already called + /// `rollback_booking` for the booking delta before invoking this DB compensation, so + /// the store counters are consistent with `proc` once both rollbacks complete. Any + /// residual drift is reconciled by the next recompute cycle from `SUM(proc)`. async fn compensate_failed_dispatch( &self, dispatch_id: Uuid, @@ -1048,7 +1013,7 @@ impl RqdDispatcherService { // Clamp the threaded reservation to the job's remaining core cap so a // single frame never reserves more than the job is allowed (which would - // be rejected by the Lua job-cap check). `None` means unlimited. + // be rejected by the job-cap check). `None` means unlimited. // The caller guarantees `remaining >= frame.min_cores`, so this never // shrinks a reservation below the frame's minimum. match job_cores_remaining { diff --git a/rust/crates/scheduler/src/pipeline/dispatcher/error.rs b/rust/crates/scheduler/src/pipeline/dispatcher/error.rs index 51d4175659..0ffcb432e4 100644 --- a/rust/crates/scheduler/src/pipeline/dispatcher/error.rs +++ b/rust/crates/scheduler/src/pipeline/dispatcher/error.rs @@ -81,16 +81,4 @@ pub enum DispatchVirtualProcError { #[error("Resource limit exceeded")] ResourceLimitExceeded(DispatchError), - - /// Redis is unreachable / transient infra fault. Operations runbook: check Redis - /// health, scheduler should self-recover when Redis comes back. - #[error("Accounting service unavailable: {0}")] - AccountingUnavailable(String), - - /// Programmer/protocol error from the accounting layer (malformed Lua return, - /// unexpected redis error type, CAS contention budget exhausted on a hot-path - /// call). Should never fire in steady state - surfacing distinctly so ops can - /// alert on it separately from `AccountingUnavailable`. - #[error("Accounting internal error: {0}")] - AccountingUnexpected(String), } diff --git a/rust/crates/scheduler/src/pipeline/dispatcher/messages.rs b/rust/crates/scheduler/src/pipeline/dispatcher/messages.rs index 5992f4b404..b9c339a690 100644 --- a/rust/crates/scheduler/src/pipeline/dispatcher/messages.rs +++ b/rust/crates/scheduler/src/pipeline/dispatcher/messages.rs @@ -33,7 +33,7 @@ use crate::{ /// * `layer` - Layer containing frames to dispatch /// * `host` - Target host with available resources /// * `job_cores_in_use` - Cores already booked for this job (the matcher's live -/// view: the Redis snapshot taken at the start of the pass plus everything +/// view: the store snapshot taken at the start of the pass plus everything /// booked locally so far). Used to clamp each frame's reservation to the job's /// remaining `max_cores` so a single (threadable) frame can't reserve more /// than the job is allowed and get rejected forever. diff --git a/rust/crates/scheduler/src/pipeline/entrypoint.rs b/rust/crates/scheduler/src/pipeline/entrypoint.rs index 39d2bb7e0c..18035b24d5 100644 --- a/rust/crates/scheduler/src/pipeline/entrypoint.rs +++ b/rust/crates/scheduler/src/pipeline/entrypoint.rs @@ -18,7 +18,7 @@ use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tracing::{debug, error, info}; -use crate::accounting::{accounting_service, bootstrap, limit_reseed, recompute}; +use crate::accounting::{accounting_service, bootstrap, limit_reseed, listener, recompute}; use crate::cluster::{ClusterFeed, FeedMessage}; use crate::config::CONFIG; use crate::dao::JobDao; @@ -41,14 +41,16 @@ use crate::pipeline::MatchingService; /// * `Ok(())` - Scheduler completed successfully /// * `Err(miette::Error)` - Fatal error occurred during processing pub async fn run(cluster_feed: ClusterFeed) -> miette::Result<()> { - // Initialize the Redis-backed accounting service. Bootstrap reseed (limits + booked - // counters) must complete before the scheduler accepts work - see design §4.3. + // Initialize the in-memory accounting service. The blocking bootstrap seed (caps + + // booked counters) must complete before the scheduler accepts work - the store is the + // only copy of this state. let accounting = accounting_service().await?; bootstrap::run_blocking_reseed(&accounting).await?; - // TODO: gate behind leader-election when multi-scheduler lands (design §5). + // Backstops: recompute reconciles booked counters from proc; limit reseed refreshes + // caps. The live NOTIFY listener feeds releases + cap changes between ticks. N=1 only. recompute::spawn_loop(accounting.clone()); - // TODO: gate behind leader-election when multi-scheduler lands (design §5). limit_reseed::spawn_loop(accounting.clone()); + listener::spawn_loop(accounting.store().clone()); let job_fetcher = Arc::new(JobDao::new().await?); let matcher = Arc::new(MatchingService::new().await?); diff --git a/rust/crates/scheduler/src/pipeline/matcher.rs b/rust/crates/scheduler/src/pipeline/matcher.rs index 5e498af57d..c0f867181f 100644 --- a/rust/crates/scheduler/src/pipeline/matcher.rs +++ b/rust/crates/scheduler/src/pipeline/matcher.rs @@ -228,41 +228,24 @@ impl MatchingService { let initial_attempts = attempts; // E-PVM live-usage snapshot taken once at permit entry (design Branch 2a). - // Locally incremented per dispatched frame within the while-loop. - // Redis read failures degrade to 0, leaving the cap unbounded by live usage - // but still bounded by `job_max_cores`. - let initial_job_cores_in_use = match self - .accounting - .redis() - .read_job_cores_in_use(dispatch_layer.job_id) - .await - { - // Redis accounting counters are already stored in cores (the - // centicore→core conversion happens once at the reseed/recompute - // write boundaries — see `lua.rs` unit invariant), so do NOT apply - // `from_multiplied` here or the value is divided by the multiplier - // a second time. - Ok(cores) => cores as i32, - Err(err) => { - debug!( - "read_job_cores_in_use failed for job {}: {}; defaulting to 0", - dispatch_layer.job_id, err - ); - 0 - } - }; + // Locally incremented per dispatched frame within the while-loop. The store + // returns booked cores directly (already in cores, not centicores), so do NOT + // apply `from_multiplied` here. A non-managed/unseen job reads 0, leaving the cap + // unbounded by live usage but still bounded by `job_max_cores`. + let initial_job_cores_in_use = + self.accounting.job_cores_in_use(dispatch_layer.job_id) as i32; let mut local_job_cores_booked: i32 = 0; // Job-level at-cap pre-check. The job's core cap is enforced - // authoritatively by the Lua BOOK_OR_FORCE call in the dispatcher, but a + // authoritatively by the accounting check in the dispatcher, but a // job sitting at its cap would otherwise be re-checked-out and re-rejected // up to host_candidate_attempts_per_layer times every pass (see the retry // loop below). Skip it cheaply here, reusing the live `initial_job_cores_in_use` // snapshot read above. Unlike the subscription skip below, we do NOT sleep // the cluster: the job cap is per-job and sibling jobs in this cluster may - // still be dispatchable. Fail-open: a failed Redis read left + // still be dispatchable. Fail-open: a failed store read left // `initial_job_cores_in_use` at 0, so the guard simply won't fire and the - // Lua call stays authoritative. + // booking call stays authoritative. if placement::job_at_core_cap( initial_job_cores_in_use, dispatch_layer.cores_min.value(), @@ -305,27 +288,13 @@ impl MatchingService { None }; let (initial_show_cores_in_use, show_burst): (i32, i32) = match alloc_id_opt { - Some(alloc_id) => match self - .accounting - .redis() - .read_sub_counters(dispatch_layer.show_id, alloc_id) - .await - { - // Both counters are already in cores (see `lua.rs` unit - // invariant); the conversion from PG centicores happened at the - // reseed write boundary. Applying `from_multiplied` here would - // divide by the multiplier a second time (e.g. a burst of 200 - // would read back as 2). - Ok((booked, burst)) => (booked as i32, burst as i32), - Err(err) => { - debug!( - "read_sub_counters failed for show={} alloc={}: {}; \ - leaving burst unbounded, Lua will decide", - dispatch_layer.show_id, alloc_id, err - ); - (0, 0) - } - }, + Some(alloc_id) => { + // Both counters are already in cores (the conversion from PG centicores + // happens at the store-seed boundary), so do NOT apply `from_multiplied` + // here. The booking call remains authoritative on the actual decision. + let (booked, burst) = self.accounting.sub_counters(dispatch_layer.show_id, alloc_id); + (booked as i32, burst as i32) + } None => (0, 0), }; @@ -334,7 +303,7 @@ impl MatchingService { // as the empty-cluster back-off and return without scanning the host // cache or pinging the dispatcher. A stale "over burst" read costs at // most a single cluster_empty_sleep window of latency on this cluster - // and self-corrects on the next wake; the Lua booking call remains + // and self-corrects on the next wake; the booking call remains // authoritative on the actual booking. if show_burst > 0 && initial_show_cores_in_use.saturating_add(dispatch_layer.cores_min.value()) @@ -394,14 +363,14 @@ impl MatchingService { layer.show_id ); - // Subscription burst pre-check was removed in PR-C: the Lua booking call inside + // Subscription burst pre-check was removed in PR-C: the booking call inside // `dispatch_virtual_proc` is now the authoritative gate. An over-burst (show, alloc) // produces a single wasted dispatch attempt - design accepts that trade-off (see // §2.1 and the PR-C plan; restoring the optimization would require an async // validation hook on the host_cache actor or a per-process subscription mirror). // // TODO: if over-burst attempts become a measurable perf drag, add a precomputed - // per-layer Redis snapshot of (show, alloc) → bookable and consult it here. + // per-layer store snapshot of (show, alloc) → bookable and consult it here. let cores_requested = layer.cores_min; let (gate, weights) = match CONFIG.queue.host_booking_strategy { crate::config::HostBookingStrategy::Epvm { weights, .. } => ( @@ -414,12 +383,12 @@ impl MatchingService { ), }; // `show_burst` / `show_cores_in_use` come from the per-(show, alloc) - // Redis snapshot taken above. They're populated for managed shows on + // store snapshot taken above. They're populated for managed shows on // Alloc clusters (where the chosen host's allocation is deterministic // from the cluster's single Tag), and 0 otherwise — in which case // `compute_max_more`'s `> 0` guards treat the cap as "unlimited" and // E-PVM scoring loses the show-burst component of `maxMore`. The - // authoritative cap remains the Lua `BOOK_OR_FORCE` call inside the + // authoritative cap remains the accounting check inside the // dispatcher; this snapshot is an optimistic input to scoring. let profile = LayerProfile { cores_min: layer.cores_min, diff --git a/rust/crates/scheduler/src/pipeline/placement.rs b/rust/crates/scheduler/src/pipeline/placement.rs index 3f472b7eec..64eb4f088e 100644 --- a/rust/crates/scheduler/src/pipeline/placement.rs +++ b/rust/crates/scheduler/src/pipeline/placement.rs @@ -33,7 +33,7 @@ use crate::{ /// /// `job_max_cores` and `show_burst` use the OpenCue convention of `<= 0` meaning /// "unlimited" cap clamps in `compute_max_more` skip those dimensions, matching -/// the Lua `BOOK_OR_FORCE > 0` guard. +/// the accounting `> 0` cap guard. #[derive(Debug, Clone)] pub struct LayerProfile { // Floor @@ -99,7 +99,7 @@ pub fn fits_floor(host: &Host, profile: &LayerProfile) -> bool { /// - a physical `_min` is 0 — the layer makes no demand on that dim, /// so the dim cannot constrain anything; or /// - a cap is `<= 0` — OpenCue's "unlimited" sentinel (mirrors the Lua -/// `BOOK_OR_FORCE > 0` guard); or +/// `> 0` cap guard); or /// - the layer has no core demand (`cores_min == 0`), in which case the /// core-denominated caps have no unit to divide by. /// @@ -115,14 +115,14 @@ pub fn fits_floor(host: &Host, profile: &LayerProfile) -> bool { /// going negative. /// True when the job is already at/over its core cap and cannot fit even one /// more `cores_min`-sized frame. `job_max_cores <= 0` is OpenCue's "unlimited" -/// sentinel (never at cap), matching the Lua `BOOK_OR_FORCE` `job_max > 0` guard +/// sentinel (never at cap), matching the accounting `job_max > 0` guard /// and `core_cap_bound` above. /// -/// All arguments are in whole cores (see the unit invariant in `lua.rs` and +/// All arguments are in whole cores (see the unit invariant in `accounting::store` and /// `DispatchLayer.job_max_cores`). Used by the matcher's pre-checkout skip so a /// job sitting at its cap isn't re-checked-out and re-rejected every pass; the -/// Lua call remains the authoritative gate, so a stale-low `job_cores_in_use` -/// (e.g. a failed Redis read defaulting to 0) only causes a missed skip, never +/// booking call remains the authoritative gate, so a stale-low `job_cores_in_use` +/// (e.g. an absent store entry defaulting to 0) only causes a missed skip, never /// an incorrect one. pub fn job_at_core_cap(job_cores_in_use: i32, cores_min: i32, job_max_cores: i32) -> bool { job_max_cores > 0 && job_cores_in_use.saturating_add(cores_min) > job_max_cores diff --git a/rust/crates/scheduler/tests/active_scan_superset.rs b/rust/crates/scheduler/tests/active_scan_superset.rs index b9a268a2ad..928a8ad5cd 100644 --- a/rust/crates/scheduler/tests/active_scan_superset.rs +++ b/rust/crates/scheduler/tests/active_scan_superset.rs @@ -42,10 +42,9 @@ mod active_scan_suite { /// (no Redis), so it only needs a migrated database. #[actix::test] async fn stress_active_scan_is_superset_of_per_cluster_query() { - // create_stress_config needs a Redis port for its accounting section, - // but this test never books, so the port is never dialed. This binary's + // This test never books, so accounting is inert here. This binary's // OVERRIDE_CONFIG is private to its own process (see module docs). - let _ = OVERRIDE_CONFIG.set(create_stress_config(6379)); + let _ = OVERRIDE_CONFIG.set(create_stress_config()); let pool = assert_ok!(test_connection_pool().await); let prefix = format!("stress_sc_{}", &Uuid::new_v4().to_string()[..6]); diff --git a/rust/crates/scheduler/tests/redis_integration.rs b/rust/crates/scheduler/tests/redis_integration.rs deleted file mode 100644 index cdf82d99ef..0000000000 --- a/rust/crates/scheduler/tests/redis_integration.rs +++ /dev/null @@ -1,481 +0,0 @@ -// Copyright Contributors to the OpenCue Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under -// the License. - -//! Integration tests for the Redis-backed accounting layer. Spawns a real Redis 7 -//! container per test via `testcontainers`. Exercises the Lua scripts directly -//! against a live Redis to ensure behavior matches the design contracts in §2.3/§2.4. -//! -//! Gated behind `--features redis-tests` (requires Docker daemon). - -#![cfg(feature = "redis-tests")] - -use std::time::Duration; - -use redis::aio::ConnectionManager; -use redis::{AsyncCommands, Client, Script}; -use scheduler::accounting::booking_delta::{BookingDelta, SEQ_KEY}; -use scheduler::accounting::lua::{BOOK_OR_FORCE, RESEED_CAS}; -use scheduler::accounting::redis_client::ReseedOp; -use testcontainers::runners::AsyncRunner; -use testcontainers::ContainerAsync; -use testcontainers_modules::redis::Redis; -use tokio::time::sleep; -use uuid::Uuid; - -struct RedisHarness { - _container: ContainerAsync, - conn: ConnectionManager, - book_script: Script, - reseed_script: Script, -} - -impl RedisHarness { - async fn new() -> Self { - let container = Redis::default().start().await.expect("start redis"); - let port = container - .get_host_port_ipv4(6379) - .await - .expect("redis port"); - let url = format!("redis://127.0.0.1:{port}/"); - let client = Client::open(url).expect("open client"); - let conn = ConnectionManager::new(client).await.expect("manager"); - RedisHarness { - _container: container, - conn, - book_script: Script::new(BOOK_OR_FORCE), - reseed_script: Script::new(RESEED_CAS), - } - } - - fn delta(show: Uuid, alloc: Uuid, cores: i64, gpus: i32) -> BookingDelta { - BookingDelta { - show_id: show, - alloc_id: alloc, - folder_id: Uuid::nil(), - job_id: Uuid::nil(), - layer_id: Uuid::nil(), - dept_id: Uuid::nil(), - core_delta: cores, - gpu_delta: gpus, - } - } - - async fn book(&self, delta: &BookingDelta, force: &str) -> redis::Value { - let mut conn = self.conn.clone(); - self.book_script - .key(delta.sub_key()) - .key(delta.folder_key()) - .key(delta.job_key()) - .key(delta.layer_key()) - .key(delta.point_key()) - .key(SEQ_KEY) - .arg(delta.core_delta.to_string()) - .arg(delta.gpu_delta.to_string()) - .arg(force) - .invoke_async(&mut conn) - .await - .expect("EVALSHA") - } - - async fn reseed(&self, seq_before: i64, ops: &[ReseedOp]) -> i64 { - let mut conn = self.conn.clone(); - let mut inv = self.reseed_script.prepare_invoke(); - inv.key(SEQ_KEY) - .arg(seq_before.to_string()) - .arg(ops.len().to_string()); - for op in ops { - inv.arg(op.key.as_str()) - .arg(op.field) - .arg(op.value.to_string()); - } - inv.invoke_async(&mut conn).await.expect("RESEED_CAS") - } - - async fn hget_i64(&self, key: &str, field: &str) -> Option { - let mut conn = self.conn.clone(); - conn.hget(key, field).await.expect("HGET") - } - - async fn get_seq(&self) -> i64 { - let mut conn = self.conn.clone(); - let v: Option = conn.get(SEQ_KEY).await.expect("GET seq"); - v.unwrap_or(0) - } - - async fn set_burst(&self, key: &str, burst: i64) { - let mut conn = self.conn.clone(); - let _: () = conn.hset(key, "burst", burst).await.expect("HSET burst"); - } - - async fn set_field(&self, key: &str, field: &str, value: i64) { - let mut conn = self.conn.clone(); - let _: () = conn.hset(key, field, value).await.expect("HSET field"); - } -} - -fn ok_seq(value: &redis::Value) -> Option { - if let redis::Value::Array(items) = value { - if let (Some(redis::Value::Int(1)), Some(redis::Value::Int(seq))) = - (items.first(), items.get(1)) - { - return Some(*seq); - } - } - None -} - -fn limit_exceeded(value: &redis::Value) -> Option<(String, i64, i64)> { - if let redis::Value::Array(items) = value { - if matches!(items.first(), Some(redis::Value::Int(0))) { - let table = match items.get(1)? { - redis::Value::BulkString(s) => String::from_utf8_lossy(s).into_owned(), - redis::Value::SimpleString(s) => s.clone(), - _ => return None, - }; - let current = match items.get(2)? { - redis::Value::Int(n) => *n, - _ => return None, - }; - let limit = match items.get(3)? { - redis::Value::Int(n) => *n, - _ => return None, - }; - return Some((table, current, limit)); - } - } - None -} - -#[tokio::test] -async fn book_happy_path_increments_counters_and_bumps_seq() { - // The Lua enforces `cur + delta > burst` unconditionally, matching Cuebot's - // semantics - `burst = 0` (the schema default) means "no bookings allowed". - // Pre-seed burst so this test models the production flow where the bootstrap - // limit reseed has already populated the subscription cap. - let h = RedisHarness::new().await; - let show = Uuid::new_v4(); - let alloc = Uuid::new_v4(); - let delta = RedisHarness::delta(show, alloc, 100, 1); - h.set_burst(&delta.sub_key(), 1000).await; - - let result = h.book(&delta, "0").await; - assert_eq!( - ok_seq(&result), - Some(1), - "expected {{1, 1}}, got {result:?}" - ); - - assert_eq!(h.hget_i64(&delta.sub_key(), "int_cores").await, Some(100)); - assert_eq!(h.hget_i64(&delta.folder_key(), "int_gpus").await, Some(1)); - assert_eq!(h.hget_i64(&delta.job_key(), "int_cores").await, Some(100)); - assert_eq!(h.hget_i64(&delta.layer_key(), "int_cores").await, Some(100)); - assert_eq!(h.hget_i64(&delta.point_key(), "int_cores").await, Some(100)); - assert_eq!(h.get_seq().await, 1); -} - -#[tokio::test] -async fn book_rejects_when_burst_is_zero() { - // Matches Cuebot's behavior: a subscription with no burst configured (or - // pre-bootstrap Redis state) rejects every booking. Verifies we no longer - // silently widen permissions for unconfigured subscriptions. - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 1, 0); - - let r = h.book(&delta, "0").await; - let (table, current, limit) = - limit_exceeded(&r).unwrap_or_else(|| panic!("expected limit-exceeded, got {r:?}")); - assert_eq!(table, "subscription"); - assert_eq!(current, 0); - assert_eq!(limit, 0); - - // No counters were written. - assert_eq!(h.hget_i64(&delta.sub_key(), "int_cores").await, None); - assert_eq!(h.get_seq().await, 0); -} - -#[tokio::test] -async fn force_mode_books_even_when_burst_is_zero() { - // Force-mode (rollback path) must succeed regardless of burst state, since the - // caller has already committed the booking elsewhere and we need to undo it. - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 50, 0); - - let result = h.book(&delta, "1").await; - assert_eq!(ok_seq(&result), Some(1)); - assert_eq!(h.hget_i64(&delta.sub_key(), "int_cores").await, Some(50)); -} - -#[tokio::test] -async fn book_over_burst_returns_structured_limit_exceeded() { - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 150, 0); - - // Pre-seed subscription burst so the cap is non-zero (a 0/missing burst means - // "unlimited" per the Lua's `burst > 0 and ...` gate). - h.set_burst(&delta.sub_key(), 200).await; - - // First booking puts us at 100 - under the 200 cap. - let first_delta = RedisHarness::delta(delta.show_id, delta.alloc_id, 100, 0); - let r1 = h.book(&first_delta, "0").await; - assert_eq!(ok_seq(&r1), Some(1)); - - // Second booking of 150 would push us to 250 - over the 200 cap. - let r2 = h.book(&delta, "0").await; - let (table, current, limit) = - limit_exceeded(&r2).unwrap_or_else(|| panic!("expected limit-exceeded, got {r2:?}")); - assert_eq!(table, "subscription"); - assert_eq!(current, 100); - assert_eq!(limit, 200); - - // Counters should be unchanged after a rejected booking. - assert_eq!(h.hget_i64(&delta.sub_key(), "int_cores").await, Some(100)); - assert_eq!(h.get_seq().await, 1); -} - -#[tokio::test] -async fn force_mode_bypasses_limit_check() { - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 500, 0); - h.set_burst(&delta.sub_key(), 100).await; - - // 500 would be over the 100 burst - but force=1 bypasses the check. - let result = h.book(&delta, "1").await; - assert_eq!(ok_seq(&result), Some(1)); - assert_eq!(h.hget_i64(&delta.sub_key(), "int_cores").await, Some(500)); -} - -#[tokio::test] -async fn reseed_cas_applies_when_seq_unchanged() { - let h = RedisHarness::new().await; - let key = format!("acct:sub:{}:{}", Uuid::new_v4(), Uuid::new_v4()); - - let seq_before = h.get_seq().await; - let ops = vec![ - ReseedOp { - key: key.clone(), - field: "int_cores", - value: 42, - }, - ReseedOp { - key: key.clone(), - field: "burst", - value: 200, - }, - ]; - let applied = h.reseed(seq_before, &ops).await; - assert_eq!(applied, 1, "CAS should succeed when seq is unchanged"); - assert_eq!(h.hget_i64(&key, "int_cores").await, Some(42)); - assert_eq!(h.hget_i64(&key, "burst").await, Some(200)); - - // RESEED_CAS must NOT bump acct:seq (it's reconciliation, not mutation). - assert_eq!(h.get_seq().await, seq_before); -} - -#[tokio::test] -async fn reseed_cas_misses_when_booking_bumped_seq_concurrently() { - let h = RedisHarness::new().await; - let show = Uuid::new_v4(); - let alloc = Uuid::new_v4(); - let delta = RedisHarness::delta(show, alloc, 50, 0); - h.set_burst(&delta.sub_key(), 1000).await; - - let seq_before = h.get_seq().await; - - // Concurrent booking bumps `acct:seq` between the caller's GET and the CAS attempt. - let _ = h.book(&delta, "0").await; - assert!(h.get_seq().await > seq_before); - - // CAS attempt with the stale seq_before should return 0 (miss). - let ops = vec![ReseedOp { - key: delta.sub_key(), - field: "int_cores", - value: 999, - }]; - let applied = h.reseed(seq_before, &ops).await; - assert_eq!(applied, 0, "CAS should miss with stale seq_before"); - - // The hot-path-written value must survive the failed reseed (i.e. no clobber). - assert_eq!(h.hget_i64(&delta.sub_key(), "int_cores").await, Some(50)); -} - -#[tokio::test] -async fn cas_guard_prevents_silent_loss_under_concurrent_booking() { - // Mirrors the trace in design §2.4: reseed reads SUM=50, booking runs (counter→60, - // seq bumped), reseed CAS detects miss and retries; the booking is preserved. - let h = RedisHarness::new().await; - let show = Uuid::new_v4(); - let alloc = Uuid::new_v4(); - - // Initial state: counter at 50, burst at 1000 (so the booking is under cap), seq=0. - let key = format!("acct:sub:{}:{}", show, alloc); - { - let mut conn = h.conn.clone(); - let _: () = conn.hset(&key, "int_cores", 50).await.unwrap(); - } - h.set_burst(&key, 1000).await; - - // (t1) Reseed reads seq_before. - let seq_before = h.get_seq().await; - - // (t2) Concurrent booking bumps the counter to 60 and increments seq. - let booking = RedisHarness::delta(show, alloc, 10, 0); - // The booking writes to its own (sub, folder, job, layer, point) hashes; the sub key - // is the same one our reseed will target. - let _ = h.book(&booking, "0").await; - let post_book = h.hget_i64(&key, "int_cores").await; - assert_eq!(post_book, Some(60), "booking applied: 50 + 10"); - - // (t3) Reseed tries to CAS-write 50 (its in-memory snapshot) - must miss. - let ops = vec![ReseedOp { - key: key.clone(), - field: "int_cores", - value: 50, - }]; - let applied = h.reseed(seq_before, &ops).await; - assert_eq!(applied, 0, "reseed must miss to protect concurrent booking"); - - // The booking is preserved - no silent loss. - assert_eq!(h.hget_i64(&key, "int_cores").await, Some(60)); - - // (t4) Reseed re-snapshots: seq_before = current seq, recompute ops with 60. - let seq_before_retry = h.get_seq().await; - let ops_retry = vec![ReseedOp { - key: key.clone(), - field: "int_cores", - value: 60, - }]; - let applied_retry = h.reseed(seq_before_retry, &ops_retry).await; - assert_eq!(applied_retry, 1, "retry succeeds with fresh snapshot"); - assert_eq!(h.hget_i64(&key, "int_cores").await, Some(60)); -} - -#[tokio::test] -async fn book_over_job_gpu_max_returns_structured_limit_exceeded() { - // Mirrors the Java `DispatchQuery.FIND_JOBS_BY_SHOW_PRIORITY_MODE` predicate - // `job_resource.int_gpus + layer.int_gpus_min < job_resource.int_max_gpus` - // that lived in PG before accounting moved to Redis. - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 100, 3); - h.set_burst(&delta.sub_key(), 10_000).await; - h.set_field(&delta.job_key(), "int_max_gpus", 4).await; - - // First booking puts the job at 2 GPUs - under the 4 cap. - let first = RedisHarness::delta(delta.show_id, delta.alloc_id, 100, 2); - let r1 = h.book(&first, "0").await; - assert_eq!(ok_seq(&r1), Some(1)); - - // Second booking would push GPUs to 5 - over the 4 cap. - let r2 = h.book(&delta, "0").await; - let (table, current, limit) = - limit_exceeded(&r2).unwrap_or_else(|| panic!("expected limit-exceeded, got {r2:?}")); - assert_eq!(table, "job_gpus"); - assert_eq!(current, 2); - assert_eq!(limit, 4); - - // Counters should be unchanged after a rejected booking. - assert_eq!(h.hget_i64(&delta.job_key(), "int_gpus").await, Some(2)); -} - -#[tokio::test] -async fn book_over_folder_gpu_max_returns_structured_limit_exceeded() { - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 100, 2); - h.set_burst(&delta.sub_key(), 10_000).await; - h.set_field(&delta.folder_key(), "int_max_gpus", 3).await; - - // First booking puts the folder at 2 GPUs - under the 3 cap. - let first = RedisHarness::delta(delta.show_id, delta.alloc_id, 100, 2); - let r1 = h.book(&first, "0").await; - assert_eq!(ok_seq(&r1), Some(1)); - - // Second booking would push folder GPUs to 4 - over the 3 cap. - let r2 = h.book(&delta, "0").await; - let (table, current, limit) = - limit_exceeded(&r2).unwrap_or_else(|| panic!("expected limit-exceeded, got {r2:?}")); - assert_eq!(table, "folder_gpus"); - assert_eq!(current, 2); - assert_eq!(limit, 3); -} - -#[tokio::test] -async fn gpu_cap_unlimited_when_negative() { - // Cuebot's `-1` sentinel = "unlimited" for `int_max_gpus`. The `> 0` guard in - // the Lua must let bookings through when the cap is unset or -1. - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 100, 99); - h.set_burst(&delta.sub_key(), 10_000).await; - h.set_field(&delta.job_key(), "int_max_gpus", -1).await; - h.set_field(&delta.folder_key(), "int_max_gpus", -1).await; - - let r = h.book(&delta, "0").await; - assert!( - ok_seq(&r).is_some(), - "expected booking accepted with -1 cap, got {r:?}" - ); - assert_eq!(h.hget_i64(&delta.job_key(), "int_gpus").await, Some(99)); -} - -#[tokio::test] -async fn gpu_cap_not_checked_when_delta_is_zero() { - // Cores-only bookings should not be rejected by a tight GPU cap (no GPU - // demand means there's nothing to check). - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 100, 0); - h.set_burst(&delta.sub_key(), 10_000).await; - h.set_field(&delta.job_key(), "int_max_gpus", 1).await; - h.set_field(&delta.job_key(), "int_gpus", 1).await; // already at the cap - - let r = h.book(&delta, "0").await; - assert!( - ok_seq(&r).is_some(), - "expected booking accepted with gpu_delta=0, got {r:?}" - ); -} - -#[tokio::test] -async fn force_mode_bypasses_gpu_cap() { - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 100, 5); - h.set_burst(&delta.sub_key(), 10_000).await; - h.set_field(&delta.job_key(), "int_max_gpus", 1).await; - - let result = h.book(&delta, "1").await; - assert_eq!(ok_seq(&result), Some(1)); - assert_eq!(h.hget_i64(&delta.job_key(), "int_gpus").await, Some(5)); -} - -#[tokio::test] -async fn force_with_negative_delta_decrements() { - // Exercises the compensation-rollback path: a successful booking followed by a - // force-rollback with negated deltas should net to zero. - let h = RedisHarness::new().await; - let delta = RedisHarness::delta(Uuid::new_v4(), Uuid::new_v4(), 250, 2); - h.set_burst(&delta.sub_key(), 1000).await; - - let _ = h.book(&delta, "0").await; - assert_eq!(h.hget_i64(&delta.sub_key(), "int_cores").await, Some(250)); - assert_eq!(h.hget_i64(&delta.folder_key(), "int_gpus").await, Some(2)); - - // Rollback with negated delta in force mode. - let rollback = BookingDelta { - core_delta: -delta.core_delta, - gpu_delta: -delta.gpu_delta, - ..delta.clone() - }; - let result = h.book(&rollback, "1").await; - assert!(ok_seq(&result).is_some(), "force rollback applied"); - assert_eq!(h.hget_i64(&delta.sub_key(), "int_cores").await, Some(0)); - assert_eq!(h.hget_i64(&delta.folder_key(), "int_gpus").await, Some(0)); - - // Sanity: brief wait to ensure no race-condition log spam. - sleep(Duration::from_millis(10)).await; -} diff --git a/rust/crates/scheduler/tests/stress_tests.rs b/rust/crates/scheduler/tests/stress_tests.rs index 7900096de0..0c971da0f5 100644 --- a/rust/crates/scheduler/tests/stress_tests.rs +++ b/rust/crates/scheduler/tests/stress_tests.rs @@ -12,7 +12,7 @@ //! Booking + accounting stress suite for the Rust scheduler. //! -//! Runs the full production pipeline (`pipeline::run`: Redis accounting bootstrap → +//! Runs the full production pipeline (`pipeline::run`: in-memory accounting bootstrap → //! cluster feed → job query → host matching → dispatch) against a realistic farm in //! two phases inside one process: //! @@ -20,20 +20,18 @@ //! throughput (frames/s over the active booking window) and requires ≥90% //! (`STRESS_DRAIN_TARGET`) of frames to dispatch. //! 2. **saturation** — demand vastly exceeds tight subscription bursts and per-job -//! core caps, so the Redis Lua cap check becomes the binding constraint. Verifies +//! core caps, so the in-memory cap check becomes the binding constraint. Verifies //! enforcement (no booking above burst / job max-cores) and that rejections //! actually flowed through the accounting hot path. //! -//! After each phase, an audit cross-checks every Redis `acct:*` hash against +//! After each phase, an audit cross-checks the in-memory accounting store against //! `SUM(proc)` in Postgres plus host/frame/stat invariants — with the recompute and //! limit-reseed loops pushed beyond the test horizon, agreement proves the dispatch -//! hot path (Lua book + force-rollback) kept accounting exact on its own. +//! hot path (book + rollback) kept accounting exact on its own. //! //! # Requirements //! //! - Postgres with migrations applied, from the repo root: `docker compose up -d flyway` -//! - A Docker daemon (the suite starts a throwaway Redis container via testcontainers; -//! all Redis state dies with the container) //! //! # Running //! @@ -73,15 +71,11 @@ mod stress_suite { use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; - use redis::aio::ConnectionManager; - use redis::AsyncCommands; use scheduler::{ cluster::{self, ClusterFeed}, config::OVERRIDE_CONFIG, host_cache, metrics, pipeline, }; - use testcontainers::runners::AsyncRunner; - use testcontainers_modules::redis::Redis as RedisImage; use tokio_test::assert_ok; use tracing::info; use uuid::Uuid; @@ -119,12 +113,10 @@ mod stress_suite { cluster_rounds: usize, frames_dispatched: u64, limit_exceeded: Vec<(&'static str, u64)>, - redis_seq: i64, } impl Counters { - async fn take(redis: &mut ConnectionManager) -> Counters { - let seq: Option = redis.get("acct:seq").await.unwrap_or(None); + fn take() -> Counters { Counters { hosts_attempted: pipeline::HOSTS_ATTEMPTED.load(Ordering::Relaxed), wasted_attempts: pipeline::WASTED_ATTEMPTS.load(Ordering::Relaxed), @@ -139,7 +131,6 @@ mod stress_suite { (*table, count) }) .collect(), - redis_seq: seq.unwrap_or(0), } } @@ -211,8 +202,7 @@ mod stress_suite { .map(|t| format!("{}={}", t, self.after.limit_exceeded_delta(&self.before, t))) .collect(); println!( - "accounting : {} redis lua ops, {} dispatches (metrics), {} booked cores, rejections [{}]", - self.after.redis_seq - self.before.redis_seq, + "accounting : {} dispatches (metrics), {} booked cores, rejections [{}]", self.after.frames_dispatched - self.before.frames_dispatched, self.audit.booked_cores, rejections.join(" ") @@ -246,14 +236,13 @@ mod stress_suite { async fn run_phase( name: &'static str, farm: &StressFarm, - redis: &mut ConnectionManager, stall: Duration, hard_timeout: Duration, ) -> PhaseResult { let pool = assert_ok!(test_connection_pool().await); let waiting_before = get_waiting_frames_count(WaitingFrameClause::JobPrefix(farm.prefix.clone())).await; - let before = Counters::take(redis).await; + let before = Counters::take(); info!( "Starting phase '{}' ({} clusters, {} frames)", @@ -285,12 +274,12 @@ mod stress_suite { None }; - let after = Counters::take(redis).await; + let after = Counters::take(); let cache_hit_pct = host_cache::hit_ratio().await; let stats = booking_stats(&pool, farm.show_id).await; let waiting_after = get_waiting_frames_count(WaitingFrameClause::JobPrefix(farm.prefix.clone())).await; - let audit = audit_accounting(&pool, redis, farm).await; + let audit = audit_accounting(&pool, farm).await; PhaseResult { name, @@ -319,21 +308,9 @@ mod stress_suite { let hard_timeout = Duration::from_secs(env_usize("STRESS_TIMEOUT_SECS", 600) as u64); let drain_target = env_f64("STRESS_DRAIN_TARGET", 0.9); - // Throwaway Redis for the accounting hot path; all acct:* state dies with it. - let redis_container = RedisImage::default() - .start() - .await - .expect("failed to start Redis testcontainer (is Docker running?)"); - let redis_port = redis_container - .get_host_port_ipv4(6379) - .await - .expect("redis port"); - let _ = OVERRIDE_CONFIG.set(create_stress_config(redis_port)); - let redis_client = - redis::Client::open(format!("redis://127.0.0.1:{redis_port}/")).expect("redis client"); - let mut redis = ConnectionManager::new(redis_client) - .await - .expect("redis connection"); + // In-memory accounting: no external store. The hot path (book + rollback) is the + // accounting system under test; recompute/limit-reseed are pushed past the horizon. + let _ = OVERRIDE_CONFIG.set(create_stress_config()); let pool = assert_ok!(test_connection_pool().await); @@ -392,8 +369,8 @@ mod stress_suite { let sat_farm = assert_ok!(seed_farm(&pool, sat_spec).await); info!("Seeding took {:?}", seed_started.elapsed()); - let drain = run_phase("drain", &drain_farm, &mut redis, stall, hard_timeout).await; - let sat = run_phase("saturation", &sat_farm, &mut redis, stall, hard_timeout).await; + let drain = run_phase("drain", &drain_farm, stall, hard_timeout).await; + let sat = run_phase("saturation", &sat_farm, stall, hard_timeout).await; drain.print(); sat.print(); @@ -435,7 +412,7 @@ mod stress_suite { } if sat.after.limit_exceeded_delta(&sat.before, "subscription") == 0 { failures.push( - "saturation phase produced no subscription-cap rejections in the Redis hot path" + "saturation phase produced no subscription-cap rejections in the booking hot path" .to_string(), ); } diff --git a/rust/crates/scheduler/tests/util/stress.rs b/rust/crates/scheduler/tests/util/stress.rs index fb9e8f8164..68446f927f 100644 --- a/rust/crates/scheduler/tests/util/stress.rs +++ b/rust/crates/scheduler/tests/util/stress.rs @@ -37,14 +37,13 @@ use std::time::Duration; use chrono::{DateTime, Utc}; use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; -use redis::aio::ConnectionManager; -use redis::AsyncCommands; use scheduler::{ + accounting::accounting_service, cluster::Cluster, cluster_key::{Tag, TagType}, config::{ AccountingConfig, Config, DatabaseConfig, HostBookingStrategy, HostCacheConfig, - LoggingConfig, QueueConfig, RedisConfig, RqdConfig, SchedulerConfig, StreamConfig, + LoggingConfig, QueueConfig, RqdConfig, SchedulerConfig, StreamConfig, }, }; use sqlx::{Pool, Postgres, QueryBuilder}; @@ -69,13 +68,13 @@ const SEED: u64 = 0x0C0FFEE; /// exits a few seconds after every cluster stops yielding jobs (either because /// the workload drained or because the watchdog paused it). /// - `recompute_interval` / `limit_reseed_interval` = 1h: the reconciliation loops -/// never fire inside the test window, so the final Redis state is the product of +/// never fire inside the test window, so the final store state is the product of /// the dispatch hot path alone — exactly what the audit wants to verify. /// - `host_staleness_threshold` = 1h: seeded hosts never report a fresh `ts_ping` /// (no RQD), so the cache must not evict them mid-run. -/// - `dry_run_mode` = true: full booking path (Redis Lua + proc insert + host -/// decrement + frame start) without gRPC calls to RQD. -pub fn create_stress_config(redis_port: u16) -> Config { +/// - `dry_run_mode` = true: full booking path (in-memory check+increment + proc insert +/// + host decrement + frame start) without gRPC calls to RQD. +pub fn create_stress_config() -> Config { Config { logging: LoggingConfig { level: "info".to_string(), @@ -136,16 +135,9 @@ pub fn create_stress_config(redis_port: u16) -> Config { }, scheduler: SchedulerConfig::default(), accounting: AccountingConfig { - redis: RedisConfig { - enabled: true, - host: "127.0.0.1".to_string(), - port: redis_port, - pool_size: 20, - }, recompute_interval: Duration::from_secs(3600), limit_reseed_interval: Duration::from_secs(3600), managed_shows_ttl: Duration::from_secs(5), - cas_max_retries: 5, }, sentry_dsn: None, } @@ -748,108 +740,134 @@ pub struct AuditOutcome { pub per_sub: Vec, } -async fn redis_hash_i64(redis: &mut ConnectionManager, key: &str, field: &str) -> i64 { - let value: Option = redis.hget(key, field).await.unwrap_or(None); - value.unwrap_or(0) +/// Compare one vertex's expected centicores/gpus against the store's booked `(cores, gpus)`. +fn cmp_vertex(label: &str, want_centi: i64, want_gpus: i64, got: (i64, i64)) -> Vec { + let want_cores = want_centi / CORE_MULT; + let mut v = Vec::new(); + if got.0 != want_cores { + v.push(format!( + "{label}: store cores={} but SUM(proc) says {want_cores}", + got.0 + )); + } + if got.1 != want_gpus { + v.push(format!( + "{label}: store gpus={} but SUM(proc) says {want_gpus}", + got.1 + )); + } + v } -/// Cross-checks the Redis accounting hashes against `SUM(proc)` in Postgres (the -/// canonical record of bookings, per the redis-accounting design), and validates -/// the cap + host/frame/stat invariants that booking must preserve: +/// Cross-checks the in-memory accounting store against `SUM(proc)` in Postgres (the +/// canonical record of bookings), and validates the cap + host/frame/stat invariants +/// that booking must preserve: /// -/// 1. Every `acct:{sub,folder,job,layer,point}` hash the show touched holds exactly -/// `SUM(proc.int_cores_reserved)/100` cores and `SUM(proc.int_gpus_reserved)` -/// GPUs for its grouping (same 5-dim grouping + unit conversion the recompute -/// loop uses — but with recompute pushed out of the test window, agreement here -/// proves the *hot path* (Lua book + force-rollback) kept Redis exact). -/// 2. Jobs with no bookings have no leaked Redis counters. +/// 1. Every subscription/folder/job booked counter in the store holds exactly +/// `SUM(proc.int_cores_reserved)/100` cores and `SUM(proc.int_gpus_reserved)` GPUs for +/// its grouping. With the recompute loop pushed past the test window, agreement here +/// proves the *hot path* (book + rollback) kept the store exact on its own. Layer and +/// point are not tracked (the booking check never reads them). +/// 2. Jobs with no bookings have no leaked store counters. /// 3. Per-(show, alloc) booked cores never exceed the subscription burst. /// 4. Per-job booked cores never exceed `job_resource.int_max_cores` (when set). /// 5. Host ledger: `int_cores - int_cores_idle == SUM(proc)` per host, never negative. /// 6. Frame/proc agreement: one RUNNING frame per proc. /// 7. Trigger-maintained `job_stat.int_waiting_count` matches the frame table. -pub async fn audit_accounting( - pool: &Pool, - redis: &mut ConnectionManager, - farm: &StressFarm, -) -> AuditOutcome { +pub async fn audit_accounting(pool: &Pool, farm: &StressFarm) -> AuditOutcome { let mut out = AuditOutcome::default(); let show = farm.show_id.to_string(); + let show_id = farm.show_id; let like = format!("{}%", farm.prefix); - // --- 1. + 2.: Redis hashes vs SUM(proc), grouped exactly like recompute --- + // --- 1. + 2.: store counters vs SUM(proc), grouped by the enforced vertices --- #[derive(sqlx::FromRow)] struct BookedRow { - pk_show: String, pk_alloc: String, pk_folder: String, pk_job: String, - pk_layer: String, - pk_dept: String, cores: i64, gpus: i64, } let rows: Vec = sqlx::query_as( - "SELECT j.pk_show, h.pk_alloc, j.pk_folder, p.pk_job, p.pk_layer, j.pk_dept, \ + "SELECT h.pk_alloc, j.pk_folder, p.pk_job, \ COALESCE(SUM(p.int_cores_reserved), 0)::bigint AS cores, \ COALESCE(SUM(p.int_gpus_reserved), 0)::bigint AS gpus \ FROM proc p \ JOIN host h ON h.pk_host = p.pk_host \ JOIN job j ON j.pk_job = p.pk_job \ WHERE j.pk_show = $1 AND p.b_local = false \ - GROUP BY j.pk_show, h.pk_alloc, j.pk_folder, p.pk_job, p.pk_layer, j.pk_dept", + GROUP BY h.pk_alloc, j.pk_folder, p.pk_job", ) .bind(&show) .fetch_all(pool) .await .expect("booked snapshot query"); - // Aggregate centicores per Redis key, then convert once - mirrors - // accounting::recompute::booked_ops_from_snapshot. - let mut expected: HashMap = HashMap::new(); - let mut booked_jobs: HashMap = HashMap::new(); + // Aggregate centicores per vertex key (mirrors the recompute aggregation). + let mut exp_sub: HashMap<(Uuid, Uuid), (i64, i64)> = HashMap::new(); + let mut exp_folder: HashMap = HashMap::new(); + let mut exp_job: HashMap = HashMap::new(); for r in &rows { - for key in [ - format!("acct:sub:{}:{}", r.pk_show, r.pk_alloc), - format!("acct:folder:{}", r.pk_folder), - format!("acct:job:{}", r.pk_job), - format!("acct:layer:{}", r.pk_layer), - format!("acct:point:{}:{}", r.pk_dept, r.pk_show), - ] { - let e = expected.entry(key).or_insert((0, 0)); - e.0 += r.cores; - e.1 += r.gpus; - } - *booked_jobs.entry(r.pk_job.clone()).or_insert(0) += r.cores; + let alloc = Uuid::parse_str(&r.pk_alloc).expect("alloc uuid"); + let folder = Uuid::parse_str(&r.pk_folder).expect("folder uuid"); + let job = Uuid::parse_str(&r.pk_job).expect("job uuid"); + let s = exp_sub.entry((show_id, alloc)).or_insert((0, 0)); + s.0 += r.cores; + s.1 += r.gpus; + let f = exp_folder.entry(folder).or_insert((0, 0)); + f.0 += r.cores; + f.1 += r.gpus; + let j = exp_job.entry(job).or_insert((0, 0)); + j.0 += r.cores; + j.1 += r.gpus; } - for (key, (cores_centi, gpus)) in &expected { - let want_cores = cores_centi / CORE_MULT; - let got_cores = redis_hash_i64(redis, key, "int_cores").await; - let got_gpus = redis_hash_i64(redis, key, "int_gpus").await; - if got_cores != want_cores { - out.violations.push(format!( - "{key}: int_cores={got_cores} but SUM(proc) says {want_cores}" - )); - } - if got_gpus != *gpus { - out.violations.push(format!( - "{key}: int_gpus={got_gpus} but SUM(proc) says {gpus}" - )); - } + // The store is the live accounting state used by the pipeline we just ran. + let store = accounting_service() + .await + .expect("accounting service") + .store() + .audit_snapshot(); + + for (&(s, a), &(c, g)) in &exp_sub { + out.violations.extend(cmp_vertex( + &format!("sub:{s}:{a}"), + c, + g, + store.sub.get(&(s, a)).copied().unwrap_or((0, 0)), + )); + } + for (&f, &(c, g)) in &exp_folder { + out.violations.extend(cmp_vertex( + &format!("folder:{f}"), + c, + g, + store.folder.get(&f).copied().unwrap_or((0, 0)), + )); + } + for (&jb, &(c, g)) in &exp_job { + out.violations.extend(cmp_vertex( + &format!("job:{jb}"), + c, + g, + store.job.get(&jb).copied().unwrap_or((0, 0)), + )); } + // 2. Jobs with no procs must not have leaked a non-zero store counter. let all_jobs: Vec = sqlx::query_scalar("SELECT pk_job FROM job WHERE pk_show = $1") .bind(&show) .fetch_all(pool) .await .expect("job list query"); for job in &all_jobs { - if !booked_jobs.contains_key(job) { - let got = redis_hash_i64(redis, &format!("acct:job:{job}"), "int_cores").await; + let job_id = Uuid::parse_str(job).expect("job uuid"); + if !exp_job.contains_key(&job_id) { + let got = store.job.get(&job_id).map(|(c, _)| *c).unwrap_or(0); if got != 0 { out.violations.push(format!( - "acct:job:{job}: int_cores={got} leaked for a job with no procs" + "job:{job}: store cores={got} leaked for a job with no procs" )); } } @@ -871,8 +889,8 @@ pub async fn audit_accounting( .await .expect("subscription query"); for sub in &subs { - let key = format!("acct:sub:{}:{}", show, sub.pk_alloc); - let booked_centi = expected.get(&key).map(|(c, _)| *c).unwrap_or(0); + let alloc = Uuid::parse_str(&sub.pk_alloc).expect("alloc uuid"); + let booked_centi = exp_sub.get(&(show_id, alloc)).map(|(c, _)| *c).unwrap_or(0); let booked_cores = booked_centi / CORE_MULT; let burst_cores = sub.int_burst / CORE_MULT; if booked_cores > burst_cores { @@ -961,11 +979,7 @@ pub async fn audit_accounting( } out.dispatched_procs = procs; - out.booked_cores = expected - .iter() - .filter(|(k, _)| k.starts_with("acct:sub:")) - .map(|(_, (c, _))| c / CORE_MULT) - .sum(); + out.booked_cores = exp_sub.values().map(|(c, _)| c / CORE_MULT).sum(); out } From ca910417bdd39bb96f39867f028c768602c88108 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Mon, 29 Jun 2026 10:30:11 -0700 Subject: [PATCH 02/20] Version up and minor refactor --- VERSION.in | 2 +- rust/crates/scheduler/src/accounting/store.rs | 179 ++++++++++++++---- 2 files changed, 141 insertions(+), 40 deletions(-) diff --git a/VERSION.in b/VERSION.in index 24cffb85f8..b0c101e630 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -1.26 +1.27 diff --git a/rust/crates/scheduler/src/accounting/store.rs b/rust/crates/scheduler/src/accounting/store.rs index 561b66f753..ba9c7e1c9f 100644 --- a/rust/crates/scheduler/src/accounting/store.rs +++ b/rust/crates/scheduler/src/accounting/store.rs @@ -66,8 +66,11 @@ struct Counter { /// Booked, proc INSERT not yet confirmed committed. In no snapshot → always carried. inflight_cores: i64, inflight_gpus: i64, - /// Confirmed bookings not yet provably in a recompute snapshot, double-buffered by - /// recompute-epoch parity. Index `e % 2` is written by confirms during epoch `e`. + /// Bookings that have been confirmed but whose `proc` row may not yet appear in a + /// recompute snapshot. Uses two alternating buckets indexed by `e % 2`: confirms + /// during epoch `e` write to bucket `e % 2`, so the recompute can identify and + /// preserve confirms that raced its snapshot read without clearing ones already + /// captured by it. settled_cores: [i64; 2], settled_gpus: [i64; 2], } @@ -210,11 +213,11 @@ impl Store { /// Hot-path booking: atomically check subscription burst and folder/job core/GPU /// caps, and on success increment all three vertices and record the delta as in-flight. pub fn book(&self, delta: &BookingDelta) -> BookOutcome { - let dc = delta.core_delta; - let dg = i64::from(delta.gpu_delta); + let core_delta = delta.core_delta; + let gpu_delta = i64::from(delta.gpu_delta); let mut inner = self.lock(); - if dc > 0 { + if core_delta > 0 { let cur_sub = inner .sub .get(&(delta.show_id, delta.alloc_id)) @@ -225,7 +228,7 @@ impl Store { .copied() .unwrap_or(0); // Subscription burst enforces 0 as "reject all". - if over_cap(cur_sub, dc, burst, true) { + if over_cap(cur_sub, core_delta, burst, true) { return BookOutcome::LimitExceeded { table: "subscription", current: cur_sub, @@ -234,8 +237,11 @@ impl Store { } let cur_folder = inner.folder.get(&delta.folder_id).map_or(0, |c| c.cores); - let folder_max = inner.folder_caps.get(&delta.folder_id).map_or(0, |c| c.max_cores); - if over_cap(cur_folder, dc, folder_max, false) { + let folder_max = inner + .folder_caps + .get(&delta.folder_id) + .map_or(0, |c| c.max_cores); + if over_cap(cur_folder, core_delta, folder_max, false) { return BookOutcome::LimitExceeded { table: "folder", current: cur_folder, @@ -245,7 +251,7 @@ impl Store { let cur_job = inner.job.get(&delta.job_id).map_or(0, |c| c.cores); let job_max = inner.job_caps.get(&delta.job_id).map_or(0, |c| c.max_cores); - if over_cap(cur_job, dc, job_max, false) { + if over_cap(cur_job, core_delta, job_max, false) { return BookOutcome::LimitExceeded { table: "job", current: cur_job, @@ -254,10 +260,13 @@ impl Store { } } - if dg > 0 { + if gpu_delta > 0 { let cur_folder_gpu = inner.folder.get(&delta.folder_id).map_or(0, |c| c.gpus); - let folder_gpu_max = inner.folder_caps.get(&delta.folder_id).map_or(0, |c| c.max_gpus); - if over_cap(cur_folder_gpu, dg, folder_gpu_max, false) { + let folder_gpu_max = inner + .folder_caps + .get(&delta.folder_id) + .map_or(0, |c| c.max_gpus); + if over_cap(cur_folder_gpu, gpu_delta, folder_gpu_max, false) { return BookOutcome::LimitExceeded { table: "folder_gpus", current: cur_folder_gpu, @@ -267,7 +276,7 @@ impl Store { let cur_job_gpu = inner.job.get(&delta.job_id).map_or(0, |c| c.gpus); let job_gpu_max = inner.job_caps.get(&delta.job_id).map_or(0, |c| c.max_gpus); - if over_cap(cur_job_gpu, dg, job_gpu_max, false) { + if over_cap(cur_job_gpu, gpu_delta, job_gpu_max, false) { return BookOutcome::LimitExceeded { table: "job_gpus", current: cur_job_gpu, @@ -276,9 +285,21 @@ impl Store { } } - inner.sub.entry((delta.show_id, delta.alloc_id)).or_default().add_booking(dc, dg); - inner.folder.entry(delta.folder_id).or_default().add_booking(dc, dg); - inner.job.entry(delta.job_id).or_default().add_booking(dc, dg); + inner + .sub + .entry((delta.show_id, delta.alloc_id)) + .or_default() + .add_booking(core_delta, gpu_delta); + inner + .folder + .entry(delta.folder_id) + .or_default() + .add_booking(core_delta, gpu_delta); + inner + .job + .entry(delta.job_id) + .or_default() + .add_booking(core_delta, gpu_delta); BookOutcome::Applied } @@ -290,9 +311,21 @@ impl Store { let dg = i64::from(delta.gpu_delta); let mut inner = self.lock(); let bucket = (inner.epoch % 2) as usize; - inner.sub.entry((delta.show_id, delta.alloc_id)).or_default().settle(dc, dg, bucket); - inner.folder.entry(delta.folder_id).or_default().settle(dc, dg, bucket); - inner.job.entry(delta.job_id).or_default().settle(dc, dg, bucket); + inner + .sub + .entry((delta.show_id, delta.alloc_id)) + .or_default() + .settle(dc, dg, bucket); + inner + .folder + .entry(delta.folder_id) + .or_default() + .settle(dc, dg, bucket); + inner + .job + .entry(delta.job_id) + .or_default() + .settle(dc, dg, bucket); } /// Booking failed before launch: undo the live increment and the in-flight delta. @@ -300,9 +333,21 @@ impl Store { let dc = delta.core_delta; let dg = i64::from(delta.gpu_delta); let mut inner = self.lock(); - inner.sub.entry((delta.show_id, delta.alloc_id)).or_default().remove_booking(dc, dg); - inner.folder.entry(delta.folder_id).or_default().remove_booking(dc, dg); - inner.job.entry(delta.job_id).or_default().remove_booking(dc, dg); + inner + .sub + .entry((delta.show_id, delta.alloc_id)) + .or_default() + .remove_booking(dc, dg); + inner + .folder + .entry(delta.folder_id) + .or_default() + .remove_booking(dc, dg); + inner + .job + .entry(delta.job_id) + .or_default() + .remove_booking(dc, dg); } /// Apply a release delta (negative cores/gpus) from the Cuebot `acct_release` NOTIFY. @@ -455,21 +500,35 @@ impl Store { folder_id, max_cores, } => { - inner.folder_caps.entry(folder_id).or_insert(MaxCap::unlimited()).max_cores = - max_cores; + inner + .folder_caps + .entry(folder_id) + .or_insert(MaxCap::unlimited()) + .max_cores = max_cores; } LimitChange::FolderMaxGpus { folder_id, max_gpus, } => { - inner.folder_caps.entry(folder_id).or_insert(MaxCap::unlimited()).max_gpus = - max_gpus; + inner + .folder_caps + .entry(folder_id) + .or_insert(MaxCap::unlimited()) + .max_gpus = max_gpus; } LimitChange::JobMaxCores { job_id, max_cores } => { - inner.job_caps.entry(job_id).or_insert(MaxCap::unlimited()).max_cores = max_cores; + inner + .job_caps + .entry(job_id) + .or_insert(MaxCap::unlimited()) + .max_cores = max_cores; } LimitChange::JobMaxGpus { job_id, max_gpus } => { - inner.job_caps.entry(job_id).or_insert(MaxCap::unlimited()).max_gpus = max_gpus; + inner + .job_caps + .entry(job_id) + .or_insert(MaxCap::unlimited()) + .max_gpus = max_gpus; } } } @@ -484,7 +543,11 @@ impl Store { pub fn sub_counters(&self, show_id: Uuid, alloc_id: Uuid) -> (i64, i64) { let inner = self.lock(); let booked = inner.sub.get(&(show_id, alloc_id)).map_or(0, |c| c.cores); - let burst = inner.sub_burst.get(&(show_id, alloc_id)).copied().unwrap_or(0); + let burst = inner + .sub_burst + .get(&(show_id, alloc_id)) + .copied() + .unwrap_or(0); (booked, burst) } @@ -537,7 +600,14 @@ pub struct AuditSnapshot { mod tests { use super::*; - fn delta(show: Uuid, alloc: Uuid, folder: Uuid, job: Uuid, cores: i64, gpus: i32) -> BookingDelta { + fn delta( + show: Uuid, + alloc: Uuid, + folder: Uuid, + job: Uuid, + cores: i64, + gpus: i32, + ) -> BookingDelta { BookingDelta { show_id: show, alloc_id: alloc, @@ -553,17 +623,32 @@ mod tests { } fn ids() -> (Uuid, Uuid, Uuid, Uuid) { - (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()) + ( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) } #[test] fn book_rejects_when_unseeded_burst_is_zero() { // Missing burst == 0 == reject all (fail closed before any seed). let store = Store::new(); - let d = delta(Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4(), 1, 0); + let d = delta( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + 1, + 0, + ); assert!(matches!( store.book(&d), - BookOutcome::LimitExceeded { table: "subscription", .. } + BookOutcome::LimitExceeded { + table: "subscription", + .. + } )); } @@ -574,10 +659,14 @@ mod tests { store.set_caps([(show, alloc, 1000)], [(folder, -1, -1)], [(job, 10, -1)]); let d = delta(show, alloc, folder, job, 6, 0); assert!(applied(store.book(&d))); // 6 <= 10 - // Second booking of 6 would reach 12 > 10 -> rejected. No partial state. + // Second booking of 6 would reach 12 > 10 -> rejected. No partial state. assert!(matches!( store.book(&d), - BookOutcome::LimitExceeded { table: "job", current: 6, limit: 10 } + BookOutcome::LimitExceeded { + table: "job", + current: 6, + limit: 10 + } )); assert_eq!(store.job_cores_in_use(job), 6); } @@ -586,7 +675,11 @@ mod tests { fn unlimited_sentinel_never_rejects() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 1_000_000)], [(folder, -1, -1)], [(job, -1, -1)]); + store.set_caps( + [(show, alloc, 1_000_000)], + [(folder, -1, -1)], + [(job, -1, -1)], + ); let d = delta(show, alloc, folder, job, 500, 4); assert!(applied(store.book(&d))); } @@ -669,7 +762,11 @@ mod tests { job: [(job, (0, 0))].into_iter().collect(), }; store.overwrite_counters(&snap, epoch); - assert_eq!(store.job_cores_in_use(job), 8, "booking confirmed after the snapshot read was erased"); + assert_eq!( + store.job_cores_in_use(job), + 8, + "booking confirmed after the snapshot read was erased" + ); // The following recompute (proc now visible) reconciles cleanly to the true value. let epoch2 = store.begin_recompute(); @@ -759,7 +856,11 @@ mod tests { // A 20-core booking would reach 110 > 100 burst -> must reject (not over-book). assert!(matches!( store.book(&delta(show, alloc, folder, job, 20, 0)), - BookOutcome::LimitExceeded { table: "subscription", current: 90, limit: 100 } + BookOutcome::LimitExceeded { + table: "subscription", + current: 90, + limit: 100 + } )); // A 10-core booking fits exactly at the burst. assert!(applied(store.book(&delta(show, alloc, folder, job, 10, 0)))); @@ -772,7 +873,7 @@ mod tests { store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, 50, -1)]); let d = delta(show, alloc, folder, job, 40, 0); assert!(applied(store.book(&d))); // 40 <= 50 - // Operator lowers the hard cap to 30 live; further bookings must reject. + // Operator lowers the hard cap to 30 live; further bookings must reject. store.apply_limit_change(&LimitChange::JobMaxCores { job_id: job, max_cores: 30, From f4b63f2e6b99988598aad6d90ad93ce379c18d16 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Mon, 29 Jun 2026 10:45:39 -0700 Subject: [PATCH 03/20] Remove unecessary migration and version change --- VERSION.in | 2 +- .../ddl/postgres/migrations/V45__show_scheduler_managed.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION.in b/VERSION.in index b0c101e630..24cffb85f8 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -1.27 +1.26 diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql index 7efeff0925..284ce1cfdd 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql @@ -12,7 +12,7 @@ -- recalculate_subs() is rewritten here (CREATE OR REPLACE) so that the -- 2-hour maintenance task skips scheduler-managed shows, per the -- "recalculate_subs() show-awareness" section of --- docs/_docs/developer-guide/scheduler-accounting.md. The body is otherwise identical to +-- docs/_docs/developer-guide/redis-accounting.md. The body is otherwise identical to -- V20__recalculate_subs_gpu.sql with two narrow changes: -- 1) the initial UPDATE-to-zero of subscription is restricted to shows -- with b_scheduler_managed = false, so Rust-owned rows are never From 66744df48d50e84e3464260f36d9c3ecead70f0c Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Tue, 14 Jul 2026 17:00:32 -0700 Subject: [PATCH 04/20] [scheduler] Correct accounting docs/comments + add recompute cycle metric Address code-review findings on the in-memory accounting model: - scheduler-accounting.md: scope "PG tables are display-only" to scheduler-managed shows (Cuebot-managed shows enforce transactionally); correct failure-mode table + kill-switch to distinguish safe release misses from cap-decrease misses that admit over-cap bookings until reseed. - deploying-scheduler.md: document the N=1 accounting constraint (one scheduler process per managed show; facility distribution alone is insufficient since instances don't share accounting state). - listener.rs: scope the "cannot over-book / self-heals" guarantee to acct_release; note dropped cap-lowering acct_limit_change stays stale-high. - job_dao.rs: comment now references the SQL's live proc aggregates / lagged PG columns instead of the in-memory Store hot-path counters. - metrics/mod.rs: ACCOUNTING_LIMIT_EXCEEDED_TOTAL doc now covers both the authoritative cap rejection and the subscription pre-check skip sites; add RECOMPUTE_CYCLE_DURATION_SECONDS histogram to validate recompute cadence. - core_size.rs: replace obsolete Redis Lua-script reference with the in-memory Store's non-positive-cap semantics. - recompute.rs: record recompute cycle wall-clock duration. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../developer-guide/scheduler-accounting.md | 65 +++++++++++++------ .../getting-started/deploying-scheduler.md | 2 + .../scheduler/src/accounting/listener.rs | 9 ++- .../scheduler/src/accounting/recompute.rs | 5 ++ rust/crates/scheduler/src/dao/job_dao.rs | 7 +- rust/crates/scheduler/src/metrics/mod.rs | 39 +++++++++-- rust/crates/scheduler/src/models/core_size.rs | 6 +- 7 files changed, 97 insertions(+), 36 deletions(-) diff --git a/docs/_docs/developer-guide/scheduler-accounting.md b/docs/_docs/developer-guide/scheduler-accounting.md index 20afe71da3..ac41403fe8 100644 --- a/docs/_docs/developer-guide/scheduler-accounting.md +++ b/docs/_docs/developer-guide/scheduler-accounting.md @@ -65,10 +65,12 @@ Three properties hold: - **The in-memory `Store` is the live enforced state.** It is what the booking check reads and increments. It is seeded from PG at startup and kept fresh by the NOTIFY feed, reconciled to `SUM(proc)` by the recompute. -- **The PG accounting tables are derived, for display only.** For - scheduler-managed shows they are refreshed by the recompute loop; for - Cuebot-managed shows Cuebot writes them transactionally as before. Nothing on - the scheduler's hot path reads them. +- **The PG accounting tables are derived and display-only *for + scheduler-managed shows*.** For those shows they are refreshed by the recompute + loop and nothing on the scheduler's hot path reads them. For Cuebot-managed + shows they are the opposite: Cuebot both reads and writes them transactionally + to enforce caps, exactly as before — they are the live enforced state for those + shows, not display-only. ### Why in-memory, not Redis or PG-on-the-hot-path @@ -377,21 +379,31 @@ refresh; removals (shows that left the managed set) still apply immediately. ## Failure modes and drift -Every failure mode is **safe-direction**: a dropped or delayed signal can only -leave a counter reading *high*, which under-books (too conservative) and -self-heals. A hard cap can only be breached if a counter reads *low*, which only -the recompute-erase hole (closed by the [pending carry-forward](#why-pending-carry-forward-is-required)) +A missed **release** signal is always **safe-direction**: it leaves a *booked +counter* reading *high*, which under-books (too conservative) and self-heals. A +booked counter can breach a hard cap only if it reads *low*, which only the +recompute-erase hole (closed by the [pending carry-forward](#why-pending-carry-forward-is-required)) or an unseeded counter (closed by the [blocking bootstrap and managed-flip seeds](#the-other-backstops-and-seeds)) could cause. +A missed **cap-change** signal is a distinct case and is **not** unconditionally +safe. It leaves a stale *limit*, not a stale booked counter. A missed cap +*increase* is harmless (the store enforces the old, lower limit — too +conservative). But a missed cap *decrease* leaves the store enforcing a **stale +higher** limit, and the hot path will **admit over-cap bookings** against it +until the [limit reseed](#limit-reseed-cap-change-backstop) re-reads the true cap +(up to ~5 min) — or immediately, if the `acct_limit_change` NOTIFY is delivered. +This is the one drift direction that can transiently exceed a cap, bounded by the +reseed interval. + | Failure | Effect | Recovery | |---|---|---| | `acct_release` NOTIFY missed (listener reconnecting) | Store missing a decrement → counter high → under-book | Next recompute (~15 s) overwrites from `SUM(proc)` | -| `acct_limit_change` NOTIFY missed | Store cap stale | Next limit reseed (~5 min) re-reads from PG | +| `acct_limit_change` NOTIFY missed | Store cap stale. A missed *decrease* leaves a stale-higher limit → admits over-cap bookings until reseed; a missed *increase* just under-books | Next limit reseed (~5 min) re-reads from PG | | Scheduler dies between `book` and `proc` INSERT | Booked increment lost on crash (store is in memory) | Bootstrap reseed from `SUM(proc)` on restart | | `proc` INSERT / RQD launch fails after `book` | `rollback` undoes the increment + pending | Immediate; recompute is a further backstop | | Recompute snapshot straddles a live booking | Snapshot misses the proc | Carry-forward keeps the booking; never under-counts | -| Cuebot admin cap change | Store stale on that cap until NOTIFY/limit-reseed | `acct_limit_change` NOTIFY (instant) or limit reseed (~5 min) | +| Cuebot admin cap change | Store stale on that cap until NOTIFY/limit-reseed; a *decrease* not yet applied admits over-cap bookings in the gap | `acct_limit_change` NOTIFY (instant) or limit reseed (~5 min) | | `b_scheduler_managed` toggle mid-flight | Brief window of stale managed-set | Stale-true heals via recompute; stale-false defers to Cuebot (safe); managed-flip seed gates enforcement | | Cuebot NOTIFY kill-switch off | No live releases/cap-changes → counters high → under-book | Recompute / limit reseed still heal; ops alerted by metric | @@ -413,17 +425,28 @@ accounting.notify.enabled=true # default; ${ACCOUNTING_NOTIFY_ENABLED} ``` With the flag **off**, Cuebot still deletes procs and updates caps -transactionally but emits no `pg_notify`. The scheduler's store then stops -receiving live releases and cap changes, so its counters only ever grow (reads -high) → it under-books → the recompute and limit-reseed loops heal it within -their intervals. This is the **safe** direction, so flag-off degrades -gracefully to backstop-only operation; it does not over-book. - -Because flag-off is safe, there is **no startup deployment guardrail** that -refuses to run (the old Redis design had one because a disabled Redis publisher -*over*-counted). Instead, when scheduler-managed shows exist and the flag is off, -Cuebot logs a WARN and exposes a `cuebot_accounting_notify_disabled` metric for -ops visibility — utilization will sag (under-booking), but correctness holds. +transactionally but emits no `pg_notify`. The two suppressed signals degrade +differently: + +- **Missed releases (booked counters):** the store's booked counters only ever + grow (read high) → it under-books → the recompute loop heals within its + interval. This is the **safe** direction. +- **Missed cap changes (limits):** the store's caps go stale until the limit + reseed re-reads PG (~5 min). A cap *decrease* made while the flag is off leaves + the store enforcing a **stale higher** limit and will **admit over-cap + bookings** in that window. This is the one flag-off behavior that is *not* + under-booking. + +So flag-off degrades mostly to backstop-only operation, but it is not +unconditionally safe: booking headroom is bounded by the limit-reseed interval, +not by live cap changes. + +There is **no startup deployment guardrail** that refuses to run (the old Redis +design had one because a disabled Redis publisher *over*-counted the booked +state). Instead, when scheduler-managed shows exist and the flag is off, Cuebot +logs a WARN and exposes a `cuebot_accounting_notify_disabled` metric for ops +visibility — utilization will sag from under-booking, and any cap *decrease* +applied while the flag is off will not be enforced until the next limit reseed. The per-show `b_scheduler_managed` toggle remains the live operational rollback: flip a show back to Cuebot-managed to take it off the scheduler entirely. diff --git a/docs/_docs/getting-started/deploying-scheduler.md b/docs/_docs/getting-started/deploying-scheduler.md index ca7ed14d51..8833ff19f6 100644 --- a/docs/_docs/getting-started/deploying-scheduler.md +++ b/docs/_docs/getting-started/deploying-scheduler.md @@ -39,6 +39,8 @@ To plan your installation of the Distributed Scheduler, consider the following: - **Network**: Low-latency connection to the OpenCue database (same requirements as Cuebot) - **Database**: PostgreSQL with the same schema as Cuebot (no additional tables required). This is the **only** datastore the scheduler needs no Redis or other external store. Per-show resource accounting is held in memory and kept fresh by a PostgreSQL `LISTEN/NOTIFY` feed from Cuebot (see the [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/)) +> **One scheduler process per managed show.** Because resource accounting lives **in memory within each scheduler process** (it is not shared across instances via the database, Redis, or any other store), a given scheduler-managed show must be handled by exactly **one** scheduler process. If two processes manage the same show, each counts only its own bookings, so the two accounting views diverge and per-show/subscription limits are effectively double-counted. Facility-based distribution (`--facility`) is only safe when it provably partitions shows so that no show's clusters are served by more than one process; splitting a single show's work across processes is **not** supported. + ## Architecture Overview The scheduler is organized around **clusters**, which represent unique combinations of: diff --git a/rust/crates/scheduler/src/accounting/listener.rs b/rust/crates/scheduler/src/accounting/listener.rs index ab4035f47a..c709bead0b 100644 --- a/rust/crates/scheduler/src/accounting/listener.rs +++ b/rust/crates/scheduler/src/accounting/listener.rs @@ -20,9 +20,12 @@ //! shows. Cores/gpus are signed deltas (negative for a release). //! - `acct_limit_change`: an enforced cap change from a cueadmin operation. //! -//! Both are best-effort optimisations: a dropped notification (listener reconnecting) -//! only leaves the store reading high → under-book → healed by the next recompute / -//! limit reseed. Nothing here can over-book a hard cap. +//! Both are best-effort optimisations, but their failure modes differ. A dropped +//! `acct_release` (listener reconnecting) only leaves the store reading high → +//! under-book → healed by the next recompute / limit reseed; nothing about a missed +//! release can over-book a hard cap. A dropped `acct_limit_change` that *lowers* a cap +//! is not self-healing in the same way: the store keeps enforcing the stale, higher cap +//! and may temporarily over-book until the next limit reseed picks up the new value. use std::sync::Arc; use std::time::Duration; diff --git a/rust/crates/scheduler/src/accounting/recompute.rs b/rust/crates/scheduler/src/accounting/recompute.rs index 42b2ca5fc0..0f7875c03a 100644 --- a/rust/crates/scheduler/src/accounting/recompute.rs +++ b/rust/crates/scheduler/src/accounting/recompute.rs @@ -77,6 +77,7 @@ pub fn spawn_loop(service: Arc) { limit_exceeded_delta ); + let cycle_start = time::Instant::now(); let result = AssertUnwindSafe(async { if let Err(err) = run_once(&service, &pg_dao).await { warn!("Recompute cycle failed: {err}"); @@ -84,6 +85,10 @@ pub fn spawn_loop(service: Arc) { }) .catch_unwind() .await; + // Cycle-duration signal for validating the recompute_interval cadence: + // the tail must stay comfortably below interval_dur. Recorded even on the + // error path above (the cycle still consumed wall-clock), but not on panic. + metrics::observe_recompute_cycle_duration(cycle_start.elapsed().as_secs_f64()); if let Err(e) = result { error!("Recompute iteration panicked: {:?}", e); } diff --git a/rust/crates/scheduler/src/dao/job_dao.rs b/rust/crates/scheduler/src/dao/job_dao.rs index f4082c8019..03ea3a124d 100644 --- a/rust/crates/scheduler/src/dao/job_dao.rs +++ b/rust/crates/scheduler/src/dao/job_dao.rs @@ -87,8 +87,11 @@ static QUERY_PENDING_BY_SHOW_FACILITY_TAG: &str = r#" -- booking, Cuebot deletes on frame completion, compensation deletes on a failed launch), -- so we sum it directly. Each CTE is scoped to this show ($1) via i_proc_pkshow and -- mirrors the recompute aggregation (so the value equals a fresher copy of the PG --- column), and joins on indexed pk_host / pk_job. Gating on stale PG would otherwise --- (a) over-fetch jobs for caps that are full in the accounting store -> wasted rejections, and worse +-- column), and joins on indexed pk_host / pk_job. This query fetches from PG (live proc +-- aggregates vs the lagged *_resource.int_cores columns) and is independent of the +-- scheduler's in-memory accounting Store used on the hot path. Gating on stale PG would +-- otherwise +-- (a) over-fetch jobs for caps that are already full per the live proc sums -> wasted rejections, and worse -- (b) FALSE-EXCLUDE: a frame completes and frees burst live, but the lagged PG column -- stays high for up to a cycle, dropping the show/folder/job from the fetch and starving -- its (esp. low-priority) jobs until the next recompute. diff --git a/rust/crates/scheduler/src/metrics/mod.rs b/rust/crates/scheduler/src/metrics/mod.rs index 425a7533b9..cde13d477c 100644 --- a/rust/crates/scheduler/src/metrics/mod.rs +++ b/rust/crates/scheduler/src/metrics/mod.rs @@ -65,16 +65,21 @@ lazy_static! { ) .expect("Failed to register time_to_book_seconds histogram"); - // Accounting metrics from accounting/mod.rs + dispatcher/actor.rs + // Accounting metrics from dispatcher/actor.rs + pipeline/matcher.rs // - // Labeled by the table whose cap was hit (subscription / folder / job). Tracks - // dispatch attempts that reached the accounting check only to be rejected by - // it. Used to decide whether the pre-CheckOut pre-check - // optimization described in `pipeline/matcher.rs::process_layer` is worth - // implementing. + // Labeled by the table whose cap was hit (subscription / folder / job). + // Incremented from BOTH sources: + // 1. Authoritative accounting-cap rejections in `apply_booking` + // (dispatcher/actor.rs) — attempts that reached the accounting check + // only to be rejected by it, labeled by the actual table. + // 2. Subscription pre-check skips in `pipeline/matcher.rs::process_layer` + // — layers skipped before checkout when the burst snapshot says the + // requested cores won't fit under the (show, alloc) subscription burst + // (always labeled `subscription`). pub static ref ACCOUNTING_LIMIT_EXCEEDED_TOTAL: CounterVec = register_counter_vec!( "scheduler_accounting_limit_exceeded_total", - "Dispatch attempts rejected by the accounting cap check, labeled by table", + "Dispatch attempts rejected by the accounting cap check plus subscription \ + pre-check skips, labeled by table", &["table"] ) .expect("Failed to register accounting_limit_exceeded_total counter"); @@ -99,6 +104,20 @@ lazy_static! { ) .expect("Failed to register job_query_duration_seconds histogram"); + // Recompute-loop metrics from accounting/recompute.rs + // + // Wall-clock of one full recompute cycle (PG recompute of the accounting + // tables + store reseed from SUM(proc)). This is the validation signal for + // the `accounting.recompute_interval` cadence: the cycle must complete + // comfortably within that interval, so buckets extend past 15s to make an + // interval overrun visible in the tail rather than clip it. + pub static ref RECOMPUTE_CYCLE_DURATION_SECONDS: Histogram = register_histogram!( + "scheduler_recompute_cycle_duration_seconds", + "Duration of one full accounting recompute cycle (PG recompute + store reseed)", + vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 60.0] + ) + .expect("Failed to register recompute_cycle_duration_seconds histogram"); + // Cluster feed metrics from cluster.rs // // Labeled by `cluster_type` (alloc / manual / hostname / hardware) so the @@ -323,6 +342,12 @@ pub fn observe_candidates_per_layer(candidates: usize) { CANDIDATES_PER_LAYER.observe(candidates as f64); } +/// Helper function to observe one recompute cycle's wall-clock duration. +#[inline] +pub fn observe_recompute_cycle_duration(seconds: f64) { + RECOMPUTE_CYCLE_DURATION_SECONDS.observe(seconds); +} + /// Helper function to increment frames dispatched counter #[inline] pub fn increment_frames_dispatched(show_name: &str) { diff --git a/rust/crates/scheduler/src/models/core_size.rs b/rust/crates/scheduler/src/models/core_size.rs index f7f357f7ca..0ac68cdef1 100644 --- a/rust/crates/scheduler/src/models/core_size.rs +++ b/rust/crates/scheduler/src/models/core_size.rs @@ -72,9 +72,9 @@ impl CoreSize { /// Like `from_multiplied` but preserves negative sentinels. Cap fields /// (`folder_resource.int_max_cores`, `job_resource.int_max_cores`) use `-1` as the - /// "unlimited" sentinel; the unguarded division would truncate that to 0. The Lua - /// script's `> 0` guard handles either, but preserving `-1` faithfully keeps cap - /// output meaningful for operators. + /// "unlimited" sentinel; the unguarded division would truncate that to 0. The + /// in-memory Store treats any non-positive cap as "no limit" when enforcing, but + /// preserving `-1` faithfully keeps cap output meaningful for operators. pub fn from_multiplied_cap(centicores: impl Into) -> CoreSize { let c = centicores.into(); if c < 0 { From 955a35936568758e7f24dc02c6dc417a5f3cb4e3 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 15 Jul 2026 16:19:21 -0700 Subject: [PATCH 05/20] Apply review suggestions and improve docs --- .../scheduler/src/accounting/managed_shows.rs | 176 +++++++++--------- rust/crates/scheduler/src/accounting/store.rs | 69 ++++--- 2 files changed, 132 insertions(+), 113 deletions(-) diff --git a/rust/crates/scheduler/src/accounting/managed_shows.rs b/rust/crates/scheduler/src/accounting/managed_shows.rs index d4b1eefb44..543eb04d8a 100644 --- a/rust/crates/scheduler/src/accounting/managed_shows.rs +++ b/rust/crates/scheduler/src/accounting/managed_shows.rs @@ -67,89 +67,11 @@ impl ManagedShowsCache { interval.tick().await; loop { interval.tick().await; - let result = AssertUnwindSafe(async { - match dao.query_managed_show_ids().await { - Ok(ids) => { - let new_set: HashSet = ids.into_iter().collect(); - - // Shows that became scheduler-managed since the last refresh. - // Their enforced caps (notably subscription `burst`) may not be - // in the store yet - bootstrap only seeded shows managed at - // startup, and the periodic limit reseed runs on a slow cadence. - // Publishing them into the cache now would flip the booking hot - // path to enforce against an unseeded burst (== 0 == "reject - // all"). So seed caps FIRST, then publish (the managed-flip gate). - let added: Vec = { - let lock = inner.read().unwrap_or_else(|p| p.into_inner()); - new_set - .iter() - .filter(|id| !lock.contains(*id)) - .copied() - .collect() - }; - if !added.is_empty() { - // Managed-flip blocking gate: seed caps AND booked counters - // for the newly-managed shows BEFORE publishing them into the - // cache. A flipped show may already have live Cuebot procs, so - // the hot path must enforce against real usage from the first - // booking, not against unseeded 0s (== full burst free == - // over-book). The booked seed is a one-shot absolute set per - // show (no epoch bump), so it does not race the recompute - // driver's begin/overwrite sequencing. - let seed = async { - limit_reseed::reseed_limits(&store, &dao).await?; - for show in &added { - let rows = - dao.query_booked_snapshot_for_show(*show).await?; - store.seed_show_booked(&recompute::snapshot_to_counters( - &rows, - &BaselineKeys::default(), - )); - } - Ok::<(), miette::Report>(()) - } - .await; - if let Err(err) = seed { - // Defer publishing only the *additions* this tick: a - // newly-managed show that is not yet in the cache is - // treated as Cuebot-managed (Cuebot still books it via PG - // until the flip lands) - strictly safer than enforcing - // against unseeded state. Retried next tick. - // - // Removals still apply: a show no longer scheduler-managed - // must drop out of the cache regardless of the seed - // outcome, otherwise apply_booking keeps enforcing the - // store for it indefinitely. - warn!( - "Seed for newly-managed show(s) {:?} failed; deferring \ - their cache publish to next tick: {err}", - added - ); - let added_set: HashSet = added.iter().copied().collect(); - let deferred: HashSet = - new_set.difference(&added_set).copied().collect(); - let mut lock = inner.write().unwrap_or_else(|p| p.into_inner()); - *lock = deferred; - return; - } - debug!( - "Seeded caps + booked counters for {} newly-managed \ - show(s) before publishing", - added.len() - ); - } - - let mut lock = inner.write().unwrap_or_else(|p| p.into_inner()); - *lock = new_set; - debug!("ManagedShowsCache refreshed: {} shows", lock.len()); - } - Err(err) => { - warn!("Failed to refresh managed-shows cache: {err}"); - } - } - }) - .catch_unwind() - .await; + // Catch panics so one bad tick can't kill the refresh loop for the + // life of the process. + let result = AssertUnwindSafe(Self::refresh_once(&inner, &dao, &store)) + .catch_unwind() + .await; if let Err(e) = result { error!("Managed-shows refresh iteration panicked: {:?}", e); } @@ -157,6 +79,94 @@ impl ManagedShowsCache { }); } + /// Performs a single refresh tick: re-queries the managed-show set, seeds caps + + /// booked counters for any newly-managed shows, then publishes the new set. Errors + /// are logged rather than propagated - the driver loop must keep running. + async fn refresh_once( + inner: &Arc>>, + dao: &Arc, + store: &Arc, + ) { + let new_set: HashSet = match dao.query_managed_show_ids().await { + Ok(ids) => ids.into_iter().collect(), + Err(err) => { + warn!("Failed to refresh managed-shows cache: {err}"); + return; + } + }; + + // Shows that became scheduler-managed since the last refresh. Their enforced + // caps (notably subscription `burst`) may not be in the store yet - bootstrap + // only seeded shows managed at startup, and the periodic limit reseed runs on a + // slow cadence. Publishing them into the cache now would flip the booking hot + // path to enforce against an unseeded burst (== 0 == "reject all"). So seed caps + // FIRST, then publish (the managed-flip gate). + let added: Vec = { + let lock = inner.read().unwrap_or_else(|p| p.into_inner()); + new_set + .iter() + .filter(|id| !lock.contains(*id)) + .copied() + .collect() + }; + + if !added.is_empty() { + if let Err(err) = Self::seed_newly_managed(store, dao, &added).await { + // Defer publishing only the *additions* this tick: a newly-managed show + // that is not yet in the cache is treated as Cuebot-managed (Cuebot still + // books it via PG until the flip lands) - strictly safer than enforcing + // against unseeded state. Retried next tick. + // + // Removals still apply: a show no longer scheduler-managed must drop out + // of the cache regardless of the seed outcome, otherwise apply_booking + // keeps enforcing the store for it indefinitely. + warn!( + "Seed for newly-managed show(s) {:?} failed; deferring their cache \ + publish to next tick: {err}", + added + ); + let added_set: HashSet = added.iter().copied().collect(); + let deferred: HashSet = new_set.difference(&added_set).copied().collect(); + let mut lock = inner.write().unwrap_or_else(|p| p.into_inner()); + *lock = deferred; + return; + } + debug!( + "Seeded caps + booked counters for {} newly-managed show(s) before publishing", + added.len() + ); + } + + let mut lock = inner.write().unwrap_or_else(|p| p.into_inner()); + *lock = new_set; + debug!("ManagedShowsCache refreshed: {} shows", lock.len()); + } + + /// Seeds a newly-managed show into the store BEFORE it is published into the cache: + /// its caps (subscription burst, folder/job limits) via the limit reseed, then its + /// booked counters via [`Store::seed_show_booked`]. + /// + /// This is the fix for the managed-flip over-book window. A show that just flipped to + /// scheduler-managed may already have live Cuebot procs, but its store counters start + /// at 0. If it were published with counters still at 0, the first booking would see 0 + /// booked, read the whole burst as free, and over-book. Seeding to real PG usage first + /// means the hot path enforces against actual usage from the very first booking. + async fn seed_newly_managed( + store: &Arc, + dao: &Arc, + added: &[Uuid], + ) -> miette::Result<()> { + limit_reseed::reseed_limits(store, dao).await?; + for show in added { + let rows = dao.query_booked_snapshot_for_show(*show).await?; + store.seed_show_booked(&recompute::snapshot_to_counters( + &rows, + &BaselineKeys::default(), + )); + } + Ok(()) + } + pub fn contains(&self, show_id: &Uuid) -> bool { self.inner .read() diff --git a/rust/crates/scheduler/src/accounting/store.rs b/rust/crates/scheduler/src/accounting/store.rs index ba9c7e1c9f..1a73e8a39d 100644 --- a/rust/crates/scheduler/src/accounting/store.rs +++ b/rust/crates/scheduler/src/accounting/store.rs @@ -25,22 +25,29 @@ //! //! ## Pending carry-forward (the hard-cap invariant) //! -//! The recompute reconciles booked counters by absolute-overwrite from a `SUM(proc)` -//! snapshot, which is read OUTSIDE the lock and is therefore stale: it can miss a proc -//! that committed after the read. To stop the overwrite from erasing such a booking (the -//! only way to under-count → over-book a hard cap), each booking is carried as "pending" -//! until a recompute whose snapshot provably includes its `proc` row has run: +//! The recompute overwrites each booked counter with a `SUM(proc)` snapshot. That snapshot +//! is read OUTSIDE the lock, so it is stale by the time the overwrite lands: a proc that +//! commits just after the read is missing from it. Blindly copying it would erase that +//! booking and under-count the cap — the one error that lets a hard cap over-book. So the +//! overwrite copies the snapshot, then adds back any booking the snapshot may have missed. //! -//! - `book` adds the delta to the live counter and to the **in-flight** bucket (proc not -//! yet committed → in no snapshot yet → always carried). -//! - `confirm` (proc committed + RQD launched) moves the delta from in-flight to a -//! **settled** bucket, double-buffered by recompute-epoch parity. -//! - `rollback` (dispatch failed) removes the delta from the live counter and in-flight. -//! - The recompute bumps the epoch under the lock *before* its snapshot read, then on -//! overwrite sets `counter = snapshot + in-flight + settled[both buckets]` and clears -//! only the settled bucket from *before* this epoch — confirms that raced the snapshot -//! read land in the other bucket and survive. Double-counting a booking that is in both -//! the snapshot and a settled bucket is harmless (over-count → under-book → safe). +//! A booking stays "pending" until a snapshot provably contains its `proc` row. Its delta: +//! +//! - `book` → live counter + **in-flight** (proc not committed yet). +//! - `confirm` → in-flight → **settled** (proc committed + RQD launched). +//! - `rollback` → removed from live + in-flight (dispatch failed). +//! +//! In-flight bookings are never in a snapshot, so they are always added back. Settled ones +//! are subtler: some are already in the snapshot (adding them back would double-count), some +//! raced the read and are missing (must be added back). The epoch tells them apart. +//! `begin_recompute` bumps it under the lock *before* the snapshot read; picture the bump as +//! opening a fresh bucket for the pass. Confirms after it — including ones racing the read — +//! land in the new bucket and are added back; confirms before it are already in the snapshot +//! and get cleared. +//! +//! Two buckets alternating by `epoch % 2` cover the only two generations ever live at once. A +//! booking that raced the read but did make the snapshot is added back anyway — a brief +//! over-count (→ under-book, safe) the next pass clears. See `overwrite_counters` for details. use std::collections::HashMap; use std::sync::Mutex; @@ -66,11 +73,10 @@ struct Counter { /// Booked, proc INSERT not yet confirmed committed. In no snapshot → always carried. inflight_cores: i64, inflight_gpus: i64, - /// Bookings that have been confirmed but whose `proc` row may not yet appear in a - /// recompute snapshot. Uses two alternating buckets indexed by `e % 2`: confirms - /// during epoch `e` write to bucket `e % 2`, so the recompute can identify and - /// preserve confirms that raced its snapshot read without clearing ones already - /// captured by it. + /// Confirmed bookings not yet provably in a recompute snapshot. Two buckets indexed by + /// `epoch % 2`; `confirm` writes the live epoch's bucket, so the overwrite can carry the + /// confirms that raced its snapshot read (`keep` bucket) while dropping the ones it + /// already counted (`clear` bucket). See the module docs. settled_cores: [i64; 2], settled_gpus: [i64; 2], } @@ -129,8 +135,9 @@ struct Inner { sub_burst: HashMap<(Uuid, Uuid), i64>, folder_caps: HashMap, job_caps: HashMap, - /// Monotonic recompute epoch. Bumped under the lock at the start of each recompute so - /// `confirm` tags the correct settled bucket relative to the in-flight snapshot read. + /// Monotonic recompute epoch, bumped under the lock at the start of each pass *before* + /// the snapshot read. Its parity picks the settled bucket, so `confirm` and the overwrite + /// agree on which confirms the snapshot already saw. See the module docs. epoch: u64, } @@ -424,14 +431,16 @@ impl Store { } } - /// One-shot absolute seed of a show's booked counters when it is flipped to - /// scheduler-managed, BEFORE it enters the managed-shows cache. At this point the show - /// has no scheduler bookings (the hot path no-ops for unpublished shows), so there is no - /// in-flight/settled pending and no concurrent booking on its keys. Unlike the recompute - /// overwrite this does NOT bump the epoch or touch the settled buckets, so it never - /// interferes with the single recompute driver's begin/overwrite sequencing. Setting the - /// live counter directly is what closes the managed-flip over-book window: the first - /// booking after publish enforces against real usage, not against 0 (= full burst free). + /// Overwrites a show's live booked counters (`cores`/`gpus`) with an absolute + /// snapshot from PG. Called once when a show flips to scheduler-managed, before it is + /// published into the managed-shows cache; see `ManagedShowsCache::seed_newly_managed` + /// for why the seed happens here. + /// + /// This needs no coordination with concurrent bookings: an unpublished show is + /// invisible to the booking hot path, so nothing else reads or mutates its counters + /// while this runs. It also stays clear of the recompute driver: unlike a recompute + /// overwrite it does not bump the epoch or clear the settled buckets, only the live + /// counters, so it cannot interfere with that driver's begin/overwrite sequencing. pub fn seed_show_booked(&self, snapshot: &CounterSnapshot) { let mut inner = self.lock(); for (&k, &(cores, gpus)) in &snapshot.sub { From 615bfdd59440a7a5a8fb1c0cea748793e3254db2 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 15 Jul 2026 16:28:20 -0700 Subject: [PATCH 06/20] Address review comments --- .../workflows/scheduler-stress-pipeline.yml | 18 +++--- .../spcue/service/AccountingNotifier.java | 4 +- .../V45__show_scheduler_managed.sql | 8 +-- .../developer-guide/scheduler-accounting.md | 63 +++++++------------ docs/_docs/developer-guide/scheduler.md | 9 +-- .../getting-started/deploying-scheduler.md | 4 +- .../scheduler/src/accounting/booking_delta.rs | 4 +- .../scheduler/src/accounting/recompute.rs | 2 +- rust/crates/scheduler/src/accounting/store.rs | 8 +-- rust/crates/scheduler/src/config/mod.rs | 2 +- rust/crates/scheduler/src/metrics/mod.rs | 2 +- .../src/pipeline/dispatcher/actor.rs | 2 +- .../scheduler/src/pipeline/entrypoint.rs | 2 +- .../scheduler/src/pipeline/placement.rs | 3 +- .../scheduler/tests/active_scan_superset.rs | 16 ++--- rust/crates/scheduler/tests/stress_tests.rs | 4 +- rust/crates/scheduler/tests/util/stress.rs | 6 +- 17 files changed, 68 insertions(+), 89 deletions(-) diff --git a/.github/workflows/scheduler-stress-pipeline.yml b/.github/workflows/scheduler-stress-pipeline.yml index 2a43ea1648..a2021b3bf5 100644 --- a/.github/workflows/scheduler-stress-pipeline.yml +++ b/.github/workflows/scheduler-stress-pipeline.yml @@ -2,22 +2,23 @@ name: OpenCue Scheduler Stress Pipeline # Runs the scheduler booking + accounting stress suite # (rust/crates/scheduler/tests/stress_tests.rs): a full pipeline::run against a -# seeded farm, with an end-of-run audit that cross-checks the Redis acct:* -# hashes against SUM(proc) in Postgres and asserts cap enforcement. +# seeded farm, with an end-of-run audit that cross-checks the in-memory +# accounting store against SUM(proc) in Postgres and asserts cap enforcement. # # When it runs — and when it deliberately doesn't: # - Pull requests: only when the scheduler crate, its proto dependency, the # DB migrations, or this workflow change. The suite needs a migrated -# Postgres plus a Redis container and takes several minutes — running it -# for Python/CueGUI/docs changes would burn runner time for zero signal. +# Postgres and takes several minutes — running it for Python/CueGUI/docs +# changes would burn runner time for zero signal. # - Nightly on master: catches drift from changes that slipped past the # paths filter (e.g. shared workspace dependencies) and gives a daily # throughput data point under fixed scale. # - Manually (workflow_dispatch): for benchmarking a branch at custom scale. # # What is a gate vs. what is informational: -# - The job FAILS on correctness regressions: accounting drift between Redis -# and Postgres, cap breaches (subscription burst / job max-cores), booking +# - The job FAILS on correctness regressions: accounting drift between the +# in-memory store and Postgres, cap breaches (subscription burst / job +# max-cores), booking # liveness (<90% drain, no saturation rejections), or leftover test data. # - The throughput numbers (frames/s) are reported in the step summary but # are NOT asserted on: shared runners are too noisy for perf gating. For @@ -66,9 +67,8 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 45 - # The suite's Postgres half. Redis is NOT listed here on purpose: the test - # starts its own throwaway Redis via testcontainers, so all accounting - # state is guaranteed to die with the test process. + # The suite's Postgres service. Accounting state lives in-process inside the + # test itself, so nothing else needs to be provisioned here. services: postgres: image: postgres:15.1 diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java index 6b6b2b188c..5a3ca4fd22 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java @@ -32,8 +32,8 @@ * Publishes per-release and admin cap-change accounting deltas to the standalone Rust scheduler via * Postgres {@code LISTEN/NOTIFY}. Every notification is emitted with {@code pg_notify(channel, * payload)} inside the same transaction as the DB write it describes, so the payload is - * delivered if and only if that transaction commits (this replaces the old afterCommit Redis - * publish with a transactional, stronger failure model). + * delivered if and only if that transaction commits, so a notification can never describe a + * write that rolled back. * *

* See the Scheduler Accounting Reference at diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql index 284ce1cfdd..45e76df10d 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql @@ -4,15 +4,15 @@ -- true -> Rust scheduler is authoritative; Cuebot must not -- mutate the show's accounting rows. -- --- The flag is wired through ShowDao/ShowInterface in PR-A but has no live --- consumer until PR-B (Cuebot Redis publisher + show-aware unbookProc) and --- PR-C (Rust scheduler accounting module). In PR-A all rows stay false in +-- The flag is wired through ShowDao/ShowInterface first, but has no live +-- consumer until the Cuebot accounting notifier (show-aware unbookProc) and +-- the Rust scheduler accounting module land. Until then all rows stay false in -- practice, so behavior is unchanged. -- -- recalculate_subs() is rewritten here (CREATE OR REPLACE) so that the -- 2-hour maintenance task skips scheduler-managed shows, per the -- "recalculate_subs() show-awareness" section of --- docs/_docs/developer-guide/redis-accounting.md. The body is otherwise identical to +-- docs/_docs/developer-guide/scheduler-accounting.md. The body is otherwise identical to -- V20__recalculate_subs_gpu.sql with two narrow changes: -- 1) the initial UPDATE-to-zero of subscription is restricted to shows -- with b_scheduler_managed = false, so Rust-owned rows are never diff --git a/docs/_docs/developer-guide/scheduler-accounting.md b/docs/_docs/developer-guide/scheduler-accounting.md index ac41403fe8..d9e3abfb40 100644 --- a/docs/_docs/developer-guide/scheduler-accounting.md +++ b/docs/_docs/developer-guide/scheduler-accounting.md @@ -30,14 +30,10 @@ Cuebot keeps the PG accounting tables fresh for CueGUI, but no external store sits on the booking hot path. A booking is a lock-guarded, in-process atomic check-and-increment. -This replaces an earlier Redis-backed design. Redis was introduced to let the -counters be shared across N scheduler instances, but the scheduler is and will -remain **single-instance (N=1)**. At N=1, Redis's only unique benefit (splitting -one show across instances) is unreachable, while it manufactured an entire class -of accounting-drift bugs (limit-seeding fail-closed, mass dispatch rejection, -double-booking, CAS starvation). A single in-process counter makes that bug -class *structurally impossible*: there is exactly one writer and one reader of -the booked state, and the check and the increment happen under the same lock. +Keeping the counter in-process makes a whole class of accounting-drift bugs +*structurally impossible*: there is exactly one writer and one reader of the +booked state, and the check and the increment happen under the same lock, so the +enforced state can never race against the writer that produced it. Source: - `rust/crates/scheduler/src/accounting/` (the store, listener, and backstop loops) @@ -72,22 +68,16 @@ Three properties hold: to enforce caps, exactly as before — they are the live enforced state for those shows, not display-only. -### Why in-memory, not Redis or PG-on-the-hot-path +### Why in-memory, not PG-on-the-hot-path -- **PG-on-the-hot-path** is what the scheduler was built to escape: the - scheduler's booking rate hammered the same accounting rows Cuebot's - `HostReportHandler` writes, and lock waits on `subscription`, - `folder_resource`, and `job_resource` limited throughput. -- **Redis** decoupled the hot path from PG locks and could in principle be - shared across schedulers, but at N=1 it bought nothing the in-process store - does not, and every reseed had to defend against a read/write race with live - hot-path writes (the `acct:seq` compare-and-swap). That race only exists - because the counter lives in a separate process from the writer. Move the - counter in-process and the race — and the CAS, and the retry loop, and the - starvation failure mode — all disappear. +**PG-on-the-hot-path** is what the scheduler was built to escape: the +scheduler's booking rate hammered the same accounting rows Cuebot's +`HostReportHandler` writes, and lock waits on `subscription`, +`folder_resource`, and `job_resource` limited throughput. -The in-process store keeps the hot path off PG locks (the win Redis gave) while -removing the cross-process coordination Redis required (the cost Redis added). +The in-process store keeps the hot path off PG locks while avoiding any +cross-process coordination: the check and the increment happen in one place, +under one lock, in the same process that owns the show's bookings. --- @@ -144,10 +134,9 @@ The brief stale window after a toggle is safe in both directions — see The five accounting tables in PG are subscription, folder, job, layer, and department point. The scheduler tracks and enforces only **three**: subscription (burst), folder (`int_max_cores`/`int_max_gpus`), and job -(`int_max_cores`/`int_max_gpus`). The booking Lua this replaces incremented -layer and point counters too, but the booking check never *read* them, so they -are not kept in the store. (Layer/point limits are still visible to CueGUI via -PG, unchanged.) +(`int_max_cores`/`int_max_gpus`). Layer and point counters are not kept in the +store because the booking check never *reads* them. (Layer/point limits are +still visible to CueGUI via PG, unchanged.) ### One lock, pure in-memory critical sections @@ -197,8 +186,7 @@ After `book` returns `Applied`, the dispatcher carries the delta through to the frame. It drops the `pending` portion of the delta but keeps the booked increment. - **`rollback`** runs if the `proc` INSERT or RQD launch fails. It undoes both - the booked increment and the `pending` delta (the in-process equivalent of the - old force-rollback Lua). + the booked increment and the `pending` delta. Exactly one of `confirm`/`rollback` runs per successful `book`. Both ignore current managed status: if `apply_booking` applied a delta, it must be settled @@ -219,10 +207,8 @@ recompute passes. Each notification is emitted with `SELECT pg_notify(channel, payload)` **in the same transaction** as the PG write it describes. `pg_notify` is transactional: the notification is delivered if and only if the -enclosing transaction commits, and is discarded on rollback. This is strictly -better than the old Redis `afterCommit` publish, which ran *after* the commit -and so had a partial-failure window (commit succeeds, publish fails). Here there -is no such window — the DELETE and its release signal are atomic. +enclosing transaction commits, and is discarded on rollback. There is no +partial-failure window — the DELETE and its release signal are atomic. The scheduler listens on both channels with a dedicated `PgListener` (`accounting/listener.rs`), separate from the query pool. On any connection @@ -441,9 +427,9 @@ So flag-off degrades mostly to backstop-only operation, but it is not unconditionally safe: booking headroom is bounded by the limit-reseed interval, not by live cap changes. -There is **no startup deployment guardrail** that refuses to run (the old Redis -design had one because a disabled Redis publisher *over*-counted the booked -state). Instead, when scheduler-managed shows exist and the flag is off, Cuebot +There is **no startup deployment guardrail** that refuses to run: disabling the +notifier makes the scheduler *under*-count rather than over-count, so it fails +safe. Instead, when scheduler-managed shows exist and the flag is off, Cuebot logs a WARN and exposes a `cuebot_accounting_notify_disabled` metric for ops visibility — utilization will sag from under-booking, and any cap *decrease* applied while the flag is off will not be enforced until the next limit reseed. @@ -480,10 +466,9 @@ can be a single in-process critical section, and the recompute can overwrite absolutely without coordinating with any peer. This is the assumption that makes the whole design correct *and* makes the drift bug class go away. -This is a deliberate trade. The Redis design existed to allow N>1 schedulers to -share counters, but at this scale N=1 is expected for the foreseeable future and -the only thing N>1 would buy (splitting one show across instances) is not -needed. +This is a deliberate trade. At this scale N=1 is expected for the foreseeable +future, and the only thing N>1 would buy (splitting one show across instances) +is not needed. **Revisit trigger:** before ever running more than one scheduler instance that could book the same show. At that point the in-memory store is no longer a diff --git a/docs/_docs/developer-guide/scheduler.md b/docs/_docs/developer-guide/scheduler.md index 9c9528e15d..0083ad155e 100644 --- a/docs/_docs/developer-guide/scheduler.md +++ b/docs/_docs/developer-guide/scheduler.md @@ -611,12 +611,9 @@ Historically these counters lived only in PostgreSQL and were updated transactio by Cuebot on every booking and release. As the Rust scheduler took over dispatch, the hot path was hammering the same accounting rows Cuebot's `HostReportHandler` writes to, and lock waits on `subscription`, `folder_resource`, and `job_resource` started -limiting throughput. A short-lived Redis-backed design moved the hot path off PG locks, -but at a single scheduler instance (N=1) it added cross-process coordination (an -`acct:seq` compare-and-swap on every reseed) without the multi-instance benefit it was -built for, and that coordination spawned a class of accounting-drift bugs. +limiting throughput. -The current design replaces both with an **in-memory `Store`** that is the single source +The current design moves the hot path off PG locks with an **in-memory `Store`** that is the single source of truth for booked counters: there is exactly one writer and reader, and the booking check-and-increment is atomic under one lock. PostgreSQL remains the durable record (the `proc` rows), and Cuebot feeds live releases and cap changes via `LISTEN/NOTIFY`. @@ -919,7 +916,7 @@ accounting: - **`managed_shows_ttl`** (default 30s): how often the cache of `b_scheduler_managed` shows is refreshed. -The accounting store needs only PostgreSQL no Redis or other external store. The +The accounting store needs only PostgreSQL; no other external store is required. The matching Cuebot side emits the live feed when `accounting.notify.enabled=true` (default), riding the existing PG connection. diff --git a/docs/_docs/getting-started/deploying-scheduler.md b/docs/_docs/getting-started/deploying-scheduler.md index 8833ff19f6..629b7d08be 100644 --- a/docs/_docs/getting-started/deploying-scheduler.md +++ b/docs/_docs/getting-started/deploying-scheduler.md @@ -37,9 +37,9 @@ To plan your installation of the Distributed Scheduler, consider the following: - **Memory**: Minimum 2GB RAM per scheduler instance (scales with number of hosts cached) - **CPU**: 2-4 cores recommended per instance - **Network**: Low-latency connection to the OpenCue database (same requirements as Cuebot) -- **Database**: PostgreSQL with the same schema as Cuebot (no additional tables required). This is the **only** datastore the scheduler needs no Redis or other external store. Per-show resource accounting is held in memory and kept fresh by a PostgreSQL `LISTEN/NOTIFY` feed from Cuebot (see the [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/)) +- **Database**: PostgreSQL with the same schema as Cuebot (no additional tables required). This is the **only** datastore the scheduler needs. Per-show resource accounting is held in memory and kept fresh by a PostgreSQL `LISTEN/NOTIFY` feed from Cuebot (see the [Scheduler Accounting Reference](/docs/developer-guide/scheduler-accounting/)) -> **One scheduler process per managed show.** Because resource accounting lives **in memory within each scheduler process** (it is not shared across instances via the database, Redis, or any other store), a given scheduler-managed show must be handled by exactly **one** scheduler process. If two processes manage the same show, each counts only its own bookings, so the two accounting views diverge and per-show/subscription limits are effectively double-counted. Facility-based distribution (`--facility`) is only safe when it provably partitions shows so that no show's clusters are served by more than one process; splitting a single show's work across processes is **not** supported. +> **One scheduler process per managed show.** Because resource accounting lives **in memory within each scheduler process** (it is not shared across instances via the database or any other store), a given scheduler-managed show must be handled by exactly **one** scheduler process. If two processes manage the same show, each counts only its own bookings, so the two accounting views diverge and per-show/subscription limits are effectively double-counted. Facility-based distribution (`--facility`) is only safe when it provably partitions shows so that no show's clusters are served by more than one process; splitting a single show's work across processes is **not** supported. ## Architecture Overview diff --git a/rust/crates/scheduler/src/accounting/booking_delta.rs b/rust/crates/scheduler/src/accounting/booking_delta.rs index 02d1c7cb89..4782593693 100644 --- a/rust/crates/scheduler/src/accounting/booking_delta.rs +++ b/rust/crates/scheduler/src/accounting/booking_delta.rs @@ -13,8 +13,8 @@ use uuid::Uuid; /// A single booking-or-release applied to the in-memory accounting store. Carries only the -/// three enforced vertices (subscription/folder/job); the legacy Lua incremented layer and -/// point too but never read them, so they are not tracked. +/// three enforced vertices (subscription/folder/job); layer and point are intentionally not +/// tracked because nothing reads or enforces them. /// /// `core_delta`/`gpu_delta` are signed: positive on a booking, negative on a release. /// Rollback and release are expressed as the store subtracting/adding these directly, so diff --git a/rust/crates/scheduler/src/accounting/recompute.rs b/rust/crates/scheduler/src/accounting/recompute.rs index 0f7875c03a..ea2673fa38 100644 --- a/rust/crates/scheduler/src/accounting/recompute.rs +++ b/rust/crates/scheduler/src/accounting/recompute.rs @@ -144,7 +144,7 @@ pub async fn reseed_store_once(service: &AccountingService) -> Result<()> { /// several rows fold into the same sub/folder/job key - aggregate before converting. /// /// Layer and point are intentionally absent: the booking check never reads them, so they -/// are not tracked in the store (the legacy Lua incremented them but never enforced them). +/// are not tracked in the store. pub(crate) fn snapshot_to_counters( rows: &[BookedSnapshotRow], baseline: &BaselineKeys, diff --git a/rust/crates/scheduler/src/accounting/store.rs b/rust/crates/scheduler/src/accounting/store.rs index 1a73e8a39d..2bff53a385 100644 --- a/rust/crates/scheduler/src/accounting/store.rs +++ b/rust/crates/scheduler/src/accounting/store.rs @@ -11,13 +11,11 @@ // the License. //! In-process accounting store. Single source of truth for the scheduler's booking -//! decisions, replacing the Redis-backed counters. See -//! `docs/_docs/developer-guide/scheduler-accounting.md` for the full design. +//! decisions. See `docs/_docs/developer-guide/scheduler-accounting.md` for the full design. //! //! Only the three enforced vertices are tracked: subscription (`burst`), folder -//! (`int_max_cores`/`int_max_gpus`) and job (`int_max_cores`/`int_max_gpus`). The -//! booking enforcement this replaces incremented layer and point counters too, but -//! never read them, so they are not kept here. +//! (`int_max_cores`/`int_max_gpus`) and job (`int_max_cores`/`int_max_gpus`). Layer and +//! point counters are not kept here because the booking check never reads them. //! //! Concurrency: one `Mutex` guards the whole state. Every critical section is pure //! in-memory arithmetic (no I/O, no `.await`), so contention is negligible at this diff --git a/rust/crates/scheduler/src/config/mod.rs b/rust/crates/scheduler/src/config/mod.rs index bc8343fd4f..88cffd8945 100644 --- a/rust/crates/scheduler/src/config/mod.rs +++ b/rust/crates/scheduler/src/config/mod.rs @@ -52,7 +52,7 @@ pub struct AccountingConfig { /// Cadence at which booked counters are reconciled from `SUM(proc)` into the in-memory /// store (and the PG accounting tables for CueGUI). This is the primary utilization /// backstop for releases now that the live `acct_release` NOTIFY feeds the store - /// between ticks, so it runs tighter than the legacy Redis cadence. + /// between ticks. #[serde(with = "humantime_serde")] pub recompute_interval: Duration, /// Cadence at which enforced caps (subscription burst, folder/job max cores+gpus) are diff --git a/rust/crates/scheduler/src/metrics/mod.rs b/rust/crates/scheduler/src/metrics/mod.rs index cde13d477c..a351a0bf36 100644 --- a/rust/crates/scheduler/src/metrics/mod.rs +++ b/rust/crates/scheduler/src/metrics/mod.rs @@ -89,7 +89,7 @@ lazy_static! { // `job_max_cores` cap would otherwise be re-checked-out and re-rejected up to // `host_candidate_attempts_per_layer` times every pass; the pre-check returns // early instead. Kept distinct from `ACCOUNTING_LIMIT_EXCEEDED_TOTAL{table="job"}` - // (Lua-rejection pressure) so this reads cleanly as "wasted attempts avoided". + // (booking-rejection pressure) so this reads cleanly as "wasted attempts avoided". pub static ref JOB_CAP_PRECHECK_SKIP_TOTAL: Counter = register_counter!( "scheduler_job_cap_precheck_skip_total", "Layers skipped pre-checkout because the job is already at its core cap" diff --git a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs index f82e47743d..d3b792cc52 100644 --- a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs +++ b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs @@ -266,7 +266,7 @@ impl RqdDispatcherService { let frame_str = format!("{}", frame); // Compute the job's remaining core budget. `job_max_cores <= 0` - // means unlimited (matches the Lua's `job_max > 0` guard). + // means unlimited (OpenCue's "unlimited" cap sentinel). let job_cores_remaining = if layer.job_max_cores > 0 { let remaining = layer.job_max_cores - (job_cores_in_use + job_cores_booked); // Normalize the frame's request into the canonical positive core diff --git a/rust/crates/scheduler/src/pipeline/entrypoint.rs b/rust/crates/scheduler/src/pipeline/entrypoint.rs index 18035b24d5..e678ed3bbe 100644 --- a/rust/crates/scheduler/src/pipeline/entrypoint.rs +++ b/rust/crates/scheduler/src/pipeline/entrypoint.rs @@ -47,7 +47,7 @@ pub async fn run(cluster_feed: ClusterFeed) -> miette::Result<()> { let accounting = accounting_service().await?; bootstrap::run_blocking_reseed(&accounting).await?; // Backstops: recompute reconciles booked counters from proc; limit reseed refreshes - // caps. The live NOTIFY listener feeds releases + cap changes between ticks. N=1 only. + // caps. The live NOTIFY listener feeds releases + cap changes between ticks. recompute::spawn_loop(accounting.clone()); limit_reseed::spawn_loop(accounting.clone()); listener::spawn_loop(accounting.store().clone()); diff --git a/rust/crates/scheduler/src/pipeline/placement.rs b/rust/crates/scheduler/src/pipeline/placement.rs index 64eb4f088e..44fcfb7588 100644 --- a/rust/crates/scheduler/src/pipeline/placement.rs +++ b/rust/crates/scheduler/src/pipeline/placement.rs @@ -98,8 +98,7 @@ pub fn fits_floor(host: &Host, profile: &LayerProfile) -> bool { /// A bound is *omitted* from the minimum (rather than contributing 0) when: /// - a physical `_min` is 0 — the layer makes no demand on that dim, /// so the dim cannot constrain anything; or -/// - a cap is `<= 0` — OpenCue's "unlimited" sentinel (mirrors the Lua -/// `> 0` cap guard); or +/// - a cap is `<= 0` — OpenCue's "unlimited" sentinel; or /// - the layer has no core demand (`cores_min == 0`), in which case the /// core-denominated caps have no unit to divide by. /// diff --git a/rust/crates/scheduler/tests/active_scan_superset.rs b/rust/crates/scheduler/tests/active_scan_superset.rs index 928a8ad5cd..06e01a261e 100644 --- a/rust/crates/scheduler/tests/active_scan_superset.rs +++ b/rust/crates/scheduler/tests/active_scan_superset.rs @@ -13,12 +13,12 @@ //! Awake-gate superset guard, kept in its own test binary. //! //! This lives apart from `stress_tests.rs` on purpose: `OVERRIDE_CONFIG` is a -//! process-global `OnceCell` (first-writer-wins), and this test installs a -//! config with a dummy Redis port it never dials. Sharing a process with -//! `stress_booking_and_accounting` would let whichever runs first pin the -//! config, so this test's dummy port could clobber the booking suite's dynamic -//! testcontainer port (or vice-versa). A separate binary = a separate process = -//! a private `OVERRIDE_CONFIG`, so neither can race the other. +//! process-global `OnceCell` (first-writer-wins), and this test installs its own +//! static config. Sharing a process with `stress_booking_and_accounting` would +//! let whichever runs first pin the config, so this test's config could clobber +//! the booking suite's dynamic testcontainer port (or vice-versa). A separate +//! binary = a separate process = a private `OVERRIDE_CONFIG`, so neither can race +//! the other. mod util; @@ -38,8 +38,8 @@ mod active_scan_suite { /// for every cluster the per-cluster dispatch query returns a job for, the /// scan must surface at least one of that cluster's tags. A regression that /// over-narrows `QUERY_ACTIVE_TAGS` (dropping a needed row) would starve - /// that cluster's jobs; this catches it directly against Postgres. Pure-PG - /// (no Redis), so it only needs a migrated database. + /// that cluster's jobs; this catches it directly against Postgres. It only + /// needs a migrated database. #[actix::test] async fn stress_active_scan_is_superset_of_per_cluster_query() { // This test never books, so accounting is inert here. This binary's diff --git a/rust/crates/scheduler/tests/stress_tests.rs b/rust/crates/scheduler/tests/stress_tests.rs index 0c971da0f5..87f7457db0 100644 --- a/rust/crates/scheduler/tests/stress_tests.rs +++ b/rust/crates/scheduler/tests/stress_tests.rs @@ -87,7 +87,7 @@ mod stress_suite { }; use crate::util::{get_waiting_frames_count, test_connection_pool, WaitingFrameClause}; - /// Every table label the booking Lua can reject on. + /// Every table label the booking check can reject on. const LIMIT_TABLES: [&str; 5] = ["subscription", "folder", "job", "folder_gpus", "job_gpus"]; fn env_usize(name: &str, default: usize) -> usize { @@ -343,7 +343,7 @@ mod stress_suite { }; // Phase 2: demand >> caps. Three tight subscription bursts plus a per-job - // max-cores cap make the Redis Lua the binding constraint; most dispatch + // max-cores cap make the accounting store the binding constraint; most dispatch // attempts must be rejected by it without ever breaching a cap. let sat_spec = FarmSpec { prefix: format!("stress_s_{run_id}"), diff --git a/rust/crates/scheduler/tests/util/stress.rs b/rust/crates/scheduler/tests/util/stress.rs index 68446f927f..b2269ccf12 100644 --- a/rust/crates/scheduler/tests/util/stress.rs +++ b/rust/crates/scheduler/tests/util/stress.rs @@ -14,7 +14,7 @@ //! (`tests/stress_tests.rs`): //! //! - [`create_stress_config`]: scheduler config tuned so a test run terminates on -//! its own and the Redis hot path is the accounting system under test +//! its own and the in-memory accounting store is the system under test //! (reconciliation loops pushed out beyond the test horizon). //! - [`seed_farm`]: deterministic bulk seeding of a complete farm (facility → //! show/allocs/subscriptions → hosts/tags → jobs/layers/frames) using multi-row @@ -23,7 +23,7 @@ //! drains, so `pipeline::run` would loop forever; the watchdog pauses the phase's //! jobs once bookings stop growing, which lets the feed quit gracefully via //! `empty_job_cycles_before_quiting`. -//! - [`audit_accounting`]: cross-checks every Redis `acct:*` hash touched by the +//! - [`audit_accounting`]: cross-checks the in-memory store's booked counters for the //! show against `SUM(proc)` in Postgres (the canonical record), plus host/frame/ //! stat invariants and cap enforcement (subscription burst, job max-cores). //! - [`clean_up_stress_data`] / [`residue_counts`]: removes everything matching the @@ -165,7 +165,7 @@ pub struct FarmSpec { /// Per-(show, alloc) subscription size, whole cores. pub sub_size_cores: i64, /// Per-(show, alloc) subscription burst, whole cores. This is the cap the - /// Redis Lua enforces on the hot path. + /// booking check enforces on the hot path. pub sub_burst_cores: i64, pub manual_tag_count: usize, pub job_count: usize, From d1c685ef1e2bd8c8510795d631ba610c2fe9e3de Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 15 Jul 2026 16:33:12 -0700 Subject: [PATCH 07/20] Changing migrations is forbidden --- .../postgres/migrations/V45__show_scheduler_managed.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql index 45e76df10d..284ce1cfdd 100644 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V45__show_scheduler_managed.sql @@ -4,15 +4,15 @@ -- true -> Rust scheduler is authoritative; Cuebot must not -- mutate the show's accounting rows. -- --- The flag is wired through ShowDao/ShowInterface first, but has no live --- consumer until the Cuebot accounting notifier (show-aware unbookProc) and --- the Rust scheduler accounting module land. Until then all rows stay false in +-- The flag is wired through ShowDao/ShowInterface in PR-A but has no live +-- consumer until PR-B (Cuebot Redis publisher + show-aware unbookProc) and +-- PR-C (Rust scheduler accounting module). In PR-A all rows stay false in -- practice, so behavior is unchanged. -- -- recalculate_subs() is rewritten here (CREATE OR REPLACE) so that the -- 2-hour maintenance task skips scheduler-managed shows, per the -- "recalculate_subs() show-awareness" section of --- docs/_docs/developer-guide/scheduler-accounting.md. The body is otherwise identical to +-- docs/_docs/developer-guide/redis-accounting.md. The body is otherwise identical to -- V20__recalculate_subs_gpu.sql with two narrow changes: -- 1) the initial UPDATE-to-zero of subscription is restricted to shows -- with b_scheduler_managed = false, so Rust-owned rows are never From 3c187173b35c373c67a394789b171fc12cf7f1cc Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 15 Jul 2026 16:42:19 -0700 Subject: [PATCH 08/20] Spotless apply --- .../com/imageworks/spcue/service/AccountingNotifier.java | 4 ++-- docs/_docs/developer-guide/scheduler-accounting.md | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java index 5a3ca4fd22..d34e8e434f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java @@ -32,8 +32,8 @@ * Publishes per-release and admin cap-change accounting deltas to the standalone Rust scheduler via * Postgres {@code LISTEN/NOTIFY}. Every notification is emitted with {@code pg_notify(channel, * payload)} inside the same transaction as the DB write it describes, so the payload is - * delivered if and only if that transaction commits, so a notification can never describe a - * write that rolled back. + * delivered if and only if that transaction commits, so a notification can never describe a write + * that rolled back. * *

* See the Scheduler Accounting Reference at diff --git a/docs/_docs/developer-guide/scheduler-accounting.md b/docs/_docs/developer-guide/scheduler-accounting.md index d9e3abfb40..c751ac1d59 100644 --- a/docs/_docs/developer-guide/scheduler-accounting.md +++ b/docs/_docs/developer-guide/scheduler-accounting.md @@ -31,9 +31,10 @@ sits on the booking hot path. A booking is a lock-guarded, in-process atomic check-and-increment. Keeping the counter in-process makes a whole class of accounting-drift bugs -*structurally impossible*: there is exactly one writer and one reader of the -booked state, and the check and the increment happen under the same lock, so the -enforced state can never race against the writer that produced it. +*structurally impossible*: the booked state is owned by a single scheduler +process and every mutation is serialized through the same mutex, so the check +and the increment happen atomically under one lock and the enforced state can +never race against the writer that produced it. Source: - `rust/crates/scheduler/src/accounting/` (the store, listener, and backstop loops) From 78b13897e6c0c53fc7b8b0a419be156fbb3b9553 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 15 Jul 2026 17:09:53 -0700 Subject: [PATCH 09/20] Optimize CICD --- .github/workflows/rust-pipeline.yml | 54 +- rust/.gitignore | 1 - rust/Cargo.lock | 6423 +++++++++++++++++++++++++++ 3 files changed, 6471 insertions(+), 7 deletions(-) create mode 100644 rust/Cargo.lock diff --git a/.github/workflows/rust-pipeline.yml b/.github/workflows/rust-pipeline.yml index 1d049df730..459b726c6b 100644 --- a/.github/workflows/rust-pipeline.yml +++ b/.github/workflows/rust-pipeline.yml @@ -14,6 +14,14 @@ env: # Be more tolerant of registry hiccups on Windows. CARGO_NET_RETRY: 10 RUSTUP_MAX_RETRIES: 10 + # CI never runs a debugger, so full debug symbols (`debug=2`) are wasted + # compile + link time. `line-tables-only` keeps panic backtraces (file:line) + # so test failures stay readable, while cutting most debuginfo generation. + # Set via env (CI-only) rather than Cargo.toml so local dev keeps full symbols. + # Applies to every job, Windows included. `test` inherits from `dev`, but we + # set both explicitly so the override sticks regardless of inheritance. + CARGO_PROFILE_DEV_DEBUG: line-tables-only + CARGO_PROFILE_TEST_DEBUG: line-tables-only jobs: build: @@ -24,7 +32,12 @@ jobs: - uses: actions/checkout@v4 - name: Install X11 dev libs run: | - sudo apt-get update && sudo apt-get install -y libx11-dev protobuf-compiler libcurl4-openssl-dev + # `mold` is a fast linker — cuts a large slice off Rust link time + # (609 deps to link). Invoked below via `mold -run`, which redirects + # the linker through LD_PRELOAD, so it needs no .cargo/config.toml + # (which would force mold on every local dev too) and works with the + # runner's default gcc regardless of version. + sudo apt-get update && sudo apt-get install -y libx11-dev protobuf-compiler libcurl4-openssl-dev mold curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rust-install.sh bash ./rust-install.sh -y - name: Cache cargo deps @@ -32,14 +45,20 @@ jobs: with: workspaces: rust cache-on-failure: true - - name: Build + # Build + test share a SINGLE (debug) profile so artifacts are reused — + # previously `cargo build --release` then `cargo test` compiled the whole + # workspace + all deps twice (release then debug), roughly doubling the + # job. CI only needs to prove it compiles and passes tests, which debug + # does. `--no-run` splits compile-vs-run timing; `--verbose` dropped to + # cut log I/O. + - name: Build tests run: | cd rust - cargo build --release --verbose + mold -run cargo test --no-run - name: Run tests run: | cd rust - cargo test --verbose + cargo test clippy: runs-on: ubuntu-22.04 @@ -49,7 +68,7 @@ jobs: - uses: actions/checkout@v4 - name: Install X11 dev libs run: | - sudo apt-get update && sudo apt-get install -y libx11-dev protobuf-compiler libcurl4-openssl-dev + sudo apt-get update && sudo apt-get install -y libx11-dev protobuf-compiler libcurl4-openssl-dev mold curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rust-install.sh bash ./rust-install.sh -y - name: Cache cargo deps @@ -60,7 +79,7 @@ jobs: - name: Run Clippy run: | cd rust - cargo clippy --verbose + mold -run cargo clippy --verbose windows-clippy: runs-on: windows-latest @@ -68,6 +87,19 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Speed up Windows build (exclude build dirs from Defender) + shell: pwsh + run: | + # Real-time AV scanning of every compiler artifact (rustc writes + # thousands of .rlib/.o/.exe files) is the dominant hidden cost on + # Windows Rust CI. Exclude the workspace + cargo dirs and disable + # realtime monitoring outright — typically a 30-50% speedup. All + # calls are best-effort (Tamper Protection may block some). + Set-MpPreference -DisableRealtimeMonitoring $true -ErrorAction SilentlyContinue + Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" -ErrorAction SilentlyContinue + Add-MpPreference -ExclusionPath "$env:USERPROFILE\.cargo" -ErrorAction SilentlyContinue + Add-MpPreference -ExclusionProcess "cargo.exe" -ErrorAction SilentlyContinue + Add-MpPreference -ExclusionProcess "rustc.exe" -ErrorAction SilentlyContinue - name: Install Rust uses: dtolnay/rust-toolchain@stable with: @@ -91,6 +123,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Speed up Windows build (exclude build dirs from Defender) + shell: pwsh + run: | + # See windows-clippy for rationale — AV scanning dominates Windows + # Rust CI time; excluding build dirs is a 30-50% win. Best-effort. + Set-MpPreference -DisableRealtimeMonitoring $true -ErrorAction SilentlyContinue + Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE" -ErrorAction SilentlyContinue + Add-MpPreference -ExclusionPath "$env:USERPROFILE\.cargo" -ErrorAction SilentlyContinue + Add-MpPreference -ExclusionProcess "cargo.exe" -ErrorAction SilentlyContinue + Add-MpPreference -ExclusionProcess "rustc.exe" -ErrorAction SilentlyContinue - name: Install Rust uses: dtolnay/rust-toolchain@stable - name: Install Protobuf diff --git a/rust/.gitignore b/rust/.gitignore index 05ded07a61..07f846ad9a 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -8,7 +8,6 @@ config/rqd.local_docker.yaml /sandbox/kafka* /reference -Cargo.lock # Localized files only meant for building docker images locally proto diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000000..8265dd50cb --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,6423 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "actix" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7fa236829ba0841304542f7614c42b80fca007455315c45c785ccfa873a85b" +dependencies = [ + "actix-macros", + "actix-rt", + "actix_derive", + "bitflags 2.13.1", + "bytes", + "crossbeam-channel", + "futures-core", + "futures-sink", + "futures-task", + "futures-util", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "smallvec", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-codec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "futures-core", + "futures-sink", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "actix-http" +version = "3.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e2faa3e7418ed780cca54829d32782a4008a077230f67457caa063415e99c2" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "bitflags 2.13.1", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "foldhash 0.2.0", + "futures-core", + "http 0.2.12", + "httparse", + "httpdate", + "itoa", + "language-tags", + "mime", + "percent-encoding", + "pin-project-lite", + "smallvec", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "actix-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "actix-router" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f8c75c51892f18d9c46150c5ac7beb81c95f78c8b83a634d49f4ca32551fe7" +dependencies = [ + "bytestring", + "cfg-if", + "http 0.2.12", + "regex-lite", + "serde", + "tracing", +] + +[[package]] +name = "actix-rt" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92589714878ca59a7626ea19734f0e07a6a875197eec751bb5d3f99e64998c63" +dependencies = [ + "futures-core", + "tokio", +] + +[[package]] +name = "actix-server" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65064ea4a457eaf07f2fba30b4c695bf43b721790e9530d26cb6f9019ff7502" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "socket2 0.5.10", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e46f36bf0e5af44bdc4bdb36fbbd421aa98c79a9bce724e1edeb3894e10dc7f" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df09e2d9239703dd64056359c920c7f3fba6535ec61a0059e0f44e095ffe02b4" +dependencies = [ + "actix-codec", + "actix-http", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "bytes", + "bytestring", + "cfg-if", + "derive_more", + "encoding_rs", + "foldhash 0.2.0", + "futures-core", + "futures-util", + "impl-more", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex-lite", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2 0.6.5", + "time", + "tracing", + "url", +] + +[[package]] +name = "actix_derive" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ac1e58cded18cb28ddc17143c4dea5345b3ad575e14f32f66e4054a56eb271" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb6fa015ebe961e9908ca4c1854e7dc7aabd4417da77b6a0466e4dfb4c8f6f69" +dependencies = [ + "async-stream-impl", + "futures-core-preview", +] + +[[package]] +name = "async-stream-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f0d8c5b411e36dcfb04388bacfec54795726b1f0148adcb0f377a96d6747e0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "hyper 1.10.1", + "hyper-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core 0.5.6", + "bytes", + "futures-util", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper 1.0.2", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http 1.4.2", + "http-body-util", + "hyper 1.10.1", + "hyper-named-pipe", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with 3.21.0", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bytesize" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" +dependencies = [ + "serde", +] + +[[package]] +name = "bytestring" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86566c496f2f47d9b8147a4c8b02ffdb69c919fe0c2b2e7195d22cbba0e635c9" +dependencies = [ + "bytes", +] + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width 0.1.14", + "vec_map", +] + +[[package]] +name = "clap" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "config" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" +dependencies = [ + "async-trait", + "convert_case 0.6.0", + "json5", + "nom", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml", + "yaml-rust2", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "serde", + "uuid", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "pem-rfc7468 1.0.0", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case 0.10.0", + "proc-macro2", + "quote", + "rustc_version 0.4.1", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "device_query" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba6a7f3347c8bc8a4d194b9d8517e3a6b8fe1929ee91aaccf0be278b980033c" +dependencies = [ + "macos-accessibility-client", + "pkg-config", + "readkey", + "readmouse", + "windows 0.48.0", + "x11", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.1", + "objc2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dummy-cuebot" +version = "0.1.11" +dependencies = [ + "async-trait", + "config", + "futures", + "itertools 0.13.0", + "miette", + "once_cell", + "opencue-proto", + "prost", + "serde", + "serde_derive", + "serde_json", + "structopt", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-postgres", + "tonic", + "users", + "uuid", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +dependencies = [ + "serde", +] + +[[package]] +name = "elasticsearch" +version = "8.5.0-alpha.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d9bd57d914cc66ce878f098f63ed7b5d5b64c30644a5adb950b008f874a6c6" +dependencies = [ + "base64 0.11.0", + "bytes", + "dyn-clone", + "lazy_static", + "percent-encoding", + "reqwest 0.11.27", + "rustc_version 0.2.3", + "serde", + "serde_json", + "serde_with 1.14.0", + "url", + "void", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-core-preview" +version = "0.3.0-alpha.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35b6263fb1ef523c3056565fa67b1d16f0a8604ff12b11b08c25f28a734c60a" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.2", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http 1.4.2", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body 1.1.0", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.15", + "http 1.4.2", + "http-body 1.1.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.10.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http 1.4.2", + "hyper 1.10.1", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.10.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.32", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.10.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-util", + "http 1.4.2", + "http-body 1.1.0", + "hyper 1.10.1", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.5", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.10.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core 0.62.2", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "impl-more" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a84fd5aa25fae5c0f4a33d9cac2ca017fc622cbd089be2229993514990f870" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "ipnetwork" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" +dependencies = [ + "serde", +] + +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +dependencies = [ + "bitflags 2.13.1", + "libc", + "plain", + "redox_syscall 0.9.0", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "local-waker" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "macos-accessibility-client" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edf7710fbff50c24124331760978fb9086d6de6288dcdb38b25a97f8b1bdebbb" +dependencies = [ + "core-foundation 0.9.4", + "core-foundation-sys", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap 0.16.2", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "log", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "monitoring-indexer" +version = "0.1.11" +dependencies = [ + "anyhow", + "chrono", + "clap 4.6.2", + "config", + "elasticsearch", + "futures", + "rayon", + "rdkafka", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-subscriber", + "url", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "no-std-net" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.7", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-data" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-io-surface", +] + +[[package]] +name = "objc2-core-image" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-location" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca347214e24bc973fc025fd0d36ebb179ff30536ed1f80252706db19ee452009" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-text" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.1", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "objc2-ui-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87d638e33c06f577498cbcc50491496a3ed4246998a7fbba7ccb98b1e7eab22" +dependencies = [ + "bitflags 2.13.1", + "block2", + "objc2", + "objc2-cloud-kit", + "objc2-core-data", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image", + "objc2-core-location", + "objc2-core-text", + "objc2-foundation", + "objc2-quartz-core", + "objc2-user-notifications", +] + +[[package]] +name = "objc2-user-notifications" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9df9128cbbfef73cda168416ccf7f837b62737d748333bfe9ab71c245d76613e" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "opencue-proto" +version = "0.1.11" +dependencies = [ + "prost", + "prost-types", + "rand 0.8.7", + "rmp", + "rmp-serde", + "serde", + "serde_derive", + "tonic", + "tonic-build", + "uuid", + "whoami 1.6.1", +] + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "os_info" +version = "3.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf20a545b305cf1da722b236b5155c9bb35f1d5ceb28c048bd96ca842f41b5b" +dependencies = [ + "android_system_properties", + "log", + "nix 0.31.3", + "objc2", + "objc2-foundation", + "objc2-ui-kit", + "serde", + "windows-sys 0.61.2", +] + +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pest_meta" +version = "2.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" +dependencies = [ + "pest", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap 2.14.0", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der 0.7.10", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.10", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "pnet" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "682396b533413cc2e009fbb48aadf93619a149d3e57defba19ff50ce0201bd0d" +dependencies = [ + "ipnetwork", + "pnet_base", + "pnet_datalink", + "pnet_packet", + "pnet_sys", + "pnet_transport", +] + +[[package]] +name = "pnet_base" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc190d4067df16af3aba49b3b74c469e611cad6314676eaf1157f31aa0fb2f7" +dependencies = [ + "no-std-net", +] + +[[package]] +name = "pnet_datalink" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79e70ec0be163102a332e1d2d5586d362ad76b01cec86f830241f2b6452a7b7" +dependencies = [ + "ipnetwork", + "libc", + "pnet_base", + "pnet_sys", + "winapi", +] + +[[package]] +name = "pnet_macros" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13325ac86ee1a80a480b0bc8e3d30c25d133616112bb16e86f712dcf8a71c863" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.119", +] + +[[package]] +name = "pnet_macros_support" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed67a952585d509dd0003049b1fc56b982ac665c8299b124b90ea2bdb3134ab" +dependencies = [ + "pnet_base", +] + +[[package]] +name = "pnet_packet" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c96ebadfab635fcc23036ba30a7d33a80c39e8461b8bd7dc7bb186acb96560f" +dependencies = [ + "glob", + "pnet_base", + "pnet_macros", + "pnet_macros_support", +] + +[[package]] +name = "pnet_sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d4643d3d4db6b08741050c2f3afa9a892c4244c085a72fcda93c9c2c9a00f4b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "pnet_transport" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f604d98bc2a6591cf719b58d3203fd882bdd6bf1db696c4ac97978e9f4776bf" +dependencies = [ + "libc", + "pnet_base", + "pnet_packet", + "pnet_sys", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac 0.13.0", + "md-5 0.11.0", + "memchr", + "rand 0.10.2", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.13+spec-1.1.0", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror 1.0.69", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.13.1", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rdkafka" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1beea247b9a7600a81d4cc33f659ce1a77e1988323d7d2809c7ed1c21f4c316d" +dependencies = [ + "futures-channel", + "futures-util", + "libc", + "log", + "rdkafka-sys", + "serde", + "serde_derive", + "serde_json", + "slab", + "tokio", +] + +[[package]] +name = "rdkafka-sys" +version = "4.10.0+2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77" +dependencies = [ + "cmake", + "libc", + "libz-sys", + "num_enum", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "readkey" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a36870cefdfcff57edbc0fa62165f42dfd4e5a0d8965117c1ea84c5700e4450" + +[[package]] +name = "readmouse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be105c72a1e6a5a1198acee3d5b506a15676b74a02ecd78060042a447f408d94" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "redox_syscall" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "async-compression", + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-tls 0.5.0", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.15", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "hyper 1.10.1", + "hyper-rustls", + "hyper-tls 0.6.0", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "sync_wrapper 1.0.2", + "tokio", + "tokio-native-tls", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags 2.13.1", + "serde", + "serde_derive", +] + +[[package]] +name = "rqd" +version = "0.1.11" +dependencies = [ + "async-stream", + "async-trait", + "bincode", + "bollard", + "bytesize", + "chrono", + "config", + "dashmap", + "device_query", + "futures", + "futures-core", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "humantime", + "humantime-serde", + "itertools 0.13.0", + "lazy_static", + "libc", + "log", + "miette", + "nix 0.29.0", + "opencue-proto", + "pin-project-lite", + "pnet", + "prost", + "rand 0.9.5", + "regex", + "serde", + "serde_derive", + "serde_json", + "sysinfo", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tonic", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-appender", + "tracing-rolling-file", + "tracing-subscriber", + "ureq", + "users", + "uuid", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver 1.0.28", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "saa" +version = "5.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f5acb362a0e75c2a963532fa7fabf13dff81626dc494df16488d30befcbea0" + +[[package]] +name = "scc" +version = "3.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f733aa28b85255811ad1358d559fe9a182e39327cf5470140ed9d444de86e6d5" +dependencies = [ + "saa", + "sdd", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scheduler" +version = "0.1.11" +dependencies = [ + "actix", + "async-stream", + "async-trait", + "axum 0.7.9", + "bytesize", + "chrono", + "config", + "futures", + "home", + "humantime", + "humantime-serde", + "indexmap 2.14.0", + "itertools 0.13.0", + "lazy_static", + "miette", + "moka", + "once_cell", + "opencue-proto", + "prometheus", + "proptest", + "prost", + "rand 0.8.7", + "regex", + "scc", + "sentry", + "serde", + "serde_derive", + "serde_json", + "serial_test", + "sqlx", + "structopt", + "thiserror 1.0.69", + "time", + "tokio", + "tokio-stream", + "tokio-test", + "tokio-util", + "tonic", + "tower-http 0.5.2", + "tracing", + "tracing-appender", + "tracing-rolling-file", + "tracing-subscriber", + "tracing-test", + "urlencoding", + "uuid", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "4.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1836bad8bdc9c6d665b63202da3d9c6d60ed1e597cae63620e21ebf89a3595a9" +dependencies = [ + "saa", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "sentry" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb25f439f97d26fea01d717fa626167ceffcd981addaa670001e70505b72acbb" +dependencies = [ + "cfg_aliases", + "httpdate", + "native-tls", + "reqwest 0.13.4", + "sentry-actix", + "sentry-backtrace", + "sentry-contexts", + "sentry-core", + "sentry-debug-images", + "sentry-panic", + "sentry-tracing", + "tokio", + "ureq", +] + +[[package]] +name = "sentry-actix" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9453d18fc9a45d841636004aad50288d80cc07c34a9e88cd4397cb66e6356f67" +dependencies = [ + "actix-http", + "actix-web", + "bytes", + "futures-util", + "sentry-core", +] + +[[package]] +name = "sentry-backtrace" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a8c2c1bd5c1f735e84f28b48e7d72efcaafc362b7541bc8253e60e8fcdffc6" +dependencies = [ + "backtrace", + "regex", + "sentry-core", +] + +[[package]] +name = "sentry-contexts" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b88a90baa654d7f0e1f4b667f6b434293d9f72c71bef16b197c76af5b7d5803" +dependencies = [ + "hostname", + "libc", + "os_info", + "rustc_version 0.4.1", + "sentry-core", + "uname", +] + +[[package]] +name = "sentry-core" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ac170a5bba8bec6e3339c90432569d89641fa7a3d3e4f44987d24f0762e6adf" +dependencies = [ + "rand 0.9.5", + "sentry-types", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "sentry-debug-images" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd9646a972b57896d4a92ed200cf76139f8e30b3cfd03b6662ae59926d26633c" +dependencies = [ + "findshlibs", + "sentry-core", +] + +[[package]] +name = "sentry-panic" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6127d3d304ba5ce0409401e85aae538e303a569f8dbb031bf64f9ba0f7174346" +dependencies = [ + "sentry-backtrace", + "sentry-core", +] + +[[package]] +name = "sentry-tracing" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27701acc51e68db5281802b709010395bfcbcb128b1d0a4e5873680d3b47ff0c" +dependencies = [ + "bitflags 2.13.1", + "sentry-backtrace", + "sentry-core", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "sentry-types" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56780cb5597d676bf22e6c11d1f062eb4def46390ea3bfb047bcbcf7dfd19bdb" +dependencies = [ + "debugid", + "hex", + "rand 0.9.5", + "serde", + "serde_json", + "thiserror 2.0.18", + "time", + "url", + "uuid", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64 0.22.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "serial_test" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "699f4197115b8a7e7ff19c9a315a4bd6fffec26cc4626ef45ecaea389e081c6d" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der 0.7.10", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink 0.10.0", + "indexmap 2.14.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.119", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.119", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.13.1", + "byteorder", + "bytes", + "chrono", + "crc", + "digest 0.10.7", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac 0.12.1", + "itoa", + "log", + "md-5 0.10.6", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.7", + "rsa", + "serde", + "sha1", + "sha2 0.10.9", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "whoami 1.6.1", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.13.1", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac 0.12.1", + "home", + "itoa", + "log", + "md-5 0.10.6", + "memchr", + "once_cell", + "rand 0.8.7", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "whoami 1.6.1", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.18", + "tracing", + "url", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap 2.34.0", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e396b6523b11ccb83120b115a0b7366de372751aa6edf19844dfb13a6af97e91" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows 0.57.0", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width 0.1.14", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "unicode-linebreak", + "unicode-width 0.2.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.5", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.2", + "socket2 0.6.5", + "tokio", + "tokio-util", + "whoami 2.1.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-test" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" +dependencies = [ + "futures-core", + "tokio", + "tokio-stream", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.14.0", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum 0.8.9", + "base64 0.22.1", + "bytes", + "h2 0.4.15", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "hyper 1.10.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.14.0", + "pin-project-lite", + "slab", + "sync_wrapper 1.0.2", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "http 1.4.2", + "http-body 1.1.0", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "futures-util", + "http 1.4.2", + "http-body 1.1.0", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-rolling-file" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd38b6b03a92e4e311682054368ef79e7cb208c2125dd96647ef6bd7833de6db" +dependencies = [ + "chrono", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "time", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "tracing-test" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a4c448db514d4f24c5ddb9f73f2ee71bfb24c526cf0c570ba142d1119e0051" +dependencies = [ + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "uname" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" +dependencies = [ + "libc", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "cookie_store", + "der 0.8.1", + "flate2", + "log", + "native-tls", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http 1.4.2", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "users" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" +dependencies = [ + "libc", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite 0.1.0", + "web-sys", +] + +[[package]] +name = "whoami" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite 1.0.2", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core 0.57.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement 0.57.0", + "windows-interface 0.57.0", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link", + "windows-result 0.4.1", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "x11" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502da5464ccd04011667b11c435cb992822c2c0dbde1770c988480d312a0db2e" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "yaml-rust2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink 0.8.4", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" From 05995aa4cb6e35d42d6612948bcd6d2bae0e03d8 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Fri, 3 Jul 2026 14:50:35 -0700 Subject: [PATCH 10/20] [scheduler/cuebot] Add slot-based scheduling DB columns (V46-V49) Slot-based scheduling adds two orthogonal axes: a per-host concurrency cap and a per subscription/folder/job max, parallel to cores/gpus. - layer.int_slots_required (0 = not slot-based) - host.int_concurrent_slots_limit (-1 = not slot-based) - proc.int_slots_reserved (single source of truth for slot accounting) - {subscription,folder_resource,job_resource}.int_max_slots (-1 unlimited, 0 reject-all, N cap) host_stat.int_running_slots is intentionally NOT added: per-host slot usage derives from SUM(proc.int_slots_reserved), so RQD need not report slots. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../migrations/V46__Add_layer_slots_required.sql | 4 ++++ .../V47__Add_host_concurrent_slots_limit.sql | 5 +++++ .../migrations/V48__Add_proc_slots_reserved.sql | 6 ++++++ .../ddl/postgres/migrations/V49__Add_slot_limits.sql | 12 ++++++++++++ 4 files changed, 27 insertions(+) create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_slots_required.sql create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_host_concurrent_slots_limit.sql create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V48__Add_proc_slots_reserved.sql create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V49__Add_slot_limits.sql diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_slots_required.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_slots_required.sql new file mode 100644 index 0000000000..9310fc6615 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_slots_required.sql @@ -0,0 +1,4 @@ +-- Mark a layer as slot-based by requiring at least this many concurrency slots per frame. +-- 0 means the layer is not slot-based and books by cores/memory as usual. +alter table layer + add int_slots_required INT DEFAULT 0 NOT NULL; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_host_concurrent_slots_limit.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_host_concurrent_slots_limit.sql new file mode 100644 index 0000000000..4f46634db5 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_host_concurrent_slots_limit.sql @@ -0,0 +1,5 @@ +-- Limit the max number of concurrent frames a host may run (slot-based host). +-- -1 means the host is not slot-based and books by cores/memory as usual. +-- When >= 0 the host only runs slot-based layers, capped at this many concurrent slots. +alter table host + add int_concurrent_slots_limit INT DEFAULT -1 NOT NULL; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V48__Add_proc_slots_reserved.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V48__Add_proc_slots_reserved.sql new file mode 100644 index 0000000000..a667f3192b --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V48__Add_proc_slots_reserved.sql @@ -0,0 +1,6 @@ +-- Number of concurrency slots reserved by a proc (booked frame) on a slot-based host. +-- 0 for regular (cores/memory) procs. This is the single source of truth for slot +-- accounting: per-host and per subscription/folder/job slot usage both derive from +-- SUM(proc.int_slots_reserved). +alter table proc + add int_slots_reserved INT DEFAULT 0 NOT NULL; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V49__Add_slot_limits.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V49__Add_slot_limits.sql new file mode 100644 index 0000000000..806dfe2676 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V49__Add_slot_limits.sql @@ -0,0 +1,12 @@ +-- Per-hierarchy hard limit on concurrent slots for slot-based layers, parallel to the +-- cores/gpus limits. Enforced by the scheduler accounting store at subscription, folder +-- and job level. -1 means unlimited; 0 means reject all slot work; N caps at N slots. +-- Regular (cores/memory) layers are unaffected by this limit. +alter table subscription + add int_max_slots INT DEFAULT -1 NOT NULL; + +alter table folder_resource + add int_max_slots INT DEFAULT -1 NOT NULL; + +alter table job_resource + add int_max_slots INT DEFAULT -1 NOT NULL; From 7045efc6f26cfa1fced636fbdfe235f278e3eeee Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Fri, 3 Jul 2026 15:03:13 -0700 Subject: [PATCH 11/20] [scheduler] Per-host slot cap for slot-based scheduling Add the per-host concurrency-slot axis to the scheduler. Slot hosts (host.concurrent_slots_limit set) and slot layers (layer.int_slots_required > 0) are strictly paired: a slot host runs only slot layers and vice-versa. - Host: concurrent_slots_limit + running_slots_count (seeded from SUM(proc.int_slots_reserved) per host, incremented in-memory on booking). - DispatchLayer/DispatchFrame/VirtualProc carry slots_required; slot layers are forced non-threadable. - placement gate (epvm_gate/saturation_gate): match_slots enforces strict pairing and the per-host cap, replacing the cores/mem floor for slot work. - host_cache index: slot hosts share a fixed sentinel key so they never slide out of the search range while free slots remain. - consume_host_virtual_resources: slot procs reserve 0 cores + 0 memory and only consume slots; proc_dao writes int_slots_reserved. 177 lib tests pass; clippy clean (no new warnings). Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/crates/scheduler/src/dao/frame_dao.rs | 6 +- rust/crates/scheduler/src/dao/host_dao.rs | 14 ++++- rust/crates/scheduler/src/dao/layer_dao.rs | 10 ++- rust/crates/scheduler/src/dao/proc_dao.rs | 6 +- rust/crates/scheduler/src/host_cache/cache.rs | 42 +++++++++++-- rust/crates/scheduler/src/host_cache/store.rs | 2 + rust/crates/scheduler/src/models/frame.rs | 3 + rust/crates/scheduler/src/models/host.rs | 17 +++++ rust/crates/scheduler/src/models/layer.rs | 5 ++ .../scheduler/src/models/virtual_proc.rs | 3 + .../src/pipeline/dispatcher/actor.rs | 42 +++++++++++++ rust/crates/scheduler/src/pipeline/matcher.rs | 1 + .../scheduler/src/pipeline/placement.rs | 62 ++++++++++++++++--- 13 files changed, 194 insertions(+), 19 deletions(-) diff --git a/rust/crates/scheduler/src/dao/frame_dao.rs b/rust/crates/scheduler/src/dao/frame_dao.rs index 79eb7a5aa4..de2094b3cd 100644 --- a/rust/crates/scheduler/src/dao/frame_dao.rs +++ b/rust/crates/scheduler/src/dao/frame_dao.rs @@ -77,6 +77,7 @@ pub struct DispatchFrameModel { pub int_version: i32, pub str_loki_url: Option, pub ts_updated: Option>, + pub int_slots_required: i64, // Env fields pub job_env: HashMap, @@ -132,7 +133,9 @@ impl From for DispatchFrame { layer_name: val.str_layer_name, job_name: val.str_job_name, min_cores: CoreSize::from_multiplied(val.int_min_cores), - threadable: val.b_threadable, + // Slot-based layers are forced non-threadable: a slot frame reserves + // exactly its declared slots, never a variable core count. + threadable: val.b_threadable && val.int_slots_required <= 0, min_gpus: val .int_gpus_min .try_into() @@ -152,6 +155,7 @@ impl From for DispatchFrame { version: val.int_version as u32, updated_at, env, + slots_required: val.int_slots_required.max(0) as u32, } } } diff --git a/rust/crates/scheduler/src/dao/host_dao.rs b/rust/crates/scheduler/src/dao/host_dao.rs index a9837f6a68..e016f2f637 100644 --- a/rust/crates/scheduler/src/dao/host_dao.rs +++ b/rust/crates/scheduler/src/dao/host_dao.rs @@ -111,6 +111,10 @@ pub struct HostModel { // Number of cores available at the subscription of the show this host has been queried on int_alloc_available_cores: i64, ts_ping: DateTime, + // `host.int_concurrent_slots_limit`: `-1` = regular host, `>= 0` = slot-based host. + int_concurrent_slots_limit: i64, + // Slots currently reserved on this host, from `SUM(proc.int_slots_reserved)`. + int_running_slots: i64, } impl From for Host { @@ -133,6 +137,10 @@ impl From for Host { alloc_id: parse_uuid(&val.pk_alloc), alloc_name: val.str_alloc_name, last_updated: val.ts_ping, + // `-1` marks a regular host; `>= 0` marks a slot-based host with that cap. + concurrent_slots_limit: (val.int_concurrent_slots_limit >= 0) + .then_some(val.int_concurrent_slots_limit as u32), + running_slots_count: val.int_running_slots.max(0) as u32, } } } @@ -161,7 +169,11 @@ SELECT DISTINCT s.int_burst - s.int_cores as int_alloc_available_cores, a.pk_alloc, a.str_name as str_alloc_name, - hs.ts_ping + hs.ts_ping, + h.int_concurrent_slots_limit, + COALESCE(( + SELECT SUM(p.int_slots_reserved) FROM proc p WHERE p.pk_host = h.pk_host + ), 0)::bigint as int_running_slots FROM host h INNER JOIN host_stat hs ON h.pk_host = hs.pk_host INNER JOIN alloc a ON h.pk_alloc = a.pk_alloc diff --git a/rust/crates/scheduler/src/dao/layer_dao.rs b/rust/crates/scheduler/src/dao/layer_dao.rs index 4e9989df61..7b0eb5bdc9 100644 --- a/rust/crates/scheduler/src/dao/layer_dao.rs +++ b/rust/crates/scheduler/src/dao/layer_dao.rs @@ -62,6 +62,8 @@ pub struct DispatchLayerModel { /// `job_resource.int_max_cores` (centicores). `-1` is the OpenCue /// "unlimited" sentinel — preserved through `from_multiplied_cap`. pub int_job_max_cores: i64, + /// `layer.int_slots_required`. `0` means not slot-based. + pub int_slots_required: i64, } /// Combined model for batched layer and frame queries. @@ -88,6 +90,7 @@ pub struct LayerWithFramesModel { pub int_gpu_mem_min: i64, pub str_tags: String, pub int_job_max_cores: i64, + pub int_slots_required: i64, pub job_env: Json>, pub layer_env: Json>, @@ -138,7 +141,8 @@ impl DispatchLayer { str_os: layer.str_os, cores_min: CoreSize::from_multiplied(layer.int_cores_min), mem_min: ByteSize::kb(layer.int_mem_min as u64), - threadable: layer.b_threadable, + // Slot-based layers are forced non-threadable. + threadable: layer.b_threadable && layer.int_slots_required <= 0, gpus_min: layer .int_gpus_min .try_into() @@ -154,6 +158,7 @@ impl DispatchLayer { // Preserves the `-1` unlimited sentinel; `compute_max_more` skips // the job cap dim when `<= 0`. job_max_cores: CoreSize::from_multiplied_cap(layer.int_job_max_cores).value(), + slots_required: layer.int_slots_required.max(0) as u32, } } } @@ -244,6 +249,7 @@ SELECT DISTINCT l.int_gpu_mem_min, l.str_tags, jr.int_max_cores::bigint AS int_job_max_cores, + l.int_slots_required::bigint AS int_slots_required, je.job_env, le.layer_env, l.int_dispatch_order, @@ -392,6 +398,7 @@ impl LayerDao { int_gpu_mem_min: model.int_gpu_mem_min, str_tags: model.str_tags.clone(), int_job_max_cores: model.int_job_max_cores, + int_slots_required: model.int_slots_required, }; layers.push((layer_model, vec![])); let slot = layers.len() - 1; @@ -430,6 +437,7 @@ impl LayerDao { int_version: model.int_version.unwrap_or(1), str_loki_url: model.str_loki_url, ts_updated: model.ts_updated, + int_slots_required: model.int_slots_required, job_env: model.job_env.0, layer_env: model.layer_env.0, }; diff --git a/rust/crates/scheduler/src/dao/proc_dao.rs b/rust/crates/scheduler/src/dao/proc_dao.rs index 914b163519..33af460d06 100644 --- a/rust/crates/scheduler/src/dao/proc_dao.rs +++ b/rust/crates/scheduler/src/dao/proc_dao.rs @@ -85,9 +85,10 @@ static INSERT_PROC: &str = r#" int_gpu_mem_reserved, int_gpu_mem_pre_reserved, int_gpu_mem_used, - b_local + b_local, + int_slots_reserved ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 ) ON CONFLICT (pk_frame) DO NOTHING "#; @@ -169,6 +170,7 @@ impl ProcDao { .bind((virtual_proc.gpu_memory_reserved.0 / KB) as i64) .bind(0) .bind(virtual_proc.is_local_dispatch) + .bind(virtual_proc.slots_required as i64) .execute(&mut **transaction) .await .map_err(|err| ProcDaoError::DbFailure { diff --git a/rust/crates/scheduler/src/host_cache/cache.rs b/rust/crates/scheduler/src/host_cache/cache.rs index bbf92af450..4516587a9e 100644 --- a/rust/crates/scheduler/src/host_cache/cache.rs +++ b/rust/crates/scheduler/src/host_cache/cache.rs @@ -54,6 +54,15 @@ use crate::{ type CoreKey = u32; type MemoryKey = u64; +/// Sentinel B-tree index key for slot-based hosts. Slot hosts are indexed +/// independently of their idle cores/memory (which are ignored for slot +/// scheduling) so a slot host can never slide out of the search range while it +/// still has free slots. Slot-layer checkouts query this exact key, so they only +/// scan slot hosts; regular checkouts may also visit this bucket, but the +/// placement gate rejects the pairing mismatch. +const SLOT_CORE_KEY: CoreKey = u32::MAX; +const SLOT_MEMORY_KEY: MemoryKey = u64::MAX; + /// A B-Tree of Hosts ordered by memory pub type MemoryBTree = BTreeMap>; @@ -270,8 +279,13 @@ impl HostCache { core_saturation: bool, memory_saturation: bool, ) -> Option { - let core_key = cores.value() as u32; - let memory_key = Self::gen_memory_key(memory); + // Slot layers query the fixed slot sentinel bucket; regular layers query + // by their cores/memory floor. + let (core_key, memory_key) = if profile.slots_required > 0 { + (SLOT_CORE_KEY, SLOT_MEMORY_KEY) + } else { + (cores.value() as u32, Self::gen_memory_key(memory)) + }; let failed_candidates: RefCell> = RefCell::new(Vec::new()); let host_validation = |host: &Host| { @@ -373,8 +387,13 @@ impl HostCache { ) -> Option { const EPVM_INNER_RETRIES: usize = 3; - let core_key = cores.value() as u32; - let memory_key = Self::gen_memory_key(memory); + // Slot layers query the fixed slot sentinel bucket; regular layers query + // by their cores/memory floor. + let (core_key, memory_key) = if profile.slots_required > 0 { + (SLOT_CORE_KEY, SLOT_MEMORY_KEY) + } else { + (cores.value() as u32, Self::gen_memory_key(memory)) + }; // Phase 3 CAS failures within this call. Any host_id added here is // skipped by subsequent re-scans so we don't burn retries scoring and @@ -477,8 +496,16 @@ impl HostCache { // Update the data_store with new version let last_host_version = HOST_STORE.insert(host, authoritative); - let core_key = last_host_version.idle_cores.value() as CoreKey; - let memory_key = Self::gen_memory_key(last_host_version.idle_memory); + // Slot hosts are indexed at a fixed sentinel key, decoupled from their + // idle cores/memory, so they stay findable while free slots remain. + let (core_key, memory_key) = if last_host_version.is_slot_host() { + (SLOT_CORE_KEY, SLOT_MEMORY_KEY) + } else { + ( + last_host_version.idle_cores.value() as CoreKey, + Self::gen_memory_key(last_host_version.idle_memory), + ) + }; // Insert at the new location, removing any entry left at the host's // previous bucket @@ -582,6 +609,8 @@ mod tests { alloc_id: Uuid::new_v4(), alloc_name: "test".to_string(), last_updated: Utc::now(), + concurrent_slots_limit: None, + running_slots_count: 0, } } @@ -591,6 +620,7 @@ mod tests { mem_min, gpus_min: 0, gpu_mem_min: ByteSize::gb(0), + slots_required: 0, os: None, threadable: true, job_max_cores: 0, diff --git a/rust/crates/scheduler/src/host_cache/store.rs b/rust/crates/scheduler/src/host_cache/store.rs index b25c723a84..ee128e8995 100644 --- a/rust/crates/scheduler/src/host_cache/store.rs +++ b/rust/crates/scheduler/src/host_cache/store.rs @@ -412,6 +412,8 @@ mod tests { alloc_id: Uuid::new_v4(), alloc_name: "test".to_string(), last_updated, + concurrent_slots_limit: None, + running_slots_count: 0, } } diff --git a/rust/crates/scheduler/src/models/frame.rs b/rust/crates/scheduler/src/models/frame.rs index e449353073..52f85d1186 100644 --- a/rust/crates/scheduler/src/models/frame.rs +++ b/rust/crates/scheduler/src/models/frame.rs @@ -60,6 +60,9 @@ pub struct DispatchFrame { pub version: u32, pub updated_at: SystemTime, pub env: HashMap, + /// Concurrency slots this frame requires (from `layer.int_slots_required`). + /// `0` for regular (cores/memory) frames. + pub slots_required: u32, } impl Display for DispatchFrame { diff --git a/rust/crates/scheduler/src/models/host.rs b/rust/crates/scheduler/src/models/host.rs index 38af887265..0d045f4cf4 100644 --- a/rust/crates/scheduler/src/models/host.rs +++ b/rust/crates/scheduler/src/models/host.rs @@ -36,6 +36,20 @@ pub struct Host { pub(crate) alloc_id: Uuid, pub(crate) alloc_name: String, pub(crate) last_updated: DateTime, + /// Max concurrent frames (slots) this host may run. `Some` marks the host as + /// slot-based: it only runs slot-based layers and ignores cores/memory for + /// scheduling. `None` is a regular cores/memory host. + pub(crate) concurrent_slots_limit: Option, + /// Slots currently reserved on this host. Seeded from `SUM(proc.int_slots_reserved)` + /// and incremented in-memory as frames are booked between refreshes. + pub(crate) running_slots_count: u32, +} + +impl Host { + /// True when the host is slot-based (has a concurrent slots limit configured). + pub(crate) fn is_slot_host(&self) -> bool { + self.concurrent_slots_limit.is_some() + } } impl Host { @@ -75,6 +89,7 @@ impl Host { alloc_available_cores: CoreSize, alloc_id: Uuid, alloc_name: String, + concurrent_slots_limit: Option, ) -> Self { Self { id, @@ -91,6 +106,8 @@ impl Host { alloc_id, alloc_name, last_updated: Local::now().with_timezone(&Utc), + concurrent_slots_limit, + running_slots_count: 0, } } } diff --git a/rust/crates/scheduler/src/models/layer.rs b/rust/crates/scheduler/src/models/layer.rs index 40b0e4edfa..a03756354c 100644 --- a/rust/crates/scheduler/src/models/layer.rs +++ b/rust/crates/scheduler/src/models/layer.rs @@ -42,6 +42,11 @@ pub struct DispatchLayer { /// when not positive. Slow-moving admin field; refreshed each time the /// matcher re-queries the job's layers (design Branch 2b). pub job_max_cores: i32, + /// Concurrency slots each frame of this layer requires (`layer.int_slots_required`). + /// `0` means the layer is not slot-based and books by cores/memory. `> 0` marks the + /// layer slot-based: it only runs on slot-based hosts and counts against the + /// subscription/folder/job slot limits. + pub slots_required: u32, } impl fmt::Display for DispatchLayer { diff --git a/rust/crates/scheduler/src/models/virtual_proc.rs b/rust/crates/scheduler/src/models/virtual_proc.rs index 2d809a5a8c..ed026a8fe8 100644 --- a/rust/crates/scheduler/src/models/virtual_proc.rs +++ b/rust/crates/scheduler/src/models/virtual_proc.rs @@ -37,6 +37,9 @@ pub struct VirtualProc { pub os: String, pub is_local_dispatch: bool, pub frame: DispatchFrame, + /// Concurrency slots reserved by this proc (written to `proc.int_slots_reserved`). + /// `0` for regular (cores/memory) procs. + pub slots_required: u32, } impl Display for VirtualProc { diff --git a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs index d3b792cc52..fe00ce0855 100644 --- a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs +++ b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs @@ -1036,6 +1036,38 @@ impl RqdDispatcherService { ) -> Result<(VirtualProc, Host), VirtualProcError> { let mut host = host.clone(); + // Slot-based hosts ignore cores/memory entirely: a slot frame reserves 0 + // cores and 0 memory and only consumes the host's concurrency slots. The + // per-host slot cap was already enforced by the placement gate at + // check-out; here we just record the reservation. + if host.is_slot_host() { + host.running_slots_count += frame.slots_required; + host.last_updated = Utc::now(); + return Ok(( + VirtualProc { + proc_id: Uuid::new_v4(), + host_id: host.id, + show_id: frame.show_id, + folder_id, + dept_id, + layer_id: frame.layer_id, + job_id: frame.job_id, + frame_id: frame.id, + alloc_id: host.alloc_id, + cores_reserved: CoreSize(0).into(), + memory_reserved: ByteSize(0), + gpus_reserved: 0, + gpu_memory_reserved: ByteSize(0), + os: host.str_os.clone().unwrap_or_default(), + is_local_dispatch: false, + frame: frame.clone(), + host_name: host.name.clone(), + slots_required: frame.slots_required, + }, + host, + )); + } + let cores_reserved = Self::calculate_core_reservation( &host, frame, @@ -1108,6 +1140,7 @@ impl RqdDispatcherService { is_local_dispatch: false, frame: frame.clone(), host_name: host.name.clone(), + slots_required: 0, }, host, )) @@ -1400,6 +1433,7 @@ mod tests { CoreSize(4), Uuid::new_v4(), "test-alloc".to_string(), + None, ) } @@ -1435,6 +1469,7 @@ mod tests { version: 1, updated_at: SystemTime::now(), env: HashMap::new(), + slots_required: 0, } } @@ -1643,6 +1678,7 @@ mod tests { CoreSize(4), Uuid::new_v4(), "test-alloc".to_string(), + None, ); let mut frame = create_test_dispatch_frame(); @@ -1679,6 +1715,7 @@ mod tests { CoreSize(4), Uuid::new_v4(), "test-alloc".to_string(), + None, ); let mut frame = create_test_dispatch_frame(); @@ -1715,6 +1752,7 @@ mod tests { CoreSize(8), Uuid::new_v4(), "test-alloc".to_string(), + None, ); let mut frame = create_test_dispatch_frame(); @@ -1750,6 +1788,7 @@ mod tests { CoreSize(8), Uuid::new_v4(), "test-alloc".to_string(), + None, ); let mut frame = create_test_dispatch_frame(); @@ -1941,6 +1980,7 @@ mod tests { is_local_dispatch: false, frame, host_name: "somehost".to_string(), + slots_required: 0, }; let result = RqdDispatcherService::prepare_rqd_run_frame(&virtual_proc); @@ -2019,6 +2059,7 @@ mod tests { is_local_dispatch: false, frame, host_name: "somehost".to_string(), + slots_required: 0, }; let result = RqdDispatcherService::prepare_rqd_run_frame(&virtual_proc); @@ -2057,6 +2098,7 @@ mod tests { is_local_dispatch: false, frame, host_name: "somehost".to_string(), + slots_required: 0, }; let result = RqdDispatcherService::prepare_rqd_run_frame(&virtual_proc); diff --git a/rust/crates/scheduler/src/pipeline/matcher.rs b/rust/crates/scheduler/src/pipeline/matcher.rs index c0f867181f..adb5863330 100644 --- a/rust/crates/scheduler/src/pipeline/matcher.rs +++ b/rust/crates/scheduler/src/pipeline/matcher.rs @@ -395,6 +395,7 @@ impl MatchingService { mem_min: layer.mem_min, gpus_min: layer.gpus_min, gpu_mem_min: layer.gpu_mem_min, + slots_required: layer.slots_required, os: layer.str_os.clone(), threadable: layer.threadable, job_max_cores: layer.job_max_cores, diff --git a/rust/crates/scheduler/src/pipeline/placement.rs b/rust/crates/scheduler/src/pipeline/placement.rs index 44fcfb7588..ab7a10a5b0 100644 --- a/rust/crates/scheduler/src/pipeline/placement.rs +++ b/rust/crates/scheduler/src/pipeline/placement.rs @@ -41,6 +41,9 @@ pub struct LayerProfile { pub mem_min: ByteSize, pub gpus_min: i32, pub gpu_mem_min: ByteSize, + // Slot-based scheduling: `> 0` marks the layer slot-based. A slot layer only + // matches slot-based hosts, and cores/memory floors are ignored on those hosts. + pub slots_required: u32, // Compatibility pub os: Option, pub threadable: bool, @@ -222,14 +225,49 @@ pub fn placement_score(host: &Host, profile: &LayerProfile) -> f64 { w.cores * cores_waste + w.mem * mem_waste + gpu_term } +/// Result of pairing a host and layer on the slot-based axis. +/// +/// Slot hosts and slot layers are strictly paired: a slot host runs only slot +/// layers and vice-versa. On a slot host, cores/memory floors are irrelevant — +/// the only constraint is the per-host concurrent slots cap. +enum SlotMatch { + /// Neither host nor layer is slot-based: use the normal cores/memory gate. + Neither, + /// Both slot-based and the host has room for the layer's slots: accept. + Accept, + /// Pairing mismatch, or slot host at capacity: reject. + Reject, +} + +fn match_slots(host: &Host, profile: &LayerProfile) -> SlotMatch { + match (host.concurrent_slots_limit, profile.slots_required) { + // Slot host + slot layer: enforce the per-host concurrency cap. + (Some(limit), req) if req > 0 => { + if host.running_slots_count + req <= limit { + SlotMatch::Accept + } else { + SlotMatch::Reject + } + } + // Regular host + regular layer. + (None, 0) => SlotMatch::Neither, + // Slot host + regular layer, or regular host + slot layer. + _ => SlotMatch::Reject, + } +} + /// Saturation strategy gate: validate-only. Returns `Some(0.0)` when the host /// is a valid candidate; the constant score is irrelevant because the /// Saturation path is first-fit, not lowest-score. pub fn saturation_gate(host: &Host, profile: &LayerProfile) -> Option { - if validate_os_and_thread_mode(host, profile) && fits_floor(host, profile) { - Some(0.0) - } else { - None + match match_slots(host, profile) { + SlotMatch::Neither => { + (validate_os_and_thread_mode(host, profile) && fits_floor(host, profile)).then_some(0.0) + } + // Slot placement: only OS compatibility matters (cores/mem/thread-mode + // are irrelevant on slot hosts; the cap was checked in match_slots). + SlotMatch::Accept => host_matches_layer_os(host, profile).then_some(0.0), + SlotMatch::Reject => None, } } @@ -237,10 +275,14 @@ pub fn saturation_gate(host: &Host, profile: &LayerProfile) -> Option { /// host fits; `None` otherwise. The cache picks the lowest score among up to /// `max_candidates` scanned. pub fn epvm_gate(host: &Host, profile: &LayerProfile) -> Option { - if validate_os_and_thread_mode(host, profile) && fits_floor(host, profile) { - Some(placement_score(host, profile)) - } else { - None + match match_slots(host, profile) { + SlotMatch::Neither => (validate_os_and_thread_mode(host, profile) + && fits_floor(host, profile)) + .then(|| placement_score(host, profile)), + // Slot placements don't strand cores/memory, so E-PVM scoring is + // meaningless — every valid slot host ties at a constant score. + SlotMatch::Accept => host_matches_layer_os(host, profile).then_some(0.0), + SlotMatch::Reject => None, } } @@ -255,6 +297,7 @@ mod tests { mem_min: ByteSize::gb(1), gpus_min: 0, gpu_mem_min: ByteSize::gb(0), + slots_required: 0, os: os.map(str::to_string), threadable, job_max_cores: 0, @@ -288,6 +331,7 @@ mod tests { CoreSize::from_multiplied(100), Uuid::new_v4(), "test-alloc".to_string(), + None, ) } @@ -415,6 +459,7 @@ mod scoring_tests { CoreSize(idle_cores), Uuid::new_v4(), "test-alloc".to_string(), + None, ) } @@ -424,6 +469,7 @@ mod scoring_tests { mem_min: ByteSize::gb(mem_min_gb), gpus_min, gpu_mem_min: ByteSize::gb(gpu_mem_min_gb), + slots_required: 0, os: None, threadable: true, job_max_cores: 0, From 03425e417ee01d329fe6528f594e98b91ac38160 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Fri, 3 Jul 2026 15:08:18 -0700 Subject: [PATCH 12/20] [rqd/scheduler] Run slot-based frames unpinned Slot frames reserve 0 cores and 0 memory, so RQD must run them without CPU pinning. reserve_cores(0) is already a safe no-op returning an empty affinity list, so the only change RQD needs is to accept num_cores == 0 when slots_required > 0. The scheduler populates RunFrame.slots_required from the proc. RQD deliberately does NOT track or report slot usage: the scheduler owns slot accounting via proc.int_slots_reserved (design decision 7c), so none of the PR's original RQD-side slot-state tracking is carried over. - proto: RunFrame.slots_required (field 30) - rqd: relax num_cores validation for slot frames - scheduler: send slots_required in prepare_rqd_run_frame Co-Authored-By: Claude Opus 4.8 (1M context) --- proto/src/rqd.proto | 4 ++++ rust/crates/dummy-cuebot/src/rqd_client.rs | 1 + rust/crates/rqd/src/frame/manager.rs | 6 ++++-- rust/crates/rqd/src/frame/running_frame.rs | 1 + rust/crates/rqd/src/system/oom.rs | 1 + rust/crates/scheduler/src/pipeline/dispatcher/actor.rs | 1 + 6 files changed, 12 insertions(+), 2 deletions(-) diff --git a/proto/src/rqd.proto b/proto/src/rqd.proto index 621a13212c..5640dab55c 100644 --- a/proto/src/rqd.proto +++ b/proto/src/rqd.proto @@ -126,6 +126,10 @@ message RunFrame { int64 hard_memory_limit = 27; int32 pid = 28; string loki_url = 29; + // Concurrency slots required by this frame (<= 0 means not slot-based). + // Slot frames reserve 0 cores/memory and run unpinned; RQD does not track + // or report slot usage (the scheduler owns slot accounting). + int32 slots_required = 30; } message RunFrameSeq { diff --git a/rust/crates/dummy-cuebot/src/rqd_client.rs b/rust/crates/dummy-cuebot/src/rqd_client.rs index ab3ce01b40..c60585a011 100644 --- a/rust/crates/dummy-cuebot/src/rqd_client.rs +++ b/rust/crates/dummy-cuebot/src/rqd_client.rs @@ -70,6 +70,7 @@ impl DummyRqdClient { hard_memory_limit: 0, pid: 0, loki_url: "".to_string(), + slots_required: 0, #[allow(deprecated)] job_temp_dir: "deprecated".to_string(), diff --git a/rust/crates/rqd/src/frame/manager.rs b/rust/crates/rqd/src/frame/manager.rs index 040b38a99f..133a1f22f5 100644 --- a/rust/crates/rqd/src/frame/manager.rs +++ b/rust/crates/rqd/src/frame/manager.rs @@ -316,8 +316,10 @@ impl FrameManager { run_frame.uid_optional )))? } - // Invalid number of cores - if run_frame.num_cores <= 0 { + // Invalid number of cores. Slot-based frames legitimately reserve 0 + // cores (they run unpinned, capped by concurrency slots instead), so + // only reject non-positive cores for regular frames. + if run_frame.num_cores <= 0 && run_frame.slots_required <= 0 { Err(FrameManagerError::InvalidArgument( "Not launching, num_cores must be positive".to_string(), ))? diff --git a/rust/crates/rqd/src/frame/running_frame.rs b/rust/crates/rqd/src/frame/running_frame.rs index 48f1756e61..2c611d980c 100644 --- a/rust/crates/rqd/src/frame/running_frame.rs +++ b/rust/crates/rqd/src/frame/running_frame.rs @@ -1557,6 +1557,7 @@ mod tests { hard_memory_limit: 0, pid: 0, loki_url: "".to_string(), + slots_required: 0, #[allow(deprecated)] job_temp_dir: "".to_string(), diff --git a/rust/crates/rqd/src/system/oom.rs b/rust/crates/rqd/src/system/oom.rs index 2ca82f69ed..48365384ed 100644 --- a/rust/crates/rqd/src/system/oom.rs +++ b/rust/crates/rqd/src/system/oom.rs @@ -231,6 +231,7 @@ mod tests { hard_memory_limit: 0, pid: 0, loki_url: "".to_string(), + slots_required: 0, #[allow(deprecated)] job_temp_dir: "".to_string(), #[allow(deprecated)] diff --git a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs index fe00ce0855..57bbe3f4fd 100644 --- a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs +++ b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs @@ -1352,6 +1352,7 @@ impl RqdDispatcherService { attributes: HashMap::new(), children: None, pid: 0, // Will be set by RQD + slots_required: proc.slots_required as i32, // Deprecated fields #[allow(deprecated)] From fbcebed90c827f97ddef229759ba12099eb139ca Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Fri, 3 Jul 2026 15:22:54 -0700 Subject: [PATCH 13/20] [scheduler] Slot accounting axis in the in-memory Store Add a slots limit axis to the accounting Store, parallel to and fully independent of cores/gpus, enforced at subscription/folder/job level. A slot frame books slots with 0 cores/gpus and vice-versa. - BookingDelta.slot_delta; Counter slot double-buffer (inflight/settled) mirrors cores/gpus through book/confirm/rollback/apply_release/overwrite/seed. - Slot caps per vertex with a distinct rule (over_slot_cap): -1 unlimited, 0/missing = reject-all (fail-closed before the seed), N caps at N. 0 is a valid admin value, so it cannot read as "unset/unlimited" like the cores path. - dao: SUM(int_slots_reserved) in the booked snapshot; int_max_slots in the three limit queries. recompute/limit_reseed carry slots (whole counts, no centicore conversion). listener: slots in acct_release, slot variants in acct_limit_change. - dispatcher sets slot_delta = slots_required on the booking delta. 182 lib tests pass (5 new slot-axis tests); clippy clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scheduler/src/accounting/booking_delta.rs | 4 + rust/crates/scheduler/src/accounting/dao.rs | 26 +- .../scheduler/src/accounting/limit_reseed.rs | 45 ++- .../scheduler/src/accounting/listener.rs | 42 ++- .../scheduler/src/accounting/recompute.rs | 39 +- rust/crates/scheduler/src/accounting/store.rs | 344 ++++++++++++++---- .../src/pipeline/dispatcher/actor.rs | 3 + 7 files changed, 402 insertions(+), 101 deletions(-) diff --git a/rust/crates/scheduler/src/accounting/booking_delta.rs b/rust/crates/scheduler/src/accounting/booking_delta.rs index 4782593693..d5d7a270ea 100644 --- a/rust/crates/scheduler/src/accounting/booking_delta.rs +++ b/rust/crates/scheduler/src/accounting/booking_delta.rs @@ -27,4 +27,8 @@ pub struct BookingDelta { pub job_id: Uuid, pub core_delta: i64, pub gpu_delta: i32, + /// Signed slot delta for slot-based frames (positive on booking, negative on + /// release). `0` for regular (cores/gpus) frames — the slot axis is fully + /// independent of the core/gpu axes. + pub slot_delta: i64, } diff --git a/rust/crates/scheduler/src/accounting/dao.rs b/rust/crates/scheduler/src/accounting/dao.rs index 413930a11e..79d2b100e8 100644 --- a/rust/crates/scheduler/src/accounting/dao.rs +++ b/rust/crates/scheduler/src/accounting/dao.rs @@ -38,6 +38,7 @@ pub struct BookedSnapshotRow { pub job_id: Uuid, pub cores: i64, pub gpus: i64, + pub slots: i64, } /// The full universe of enumerable accounting keys for scheduler-managed shows, used by @@ -57,6 +58,7 @@ pub struct SubscriptionLimitsRow { pub show_id: Uuid, pub alloc_id: Uuid, pub burst: i64, + pub max_slots: i64, } #[derive(Debug, Clone)] @@ -64,6 +66,7 @@ pub struct FolderLimitsRow { pub folder_id: Uuid, pub max_cores: i64, pub max_gpus: i64, + pub max_slots: i64, } #[derive(Debug, Clone)] @@ -71,6 +74,7 @@ pub struct JobLimitsRow { pub job_id: Uuid, pub max_cores: i64, pub max_gpus: i64, + pub max_slots: i64, } static QUERY_MANAGED_SHOW_IDS: &str = r#" @@ -88,7 +92,8 @@ static QUERY_MANAGED_SHOW_IDS: &str = r#" static QUERY_BOOKED: &str = r#" SELECT j.pk_show, h.pk_alloc, j.pk_folder, p.pk_job, COALESCE(SUM(p.int_cores_reserved), 0)::bigint as cores, - COALESCE(SUM(p.int_gpus_reserved), 0)::bigint as gpus + COALESCE(SUM(p.int_gpus_reserved), 0)::bigint as gpus, + COALESCE(SUM(p.int_slots_reserved), 0)::bigint as slots FROM proc p JOIN host h ON h.pk_host = p.pk_host JOIN job j ON j.pk_job = p.pk_job AND j.str_state <> 'FINISHED' @@ -102,7 +107,8 @@ static QUERY_BOOKED: &str = r#" static QUERY_BOOKED_FOR_SHOW: &str = r#" SELECT h.pk_alloc, j.pk_folder, p.pk_job, COALESCE(SUM(p.int_cores_reserved), 0)::bigint as cores, - COALESCE(SUM(p.int_gpus_reserved), 0)::bigint as gpus + COALESCE(SUM(p.int_gpus_reserved), 0)::bigint as gpus, + COALESCE(SUM(p.int_slots_reserved), 0)::bigint as slots FROM proc p JOIN host h ON h.pk_host = p.pk_host JOIN job j ON j.pk_job = p.pk_job AND j.str_state <> 'FINISHED' @@ -111,20 +117,20 @@ static QUERY_BOOKED_FOR_SHOW: &str = r#" "#; static QUERY_SUBSCRIPTION_LIMITS: &str = r#" - SELECT s.pk_show, s.pk_alloc, s.int_burst + SELECT s.pk_show, s.pk_alloc, s.int_burst, s.int_max_slots FROM subscription s JOIN show sh ON sh.pk_show = s.pk_show AND sh.b_scheduler_managed = true "#; static QUERY_FOLDER_LIMITS: &str = r#" - SELECT fr.pk_folder, fr.int_max_cores, fr.int_max_gpus + SELECT fr.pk_folder, fr.int_max_cores, fr.int_max_gpus, fr.int_max_slots FROM folder_resource fr JOIN folder f ON f.pk_folder = fr.pk_folder JOIN show s ON s.pk_show = f.pk_show AND s.b_scheduler_managed = true "#; static QUERY_JOB_LIMITS: &str = r#" - SELECT jr.pk_job, jr.int_max_cores, jr.int_max_gpus + SELECT jr.pk_job, jr.int_max_cores, jr.int_max_gpus, jr.int_max_slots FROM job_resource jr JOIN job j ON j.pk_job = jr.pk_job AND j.str_state <> 'FINISHED' JOIN show s ON s.pk_show = j.pk_show AND s.b_scheduler_managed = true @@ -160,6 +166,7 @@ impl AccountingDao { pk_job: String, cores: i64, gpus: i64, + slots: i64, } let rows: Vec = sqlx::query_as(QUERY_BOOKED) .fetch_all(self.connection_pool.as_ref()) @@ -176,6 +183,7 @@ impl AccountingDao { job_id: parse_uuid(&r.pk_job), cores: r.cores, gpus: r.gpus, + slots: r.slots, }) .collect()) } @@ -193,6 +201,7 @@ impl AccountingDao { pk_job: String, cores: i64, gpus: i64, + slots: i64, } let rows: Vec = sqlx::query_as(QUERY_BOOKED_FOR_SHOW) .bind(show_id.to_string()) @@ -209,6 +218,7 @@ impl AccountingDao { job_id: parse_uuid(&r.pk_job), cores: r.cores, gpus: r.gpus, + slots: r.slots, }) .collect()) } @@ -237,6 +247,7 @@ impl AccountingDao { pk_show: String, pk_alloc: String, int_burst: i64, + int_max_slots: i32, } let rows: Vec = sqlx::query_as(QUERY_SUBSCRIPTION_LIMITS) .fetch_all(self.connection_pool.as_ref()) @@ -249,6 +260,7 @@ impl AccountingDao { show_id: parse_uuid(&r.pk_show), alloc_id: parse_uuid(&r.pk_alloc), burst: r.int_burst, + max_slots: i64::from(r.int_max_slots), }) .collect()) } @@ -260,6 +272,7 @@ impl AccountingDao { pk_folder: String, int_max_cores: i32, int_max_gpus: i32, + int_max_slots: i32, } let rows: Vec = sqlx::query_as(QUERY_FOLDER_LIMITS) .fetch_all(self.connection_pool.as_ref()) @@ -272,6 +285,7 @@ impl AccountingDao { folder_id: parse_uuid(&r.pk_folder), max_cores: i64::from(r.int_max_cores), max_gpus: i64::from(r.int_max_gpus), + max_slots: i64::from(r.int_max_slots), }) .collect()) } @@ -283,6 +297,7 @@ impl AccountingDao { pk_job: String, int_max_cores: i32, int_max_gpus: i32, + int_max_slots: i32, } let rows: Vec = sqlx::query_as(QUERY_JOB_LIMITS) .fetch_all(self.connection_pool.as_ref()) @@ -295,6 +310,7 @@ impl AccountingDao { job_id: parse_uuid(&r.pk_job), max_cores: i64::from(r.int_max_cores), max_gpus: i64::from(r.int_max_gpus), + max_slots: i64::from(r.int_max_slots), }) .collect()) } diff --git a/rust/crates/scheduler/src/accounting/limit_reseed.rs b/rust/crates/scheduler/src/accounting/limit_reseed.rs index 0d83d09355..ec538a4c02 100644 --- a/rust/crates/scheduler/src/accounting/limit_reseed.rs +++ b/rust/crates/scheduler/src/accounting/limit_reseed.rs @@ -88,20 +88,36 @@ pub async fn reseed_once(service: &AccountingService) -> Result<()> { reseed_limits(service.store(), service.dao()).await } -/// `(show, alloc, burst_cores)`. Burst is PG centicores → cores; never the `-1` sentinel. -fn sub_cap(r: &SubscriptionLimitsRow) -> (Uuid, Uuid, i64) { - (r.show_id, r.alloc_id, centicores_to_cores(r.burst)) +/// `(show, alloc, burst_cores, max_slots)`. Burst is PG centicores → cores; never the `-1` +/// sentinel. Slots are whole counts (no centicore conversion); `-1` unlimited passes through. +fn sub_cap(r: &SubscriptionLimitsRow) -> (Uuid, Uuid, i64, i64) { + ( + r.show_id, + r.alloc_id, + centicores_to_cores(r.burst), + r.max_slots, + ) } -/// `(folder, max_cores, max_gpus)`. Cores preserve the `-1` unlimited sentinel; GPUs pass -/// through unconverted (their `-1` sentinel survives a verbatim copy). -fn folder_cap(r: &FolderLimitsRow) -> (Uuid, i64, i64) { - (r.folder_id, centicores_to_cores_cap(r.max_cores), r.max_gpus) +/// `(folder, max_cores, max_gpus, max_slots)`. Cores preserve the `-1` unlimited sentinel; +/// GPUs and slots pass through unconverted (whole counts; their `-1` sentinel survives). +fn folder_cap(r: &FolderLimitsRow) -> (Uuid, i64, i64, i64) { + ( + r.folder_id, + centicores_to_cores_cap(r.max_cores), + r.max_gpus, + r.max_slots, + ) } -/// `(job, max_cores, max_gpus)`. Same conventions as [`folder_cap`]. -fn job_cap(r: &JobLimitsRow) -> (Uuid, i64, i64) { - (r.job_id, centicores_to_cores_cap(r.max_cores), r.max_gpus) +/// `(job, max_cores, max_gpus, max_slots)`. Same conventions as [`folder_cap`]. +fn job_cap(r: &JobLimitsRow) -> (Uuid, i64, i64, i64) { + ( + r.job_id, + centicores_to_cores_cap(r.max_cores), + r.max_gpus, + r.max_slots, + ) } #[cfg(test)] @@ -114,8 +130,9 @@ mod tests { show_id: Uuid::nil(), alloc_id: Uuid::nil(), burst: 1000, + max_slots: -1, }; - assert_eq!(sub_cap(&r), (Uuid::nil(), Uuid::nil(), 10)); + assert_eq!(sub_cap(&r), (Uuid::nil(), Uuid::nil(), 10, -1)); } #[test] @@ -124,8 +141,9 @@ mod tests { folder_id: Uuid::nil(), max_cores: -1, max_gpus: -1, + max_slots: 5, }; - assert_eq!(folder_cap(&r), (Uuid::nil(), -1, -1)); + assert_eq!(folder_cap(&r), (Uuid::nil(), -1, -1, 5)); } #[test] @@ -134,7 +152,8 @@ mod tests { job_id: Uuid::nil(), max_cores: 2000, max_gpus: 4, + max_slots: 0, }; - assert_eq!(job_cap(&r), (Uuid::nil(), 20, 4)); + assert_eq!(job_cap(&r), (Uuid::nil(), 20, 4, 0)); } } diff --git a/rust/crates/scheduler/src/accounting/listener.rs b/rust/crates/scheduler/src/accounting/listener.rs index c709bead0b..56ca7f6ea9 100644 --- a/rust/crates/scheduler/src/accounting/listener.rs +++ b/rust/crates/scheduler/src/accounting/listener.rs @@ -55,6 +55,9 @@ struct ReleasePayload { job: Uuid, cores: i64, gpus: i32, + /// Slots released (defaults to 0 for regular procs / older Cuebot payloads). + #[serde(default)] + slots: i64, } /// Cap change payload on `acct_limit_change`. Values are in cores (`-1` = unlimited), @@ -66,7 +69,10 @@ enum LimitChangePayload { Sub { show: Uuid, alloc: Uuid, - burst: i64, + #[serde(default)] + burst: Option, + #[serde(default)] + max_slots: Option, }, #[serde(rename = "folder")] Folder { @@ -75,6 +81,8 @@ enum LimitChangePayload { max_cores: Option, #[serde(default)] max_gpus: Option, + #[serde(default)] + max_slots: Option, }, #[serde(rename = "job")] Job { @@ -83,6 +91,8 @@ enum LimitChangePayload { max_cores: Option, #[serde(default)] max_gpus: Option, + #[serde(default)] + max_slots: Option, }, } @@ -129,6 +139,7 @@ fn handle_release(store: &Store, payload: &str) { job_id: p.job, core_delta: p.cores, gpu_delta: p.gpus, + slot_delta: p.slots, }), Err(err) => warn!("Dropping malformed acct_release payload ({err}): {payload}"), } @@ -155,15 +166,25 @@ fn limit_changes(p: LimitChangePayload) -> Vec { show, alloc, burst, - } => vec![LimitChange::SubBurst { - show_id: show, - alloc_id: alloc, - burst, - }], + max_slots, + } => burst + .map(|b| LimitChange::SubBurst { + show_id: show, + alloc_id: alloc, + burst: b, + }) + .into_iter() + .chain(max_slots.map(|s| LimitChange::SubMaxSlots { + show_id: show, + alloc_id: alloc, + max_slots: s, + })) + .collect(), LimitChangePayload::Folder { id, max_cores, max_gpus, + max_slots, } => max_cores .map(|c| LimitChange::FolderMaxCores { folder_id: id, @@ -174,11 +195,16 @@ fn limit_changes(p: LimitChangePayload) -> Vec { folder_id: id, max_gpus: g, })) + .chain(max_slots.map(|s| LimitChange::FolderMaxSlots { + folder_id: id, + max_slots: s, + })) .collect(), LimitChangePayload::Job { id, max_cores, max_gpus, + max_slots, } => max_cores .map(|c| LimitChange::JobMaxCores { job_id: id, @@ -189,6 +215,10 @@ fn limit_changes(p: LimitChangePayload) -> Vec { job_id: id, max_gpus: g, })) + .chain(max_slots.map(|s| LimitChange::JobMaxSlots { + job_id: id, + max_slots: s, + })) .collect(), } } diff --git a/rust/crates/scheduler/src/accounting/recompute.rs b/rust/crates/scheduler/src/accounting/recompute.rs index ea2673fa38..46494dc918 100644 --- a/rust/crates/scheduler/src/accounting/recompute.rs +++ b/rust/crates/scheduler/src/accounting/recompute.rs @@ -149,11 +149,11 @@ pub(crate) fn snapshot_to_counters( rows: &[BookedSnapshotRow], baseline: &BaselineKeys, ) -> CounterSnapshot { - let mut sub: HashMap<(Uuid, Uuid), (i64, i64)> = HashMap::new(); - let mut folder: HashMap = HashMap::new(); - let mut job: HashMap = HashMap::new(); + let mut sub: HashMap<(Uuid, Uuid), (i64, i64, i64)> = HashMap::new(); + let mut folder: HashMap = HashMap::new(); + let mut job: HashMap = HashMap::new(); - // Zero-baseline first: every enumerable key gets a (0, 0) entry so drained keys still + // Zero-baseline first: every enumerable key gets a (0, 0, 0) entry so drained keys still // reset. Centicore sums are folded on top below, then converted once at the end. for &k in &baseline.subs { sub.entry(k).or_default(); @@ -165,17 +165,21 @@ pub(crate) fn snapshot_to_counters( job.entry(k).or_default(); } - // Accumulate centicores; convert to cores after summing so truncation happens once. + // Accumulate centicores (cores) / whole counts (gpus, slots); convert cores to whole + // cores after summing so truncation happens once. Slots are already whole counts. for r in rows { let s = sub.entry((r.show_id, r.alloc_id)).or_default(); s.0 += r.cores; s.1 += r.gpus; + s.2 += r.slots; let f = folder.entry(r.folder_id).or_default(); f.0 += r.cores; f.1 += r.gpus; + f.2 += r.slots; let j = job.entry(r.job_id).or_default(); j.0 += r.cores; j.1 += r.gpus; + j.2 += r.slots; } CounterSnapshot { @@ -185,10 +189,13 @@ pub(crate) fn snapshot_to_counters( } } -/// Convert each key's accumulated centicore total to cores (GPUs pass through). -fn to_cores(m: HashMap) -> HashMap { +/// Convert each key's accumulated centicore total to cores (GPUs and slots pass through as +/// whole counts). +fn to_cores( + m: HashMap, +) -> HashMap { m.into_iter() - .map(|(k, (cores, gpus))| (k, (centicores_to_cores(cores), gpus))) + .map(|(k, (cores, gpus, slots))| (k, (centicores_to_cores(cores), gpus, slots))) .collect() } @@ -205,6 +212,7 @@ mod tests { job_id: Uuid::nil(), cores: 4200, gpus: 3, + slots: 5, } } @@ -215,9 +223,9 @@ mod tests { #[test] fn single_row_converts_centicores_to_cores() { let snap = snapshot_to_counters(&[fixture_row()], &empty_baseline()); - assert_eq!(snap.sub[&(Uuid::nil(), Uuid::nil())], (42, 3)); - assert_eq!(snap.folder[&Uuid::nil()], (42, 3)); - assert_eq!(snap.job[&Uuid::nil()], (42, 3)); + assert_eq!(snap.sub[&(Uuid::nil(), Uuid::nil())], (42, 3, 5)); + assert_eq!(snap.folder[&Uuid::nil()], (42, 3, 5)); + assert_eq!(snap.job[&Uuid::nil()], (42, 3, 5)); } /// Two jobs in the same folder/sub: the coarse keys SUM across rows (not last-write). @@ -233,6 +241,7 @@ mod tests { job_id: Uuid::new_v4(), cores: 1000, // 10 cores gpus: 1, + slots: 2, }; let row_b = BookedSnapshotRow { cores: 2500, // 25 cores @@ -241,9 +250,9 @@ mod tests { ..row_a.clone() }; let snap = snapshot_to_counters(&[row_a, row_b], &empty_baseline()); - // 3500 centicores summed then /100 -> 35 cores; gpus 3. - assert_eq!(snap.sub[&(show, alloc)], (35, 3)); - assert_eq!(snap.folder[&folder], (35, 3)); + // 3500 centicores summed then /100 -> 35 cores; gpus 3; slots 2+2=4. + assert_eq!(snap.sub[&(show, alloc)], (35, 3, 4)); + assert_eq!(snap.folder[&folder], (35, 3, 4)); } /// A baseline key with no proc row (drained to zero) must reset to 0. @@ -255,6 +264,6 @@ mod tests { ..Default::default() }; let snap = snapshot_to_counters(&[], &baseline); - assert_eq!(snap.job[&job], (0, 0)); + assert_eq!(snap.job[&job], (0, 0, 0)); } } diff --git a/rust/crates/scheduler/src/accounting/store.rs b/rust/crates/scheduler/src/accounting/store.rs index 2bff53a385..48a4a1ffcf 100644 --- a/rust/crates/scheduler/src/accounting/store.rs +++ b/rust/crates/scheduler/src/accounting/store.rs @@ -77,6 +77,11 @@ struct Counter { /// already counted (`clear` bucket). See the module docs. settled_cores: [i64; 2], settled_gpus: [i64; 2], + /// Slot booking counters. Independent axis: slot frames book slots with 0 + /// cores/gpus and vice-versa. Mirrors the cores/gpus double-buffer exactly. + slots: i64, + inflight_slots: i64, + settled_slots: [i64; 2], } impl Counter { @@ -90,29 +95,38 @@ impl Counter { fn carried_gpus(&self, keep: usize) -> i64 { self.inflight_gpus + self.settled_gpus[keep] } + fn carried_slots(&self, keep: usize) -> i64 { + self.inflight_slots + self.settled_slots[keep] + } /// `book`: add to the live total and the in-flight bucket. - fn add_booking(&mut self, dc: i64, dg: i64) { + fn add_booking(&mut self, dc: i64, dg: i64, ds: i64) { self.cores += dc; self.gpus += dg; + self.slots += ds; self.inflight_cores += dc; self.inflight_gpus += dg; + self.inflight_slots += ds; } /// `confirm`: move from in-flight to the current epoch's settled bucket (live unchanged). - fn settle(&mut self, dc: i64, dg: i64, bucket: usize) { + fn settle(&mut self, dc: i64, dg: i64, ds: i64, bucket: usize) { self.inflight_cores -= dc; self.inflight_gpus -= dg; + self.inflight_slots -= ds; self.settled_cores[bucket] += dc; self.settled_gpus[bucket] += dg; + self.settled_slots[bucket] += ds; } /// `rollback`: undo a `book` (live total and in-flight). - fn remove_booking(&mut self, dc: i64, dg: i64) { + fn remove_booking(&mut self, dc: i64, dg: i64, ds: i64) { self.cores -= dc; self.gpus -= dg; + self.slots -= ds; self.inflight_cores -= dc; self.inflight_gpus -= dg; + self.inflight_slots -= ds; } } @@ -133,6 +147,12 @@ struct Inner { sub_burst: HashMap<(Uuid, Uuid), i64>, folder_caps: HashMap, job_caps: HashMap, + /// Slot caps per vertex (subscription/folder/job), a hard max independent of + /// cores/gpus. `-1` = unlimited, `0`/missing = reject-all (fail-closed before + /// the seed). See [`over_slot_cap`]. + sub_slot_caps: HashMap<(Uuid, Uuid), i64>, + folder_slot_caps: HashMap, + job_slot_caps: HashMap, /// Monotonic recompute epoch, bumped under the lock at the start of each pass *before* /// the snapshot read. Its parity picks the settled bucket, so `confirm` and the overwrite /// agree on which confirms the snapshot already saw. See the module docs. @@ -177,15 +197,29 @@ pub enum LimitChange { job_id: Uuid, max_gpus: i64, }, + SubMaxSlots { + show_id: Uuid, + alloc_id: Uuid, + max_slots: i64, + }, + FolderMaxSlots { + folder_id: Uuid, + max_slots: i64, + }, + JobMaxSlots { + job_id: Uuid, + max_slots: i64, + }, } /// Aggregated `SUM(proc)` totals for one recompute pass, already converted to cores and /// overlaid on the zero-baseline (every enumerable key present, drained keys carrying 0). +/// Per-vertex booked totals `(cores, gpus, slots)`. #[derive(Default, Debug)] pub struct CounterSnapshot { - pub sub: HashMap<(Uuid, Uuid), (i64, i64)>, - pub folder: HashMap, - pub job: HashMap, + pub sub: HashMap<(Uuid, Uuid), (i64, i64, i64)>, + pub folder: HashMap, + pub job: HashMap, } /// Process-wide in-memory accounting state. @@ -206,6 +240,14 @@ fn over_cap(cur: i64, delta: i64, cap: i64, enforce_zero: bool) -> bool { } } +/// Slot cap rule: `-1` (negative) is unlimited; `0` and any missing cap mean +/// "reject all" (fail-closed before the seed); `N >= 0` caps at N. Distinct from +/// [`over_cap`] because `0` is a *valid* admin value (reject all slot work here), +/// so it must not read as "unset/unlimited". +fn over_slot_cap(cur: i64, delta: i64, cap: i64) -> bool { + cap >= 0 && cur + delta > cap +} + impl Store { pub fn new() -> Self { Self::default() @@ -220,6 +262,7 @@ impl Store { pub fn book(&self, delta: &BookingDelta) -> BookOutcome { let core_delta = delta.core_delta; let gpu_delta = i64::from(delta.gpu_delta); + let slot_delta = delta.slot_delta; let mut inner = self.lock(); if core_delta > 0 { @@ -290,21 +333,69 @@ impl Store { } } + if slot_delta > 0 { + // Slot caps: `-1` unlimited, `0`/missing reject-all (fail-closed). + let cur_sub_slots = inner + .sub + .get(&(delta.show_id, delta.alloc_id)) + .map_or(0, |c| c.slots); + let sub_slot_max = inner + .sub_slot_caps + .get(&(delta.show_id, delta.alloc_id)) + .copied() + .unwrap_or(0); + if over_slot_cap(cur_sub_slots, slot_delta, sub_slot_max) { + return BookOutcome::LimitExceeded { + table: "subscription_slots", + current: cur_sub_slots, + limit: sub_slot_max, + }; + } + + let cur_folder_slots = inner.folder.get(&delta.folder_id).map_or(0, |c| c.slots); + let folder_slot_max = inner + .folder_slot_caps + .get(&delta.folder_id) + .copied() + .unwrap_or(0); + if over_slot_cap(cur_folder_slots, slot_delta, folder_slot_max) { + return BookOutcome::LimitExceeded { + table: "folder_slots", + current: cur_folder_slots, + limit: folder_slot_max, + }; + } + + let cur_job_slots = inner.job.get(&delta.job_id).map_or(0, |c| c.slots); + let job_slot_max = inner + .job_slot_caps + .get(&delta.job_id) + .copied() + .unwrap_or(0); + if over_slot_cap(cur_job_slots, slot_delta, job_slot_max) { + return BookOutcome::LimitExceeded { + table: "job_slots", + current: cur_job_slots, + limit: job_slot_max, + }; + } + } + inner .sub .entry((delta.show_id, delta.alloc_id)) .or_default() - .add_booking(core_delta, gpu_delta); + .add_booking(core_delta, gpu_delta, slot_delta); inner .folder .entry(delta.folder_id) .or_default() - .add_booking(core_delta, gpu_delta); + .add_booking(core_delta, gpu_delta, slot_delta); inner .job .entry(delta.job_id) .or_default() - .add_booking(core_delta, gpu_delta); + .add_booking(core_delta, gpu_delta, slot_delta); BookOutcome::Applied } @@ -314,45 +405,47 @@ impl Store { pub fn confirm(&self, delta: &BookingDelta) { let dc = delta.core_delta; let dg = i64::from(delta.gpu_delta); + let ds = delta.slot_delta; let mut inner = self.lock(); let bucket = (inner.epoch % 2) as usize; inner .sub .entry((delta.show_id, delta.alloc_id)) .or_default() - .settle(dc, dg, bucket); + .settle(dc, dg, ds, bucket); inner .folder .entry(delta.folder_id) .or_default() - .settle(dc, dg, bucket); + .settle(dc, dg, ds, bucket); inner .job .entry(delta.job_id) .or_default() - .settle(dc, dg, bucket); + .settle(dc, dg, ds, bucket); } /// Booking failed before launch: undo the live increment and the in-flight delta. pub fn rollback(&self, delta: &BookingDelta) { let dc = delta.core_delta; let dg = i64::from(delta.gpu_delta); + let ds = delta.slot_delta; let mut inner = self.lock(); inner .sub .entry((delta.show_id, delta.alloc_id)) .or_default() - .remove_booking(dc, dg); + .remove_booking(dc, dg, ds); inner .folder .entry(delta.folder_id) .or_default() - .remove_booking(dc, dg); + .remove_booking(dc, dg, ds); inner .job .entry(delta.job_id) .or_default() - .remove_booking(dc, dg); + .remove_booking(dc, dg, ds); } /// Apply a release delta (negative cores/gpus) from the Cuebot `acct_release` NOTIFY. @@ -360,18 +453,22 @@ impl Store { pub fn apply_release(&self, delta: &BookingDelta) { let dc = delta.core_delta; let dg = i64::from(delta.gpu_delta); + let ds = delta.slot_delta; let mut inner = self.lock(); if let Some(c) = inner.sub.get_mut(&(delta.show_id, delta.alloc_id)) { c.cores += dc; c.gpus += dg; + c.slots += ds; } if let Some(c) = inner.folder.get_mut(&delta.folder_id) { c.cores += dc; c.gpus += dg; + c.slots += ds; } if let Some(c) = inner.job.get_mut(&delta.job_id) { c.cores += dc; c.gpus += dg; + c.slots += ds; } } @@ -398,34 +495,40 @@ impl Store { let clear = (epoch % 2) as usize; let keep = 1 - clear; let mut inner = self.lock(); - for (&k, &(cores, gpus)) in &snapshot.sub { + for (&k, &(cores, gpus, slots)) in &snapshot.sub { let c = inner.sub.entry(k).or_default(); c.cores = cores + c.carried_cores(keep); c.gpus = gpus + c.carried_gpus(keep); + c.slots = slots + c.carried_slots(keep); } - for (&k, &(cores, gpus)) in &snapshot.folder { + for (&k, &(cores, gpus, slots)) in &snapshot.folder { let c = inner.folder.entry(k).or_default(); c.cores = cores + c.carried_cores(keep); c.gpus = gpus + c.carried_gpus(keep); + c.slots = slots + c.carried_slots(keep); } - for (&k, &(cores, gpus)) in &snapshot.job { + for (&k, &(cores, gpus, slots)) in &snapshot.job { let c = inner.job.entry(k).or_default(); c.cores = cores + c.carried_cores(keep); c.gpus = gpus + c.carried_gpus(keep); + c.slots = slots + c.carried_slots(keep); } // Clear the pre-snapshot settled bucket across all keys (confirms older than this // pass's snapshot read are provably reflected in the snapshot now). for c in inner.sub.values_mut() { c.settled_cores[clear] = 0; c.settled_gpus[clear] = 0; + c.settled_slots[clear] = 0; } for c in inner.folder.values_mut() { c.settled_cores[clear] = 0; c.settled_gpus[clear] = 0; + c.settled_slots[clear] = 0; } for c in inner.job.values_mut() { c.settled_cores[clear] = 0; c.settled_gpus[clear] = 0; + c.settled_slots[clear] = 0; } } @@ -441,20 +544,23 @@ impl Store { /// counters, so it cannot interfere with that driver's begin/overwrite sequencing. pub fn seed_show_booked(&self, snapshot: &CounterSnapshot) { let mut inner = self.lock(); - for (&k, &(cores, gpus)) in &snapshot.sub { + for (&k, &(cores, gpus, slots)) in &snapshot.sub { let c = inner.sub.entry(k).or_default(); c.cores = cores; c.gpus = gpus; + c.slots = slots; } - for (&k, &(cores, gpus)) in &snapshot.folder { + for (&k, &(cores, gpus, slots)) in &snapshot.folder { let c = inner.folder.entry(k).or_default(); c.cores = cores; c.gpus = gpus; + c.slots = slots; } - for (&k, &(cores, gpus)) in &snapshot.job { + for (&k, &(cores, gpus, slots)) in &snapshot.job { let c = inner.job.entry(k).or_default(); c.cores = cores; c.gpus = gpus; + c.slots = slots; } } @@ -462,15 +568,16 @@ impl Store { /// unlimited sentinels are preserved by the caller. pub fn set_caps( &self, - subs: impl IntoIterator, - folders: impl IntoIterator, - jobs: impl IntoIterator, + subs: impl IntoIterator, + folders: impl IntoIterator, + jobs: impl IntoIterator, ) { let mut inner = self.lock(); - for (show_id, alloc_id, burst) in subs { + for (show_id, alloc_id, burst, max_slots) in subs { inner.sub_burst.insert((show_id, alloc_id), burst); + inner.sub_slot_caps.insert((show_id, alloc_id), max_slots); } - for (folder_id, max_cores, max_gpus) in folders { + for (folder_id, max_cores, max_gpus, max_slots) in folders { inner.folder_caps.insert( folder_id, MaxCap { @@ -478,8 +585,9 @@ impl Store { max_gpus, }, ); + inner.folder_slot_caps.insert(folder_id, max_slots); } - for (job_id, max_cores, max_gpus) in jobs { + for (job_id, max_cores, max_gpus, max_slots) in jobs { inner.job_caps.insert( job_id, MaxCap { @@ -487,6 +595,7 @@ impl Store { max_gpus, }, ); + inner.job_slot_caps.insert(job_id, max_slots); } } @@ -537,6 +646,22 @@ impl Store { .or_insert(MaxCap::unlimited()) .max_gpus = max_gpus; } + LimitChange::SubMaxSlots { + show_id, + alloc_id, + max_slots, + } => { + inner.sub_slot_caps.insert((show_id, alloc_id), max_slots); + } + LimitChange::FolderMaxSlots { + folder_id, + max_slots, + } => { + inner.folder_slot_caps.insert(folder_id, max_slots); + } + LimitChange::JobMaxSlots { job_id, max_slots } => { + inner.job_slot_caps.insert(job_id, max_slots); + } } } @@ -545,6 +670,11 @@ impl Store { self.lock().job.get(&job_id).map_or(0, |c| c.cores) } + /// Live booked slots for a job (slot-axis observability). 0 if unseen. + pub fn job_slots_in_use(&self, job_id: Uuid) -> i64 { + self.lock().job.get(&job_id).map_or(0, |c| c.slots) + } + /// `(booked_cores, burst)` for a subscription (matcher over-burst pre-check). Both in /// cores; missing entries read as 0. pub fn sub_counters(&self, show_id: Uuid, alloc_id: Uuid) -> (i64, i64) { @@ -622,6 +752,20 @@ mod tests { job_id: job, core_delta: cores, gpu_delta: gpus, + slot_delta: 0, + } + } + + /// A pure slot booking delta (0 cores/gpus) for the slot-axis tests. + fn slot_delta(show: Uuid, alloc: Uuid, folder: Uuid, job: Uuid, slots: i64) -> BookingDelta { + BookingDelta { + show_id: show, + alloc_id: alloc, + folder_id: folder, + job_id: job, + core_delta: 0, + gpu_delta: 0, + slot_delta: slots, } } @@ -663,7 +807,7 @@ mod tests { fn book_enforces_job_hard_cap_atomically() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 1000)], [(folder, -1, -1)], [(job, 10, -1)]); + store.set_caps([(show, alloc, 1000, -1)], [(folder, -1, -1, -1)], [(job, 10, -1, -1)]); let d = delta(show, alloc, folder, job, 6, 0); assert!(applied(store.book(&d))); // 6 <= 10 // Second booking of 6 would reach 12 > 10 -> rejected. No partial state. @@ -683,9 +827,9 @@ mod tests { let (show, alloc, folder, job) = ids(); let store = Store::new(); store.set_caps( - [(show, alloc, 1_000_000)], - [(folder, -1, -1)], - [(job, -1, -1)], + [(show, alloc, 1_000_000, -1)], + [(folder, -1, -1, -1)], + [(job, -1, -1, -1)], ); let d = delta(show, alloc, folder, job, 500, 4); assert!(applied(store.book(&d))); @@ -696,13 +840,13 @@ mod tests { fn confirm_then_recompute_keeps_booked() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, -1, -1, -1)]); let d = delta(show, alloc, folder, job, 10, 0); assert!(applied(store.book(&d))); store.confirm(&d); let epoch = store.begin_recompute(); let snap = CounterSnapshot { - job: [(job, (10, 0))].into_iter().collect(), + job: [(job, (10, 0, 0))].into_iter().collect(), ..Default::default() }; store.overwrite_counters(&snap, epoch); @@ -713,14 +857,14 @@ mod tests { fn rollback_undoes_book() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, -1, -1, -1)]); let d = delta(show, alloc, folder, job, 10, 0); assert!(applied(store.book(&d))); store.rollback(&d); assert_eq!(store.job_cores_in_use(job), 0); let epoch = store.begin_recompute(); let snap = CounterSnapshot { - job: [(job, (0, 0))].into_iter().collect(), + job: [(job, (0, 0, 0))].into_iter().collect(), ..Default::default() }; store.overwrite_counters(&snap, epoch); @@ -732,15 +876,15 @@ mod tests { fn recompute_carries_forward_in_flight_booking() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, 20, -1)]); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, 20, -1, -1)]); let d = delta(show, alloc, folder, job, 8, 0); assert!(applied(store.book(&d))); // booked, still in-flight (not confirmed) let epoch = store.begin_recompute(); let snap = CounterSnapshot { - sub: [((show, alloc), (0, 0))].into_iter().collect(), - folder: [(folder, (0, 0))].into_iter().collect(), - job: [(job, (0, 0))].into_iter().collect(), + sub: [((show, alloc), (0, 0, 0))].into_iter().collect(), + folder: [(folder, (0, 0, 0))].into_iter().collect(), + job: [(job, (0, 0, 0))].into_iter().collect(), }; store.overwrite_counters(&snap, epoch); assert_eq!(store.job_cores_in_use(job), 8); @@ -754,7 +898,7 @@ mod tests { fn recompute_does_not_erase_booking_confirmed_after_snapshot_read() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, 20, -1)]); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, 20, -1, -1)]); let d = delta(show, alloc, folder, job, 8, 0); assert!(applied(store.book(&d))); @@ -764,9 +908,9 @@ mod tests { store.confirm(&d); // Overwrite lands. The confirm tagged the other settled bucket, so 8 survives. let snap = CounterSnapshot { - sub: [((show, alloc), (0, 0))].into_iter().collect(), - folder: [(folder, (0, 0))].into_iter().collect(), - job: [(job, (0, 0))].into_iter().collect(), + sub: [((show, alloc), (0, 0, 0))].into_iter().collect(), + folder: [(folder, (0, 0, 0))].into_iter().collect(), + job: [(job, (0, 0, 0))].into_iter().collect(), }; store.overwrite_counters(&snap, epoch); assert_eq!( @@ -778,9 +922,9 @@ mod tests { // The following recompute (proc now visible) reconciles cleanly to the true value. let epoch2 = store.begin_recompute(); let snap2 = CounterSnapshot { - sub: [((show, alloc), (8 * 100 / 100, 0))].into_iter().collect(), - folder: [(folder, (8, 0))].into_iter().collect(), - job: [(job, (8, 0))].into_iter().collect(), + sub: [((show, alloc), (8 * 100 / 100, 0, 0))].into_iter().collect(), + folder: [(folder, (8, 0, 0))].into_iter().collect(), + job: [(job, (8, 0, 0))].into_iter().collect(), }; store.overwrite_counters(&snap2, epoch2); assert_eq!(store.job_cores_in_use(job), 8); @@ -790,7 +934,7 @@ mod tests { fn release_decrements_unconditionally() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, -1, -1, -1)]); let d = delta(show, alloc, folder, job, 10, 0); assert!(applied(store.book(&d))); store.confirm(&d); @@ -805,7 +949,7 @@ mod tests { fn missed_release_heals_via_recompute() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, -1, -1, -1)]); let d = delta(show, alloc, folder, job, 10, 0); assert!(applied(store.book(&d))); store.confirm(&d); @@ -813,18 +957,18 @@ mod tests { let e1 = store.begin_recompute(); store.overwrite_counters( &CounterSnapshot { - sub: [((show, alloc), (10, 0))].into_iter().collect(), - folder: [(folder, (10, 0))].into_iter().collect(), - job: [(job, (10, 0))].into_iter().collect(), + sub: [((show, alloc), (10, 0, 0))].into_iter().collect(), + folder: [(folder, (10, 0, 0))].into_iter().collect(), + job: [(job, (10, 0, 0))].into_iter().collect(), }, e1, ); let e2 = store.begin_recompute(); store.overwrite_counters( &CounterSnapshot { - sub: [((show, alloc), (10, 0))].into_iter().collect(), - folder: [(folder, (10, 0))].into_iter().collect(), - job: [(job, (10, 0))].into_iter().collect(), + sub: [((show, alloc), (10, 0, 0))].into_iter().collect(), + folder: [(folder, (10, 0, 0))].into_iter().collect(), + job: [(job, (10, 0, 0))].into_iter().collect(), }, e2, ); @@ -835,9 +979,9 @@ mod tests { let e3 = store.begin_recompute(); store.overwrite_counters( &CounterSnapshot { - sub: [((show, alloc), (0, 0))].into_iter().collect(), - folder: [(folder, (0, 0))].into_iter().collect(), - job: [(job, (0, 0))].into_iter().collect(), + sub: [((show, alloc), (0, 0, 0))].into_iter().collect(), + folder: [(folder, (0, 0, 0))].into_iter().collect(), + job: [(job, (0, 0, 0))].into_iter().collect(), }, e3, ); @@ -851,12 +995,12 @@ mod tests { fn managed_flip_seed_prevents_overbook() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, -1, -1)]); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, -1, -1, -1)]); // Cuebot already has 90 cores booked on this (show, alloc) at flip time. let seed = CounterSnapshot { - sub: [((show, alloc), (90, 0))].into_iter().collect(), - folder: [(folder, (90, 0))].into_iter().collect(), - job: [(job, (90, 0))].into_iter().collect(), + sub: [((show, alloc), (90, 0, 0))].into_iter().collect(), + folder: [(folder, (90, 0, 0))].into_iter().collect(), + job: [(job, (90, 0, 0))].into_iter().collect(), }; store.seed_show_booked(&seed); assert_eq!(store.sub_counters(show, alloc), (90, 100)); @@ -877,7 +1021,7 @@ mod tests { fn live_limit_change_updates_cap() { let (show, alloc, folder, job) = ids(); let store = Store::new(); - store.set_caps([(show, alloc, 100)], [(folder, -1, -1)], [(job, 50, -1)]); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, 50, -1, -1)]); let d = delta(show, alloc, folder, job, 40, 0); assert!(applied(store.book(&d))); // 40 <= 50 // Operator lowers the hard cap to 30 live; further bookings must reject. @@ -891,4 +1035,80 @@ mod tests { BookOutcome::LimitExceeded { table: "job", .. } )); } + + // ── Slot axis ──────────────────────────────────────────────────────────── + + /// Missing slot cap (unseeded) rejects all slot work — fail-closed. + #[test] + fn slot_book_missing_cap_rejects_all() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + // No set_caps: slot caps default to 0 = reject-all. + assert!(matches!( + store.book(&slot_delta(show, alloc, folder, job, 1)), + BookOutcome::LimitExceeded { + table: "subscription_slots", + .. + } + )); + } + + /// `-1` slot cap is unlimited; many slot bookings succeed. + #[test] + fn slot_unlimited_cap_never_rejects() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, -1, -1, -1)]); + for _ in 0..100 { + assert!(applied(store.book(&slot_delta(show, alloc, folder, job, 1)))); + } + assert_eq!(store.job_slots_in_use(job), 100); + } + + /// The folder slot cap is enforced (and `0` means reject-all even at the folder level). + #[test] + fn slot_book_respects_folder_cap() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + // sub/job unlimited, folder capped at 2 slots. + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, 2)], [(job, -1, -1, -1)]); + assert!(applied(store.book(&slot_delta(show, alloc, folder, job, 2)))); // fills the cap + assert!(matches!( + store.book(&slot_delta(show, alloc, folder, job, 1)), + BookOutcome::LimitExceeded { + table: "folder_slots", + current: 2, + limit: 2 + } + )); + } + + /// The slot axis is fully independent of the core/gpu axes: a slot booking + /// consumes no core budget, and a core booking consumes no slot budget. + #[test] + fn slot_axis_independent_from_cores() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + // Cores capped tight at 1; slots capped generously. + store.set_caps([(show, alloc, 1, -1)], [(folder, 1, -1, -1)], [(job, 1, -1, 10)]); + // A 5-slot booking (0 cores) is unaffected by the 1-core cap. + assert!(applied(store.book(&slot_delta(show, alloc, folder, job, 5)))); + assert_eq!(store.job_slots_in_use(job), 5); + assert_eq!(store.job_cores_in_use(job), 0); + // A 1-core booking still fits its own budget, untouched by slot usage. + assert!(applied(store.book(&delta(show, alloc, folder, job, 1, 0)))); + assert_eq!(store.job_cores_in_use(job), 1); + } + + /// A release NOTIFY with a negative slot delta decrements the slot counters. + #[test] + fn slot_release_decrements() { + let (show, alloc, folder, job) = ids(); + let store = Store::new(); + store.set_caps([(show, alloc, 100, -1)], [(folder, -1, -1, -1)], [(job, -1, -1, 10)]); + assert!(applied(store.book(&slot_delta(show, alloc, folder, job, 4)))); + assert_eq!(store.job_slots_in_use(job), 4); + store.apply_release(&slot_delta(show, alloc, folder, job, -3)); + assert_eq!(store.job_slots_in_use(job), 1); + } } diff --git a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs index 57bbe3f4fd..950884a4d0 100644 --- a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs +++ b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs @@ -523,6 +523,9 @@ impl RqdDispatcherService { job_id: virtual_proc.job_id, core_delta: i64::from(cores_reserved.value()), gpu_delta: virtual_proc.gpus_reserved as i32, + // Slot frames reserve 0 cores/gpus; regular frames reserve 0 slots. + // The axes are independent, so exactly one of these is non-zero. + slot_delta: virtual_proc.slots_required as i64, }; // Per-cluster host accounting check - this dispatcher iteration may have already From 4c6f626e20c56882c8b9aabcb64673a93bedc5ea Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Fri, 3 Jul 2026 15:31:37 -0700 Subject: [PATCH 14/20] [cuebot] Publish slot deltas to the scheduler accounting store Complete the slot accounting round-trip on the Cuebot side: - VirtualProc.slotsReserved, hydrated from proc.int_slots_reserved by the shared VIRTUAL_PROC_MAPPER (GET_VIRTUAL_PROC gains the column; GET_VIRTUAL_PROC_LIST already selects proc.*). - AccountingNotifier.notifyRelease includes negated slots in the acct_release payload (0 for regular procs, so a no-op on the slot axis for them). - notifySubscriptionMaxSlots / notifyFolderMaxSlots / notifyJobMaxSlots emit acct_limit_change on the new slot caps (whole counts, -1 unlimited, 0 reject-all), called by the admin setters added in the management-surface change. compileJava passes (JDK 11). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../com/imageworks/spcue/VirtualProc.java | 3 ++ .../spcue/dao/postgres/ProcDaoJdbc.java | 2 + .../spcue/service/AccountingNotifier.java | 41 ++++++++++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java index c48b8ff426..1f4739ff4e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java @@ -45,6 +45,9 @@ public class VirtualProc extends FrameEntity implements ProcInterface { public int gpusReserved; public long gpuMemoryReserved; public long gpuMemoryUsed; + + // Concurrency slots reserved by this proc (slot-based scheduling). 0 for regular procs. + public int slotsReserved; public long gpuMemoryMax; public boolean unbooked; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 3400fd5e46..c3dccb6733 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -341,6 +341,7 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { proc.memoryReserved = rs.getLong("int_mem_reserved"); proc.memoryMax = rs.getLong("int_mem_max_used"); proc.gpusReserved = rs.getInt("int_gpus_reserved"); + proc.slotsReserved = rs.getInt("int_slots_reserved"); proc.gpuMemoryReserved = rs.getLong("int_gpu_mem_reserved"); proc.gpuMemoryMax = rs.getLong("int_gpu_mem_max_used"); proc.virtualMemoryMax = rs.getLong("int_virt_max_used"); @@ -370,6 +371,7 @@ public VirtualProc mapRow(ResultSet rs, int rowNum) throws SQLException { + "host.pk_alloc, " + "alloc.pk_facility," + "proc.int_cores_reserved," + + "proc.int_slots_reserved," + "proc.int_mem_reserved," + "proc.int_mem_max_used," + "proc.int_mem_used," diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java index d34e8e434f..f00e0e0d28 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java @@ -129,11 +129,14 @@ public void notifyRelease(VirtualProc proc) { int cores = -proc.coresReserved / CENTICORES_PER_CORE; int gpus = -proc.gpusReserved; + // Slots are whole counts (no centicore conversion); negated like cores/gpus. 0 for + // regular procs, so this is a no-op on the scheduler's slot axis for non-slot procs. + int slots = -proc.slotsReserved; String payload = String.format( "{\"show\":\"%s\",\"alloc\":\"%s\",\"folder\":\"%s\",\"job\":\"%s\"," - + "\"layer\":\"%s\",\"dept\":\"%s\",\"cores\":%d,\"gpus\":%d}", + + "\"layer\":\"%s\",\"dept\":\"%s\",\"cores\":%d,\"gpus\":%d,\"slots\":%d}", proc.getShowId(), proc.getAllocationId(), proc.folderId, proc.getJobId(), - proc.getLayerId(), proc.deptId, cores, gpus); + proc.getLayerId(), proc.deptId, cores, gpus, slots); notify(CHANNEL_RELEASE, payload); } @@ -194,6 +197,40 @@ public void notifyJobMaxGpus(String jobId, int value) { notify(CHANNEL_LIMIT_CHANGE, payload); } + /** + * Emit a subscription max-slots cap change. Slots are whole counts (no centicore + * conversion); {@code -1} = unlimited, {@code 0} = reject all slot work. + */ + public void notifySubscriptionMaxSlots(String showId, String allocId, int value) { + if (!notifyEnabled) { + return; + } + String payload = String.format( + "{\"vertex\":\"sub\",\"show\":\"%s\",\"alloc\":\"%s\",\"max_slots\":%d}", showId, + allocId, value); + notify(CHANNEL_LIMIT_CHANGE, payload); + } + + /** Emit a folder max-slots cap change. Whole counts; {@code -1} = unlimited. */ + public void notifyFolderMaxSlots(String folderId, int value) { + if (!notifyEnabled) { + return; + } + String payload = String.format("{\"vertex\":\"folder\",\"id\":\"%s\",\"max_slots\":%d}", + folderId, value); + notify(CHANNEL_LIMIT_CHANGE, payload); + } + + /** Emit a job max-slots cap change. Whole counts; {@code -1} = unlimited. */ + public void notifyJobMaxSlots(String jobId, int value) { + if (!notifyEnabled) { + return; + } + String payload = String.format("{\"vertex\":\"job\",\"id\":\"%s\",\"max_slots\":%d}", jobId, + value); + notify(CHANNEL_LIMIT_CHANGE, payload); + } + /** * Issue the {@code pg_notify} inside the current transaction. All payload fields are UUIDs or * integers (built via String.format), so no JSON escaping is required. From 6c4fb93f8322170f6fe798faef1660047dd6641c Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Fri, 3 Jul 2026 15:54:37 -0700 Subject: [PATCH 15/20] [proto/cuebot/pycue/cuegui/pyoutline] Slot-based scheduling management surface Expose slot configuration across the stack, mirroring the max_cores/burst pattern. Proto: max_slots on Group/Job/Subscription + SetMaxSlots RPCs; slots_required on Layer; concurrent_slots_limit on Host + SetConcurrentSlotsLimit RPC. Cuebot: setMaxSlots down servant -> service -> DAO for group/job/subscription (each DAO fires AccountingNotifier.notify*MaxSlots when scheduler-managed); host setConcurrentSlotsLimit (per-host cap, no notifier); whiteboard mappers + SELECTs read the new columns for display (incl. the reused Nested job mapper). Layer submission path: cjsl-1.16.dtd slots_required element, JobSpec parse, LayerDetail.slotsRequired, LayerDaoJdbc insert. pycue: setMaxSlots (group/job/subscription), setConcurrentSlotsLimit (host). pyoutline: slots_required layer arg + serialization (spec_version 1.16). cuegui: HostMonitorTree "Concurrent Slots" column + setConcurrentSlotsLimit action (-1 disable / 0 reject-all / N cap). Slots are whole counts (no centicore conversion). compileJava + spotlessJavaCheck pass (JDK 11); pycue/cuegui/pyoutline py_compile clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../com/imageworks/spcue/LayerDetail.java | 2 + .../com/imageworks/spcue/dao/GroupDao.java | 9 ++ .../com/imageworks/spcue/dao/HostDao.java | 9 ++ .../java/com/imageworks/spcue/dao/JobDao.java | 16 +++ .../imageworks/spcue/dao/SubscriptionDao.java | 9 ++ .../spcue/dao/postgres/GroupDaoJdbc.java | 16 +++ .../spcue/dao/postgres/HostDaoJdbc.java | 11 ++ .../spcue/dao/postgres/JobDaoJdbc.java | 26 +++++ .../spcue/dao/postgres/LayerDaoJdbc.java | 7 +- .../dao/postgres/NestedWhiteboardDaoJdbc.java | 1 + .../dao/postgres/SubscriptionDaoJdbc.java | 15 +++ .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 9 +- .../imageworks/spcue/servant/ManageGroup.java | 12 ++ .../imageworks/spcue/servant/ManageHost.java | 11 ++ .../imageworks/spcue/servant/ManageJob.java | 19 ++++ .../spcue/servant/ManageSubscription.java | 11 ++ .../spcue/service/AccountingNotifier.java | 8 +- .../spcue/service/AdminManager.java | 2 + .../spcue/service/AdminManagerService.java | 4 + .../spcue/service/GroupManager.java | 2 + .../spcue/service/GroupManagerService.java | 5 + .../com/imageworks/spcue/service/JobSpec.java | 4 + .../main/resources/public/dtd/cjsl-1.16.dtd | 106 ++++++++++++++++++ cuegui/cuegui/HostMonitorTree.py | 9 ++ cuegui/cuegui/MenuActions.py | 38 +++++++ proto/src/host.proto | 14 +++ proto/src/job.proto | 26 +++++ proto/src/subscription.proto | 13 +++ pycue/opencue/wrappers/group.py | 9 ++ pycue/opencue/wrappers/host.py | 12 ++ pycue/opencue/wrappers/job.py | 9 ++ pycue/opencue/wrappers/subscription.py | 12 ++ pyoutline/outline/backend/cue.py | 7 ++ pyoutline/outline/layer.py | 2 + pyoutline/outline/outline.cfg | 2 +- 35 files changed, 458 insertions(+), 9 deletions(-) create mode 100644 cuebot/src/main/resources/public/dtd/cjsl-1.16.dtd diff --git a/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java b/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java index 5721390390..1de4667e6e 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java +++ b/cuebot/src/main/java/com/imageworks/spcue/LayerDetail.java @@ -38,6 +38,8 @@ public class LayerDetail extends LayerEntity implements LayerInterface { public int timeout_llu; public int dispatchOrder; public int totalFrameCount; + // Concurrency slots each frame requires (slot-based scheduling). 0 = not slot-based. + public int slotsRequired; public Set tags = new LinkedHashSet(); public Set services = new LinkedHashSet(); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java index be6f0a5e4c..cff99891c4 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/GroupDao.java @@ -125,6 +125,15 @@ public interface GroupDao { */ public void updateMaxCores(GroupInterface group, int value); + /** + * Sets the max concurrent slots for slot-based layers in the group's folder. -1 unlimited, 0 + * reject-all, N caps at N. + * + * @param group + * @param value + */ + public void updateMaxSlots(GroupInterface group, int value); + /** * Set the minimum number of cores for this group * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java index 2193e86d6d..07a23d6785 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/HostDao.java @@ -245,6 +245,15 @@ public interface HostDao { */ void updateThreadMode(HostInterface host, ThreadMode mode); + /** + * Sets the host's concurrent slots limit. -1 disables slot mode (regular host); >= 0 makes the + * host slot-based, running only slot layers up to this many concurrent slots. + * + * @param host HostInterface + * @param limit int + */ + void updateConcurrentSlotsLimit(HostInterface host, int limit); + /** * Update the specified host's hardware information. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java index 37736cbb46..1064ed517b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java @@ -45,6 +45,14 @@ public interface JobDao { */ public void updateMaxCores(GroupInterface g, int cores); + /** + * Updates all jobs in the specified group to the max slots value. + * + * @param g + * @param slots + */ + public void updateMaxSlots(GroupInterface g, int slots); + /** * Updates all jobs in the specifid group to the min cores value. * @@ -306,6 +314,14 @@ public interface JobDao { */ void updateMaxCores(JobInterface j, int v); + /** + * Sets the job's max concurrent slots for slot-based layers. -1 unlimited, 0 reject-all. + * + * @param j + * @param v + */ + void updateMaxSlots(JobInterface j, int v); + /** * sets the jobs new min gpu value * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/SubscriptionDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/SubscriptionDao.java index 70aa8c3ce5..9ece29a4f5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/SubscriptionDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/SubscriptionDao.java @@ -114,4 +114,13 @@ public interface SubscriptionDao { * @param size int */ void updateSubscriptionBurst(SubscriptionInterface sub, int size); + + /** + * update the subscription max concurrent slots for slot-based layers (-1 unlimited, 0 + * reject-all, N cap) + * + * @param sub SubscriptionInterface + * @param maxSlots int + */ + void updateSubscriptionMaxSlots(SubscriptionInterface sub, int maxSlots); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java index 5aa06d1ec0..5acf611e82 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/GroupDaoJdbc.java @@ -215,6 +215,22 @@ public void updateMaxCores(GroupInterface group, int value) { } } + @Override + public void updateMaxSlots(GroupInterface group, int value) { + // Slots are whole counts: -1 = unlimited, 0 = reject-all, N = cap. Normalize any + // negative to the -1 sentinel. + if (value < 0) { + value = CueUtil.FEATURE_DISABLED; + } + + getJdbcTemplate().update("UPDATE folder_resource SET int_max_slots=? WHERE pk_folder=?", + value, group.getId()); + + if (accountingNotifier.isEnabled() && showDao.isSchedulerManaged(group.getShowId())) { + accountingNotifier.notifyFolderMaxSlots(group.getId(), value); + } + } + @Override public void updateMinCores(GroupInterface group, int value) { if (value < 0) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index 9007a5345c..d74eae703a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -579,6 +579,17 @@ public void updateThreadMode(HostInterface host, ThreadMode mode) { mode.getNumber(), host.getHostId()); } + @Override + public void updateConcurrentSlotsLimit(HostInterface host, int limit) { + // -1 disables slot mode; >= 0 caps concurrent slots. The per-host slot cap is enforced + // in the scheduler's host cache (not the accounting store), so no NOTIFY is emitted. + if (limit < 0) { + limit = -1; + } + getJdbcTemplate().update("UPDATE host SET int_concurrent_slots_limit=? WHERE pk_host=?", + limit, host.getHostId()); + } + @Override public void updateHostOs(HostInterface host, String os) { getJdbcTemplate().update("UPDATE host_stat SET str_os=? WHERE pk_host=?", os, diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java index a35c84ba6c..faa4e193d5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java @@ -385,6 +385,32 @@ public void updateMaxCores(JobInterface j, int v) { } } + @Override + public void updateMaxSlots(GroupInterface g, int v) { + // Slots are whole counts: -1 unlimited, 0 reject-all, N cap. + if (v < 0) { + v = CueUtil.FEATURE_DISABLED; + } + getJdbcTemplate().update( + "UPDATE job_resource SET int_max_slots=? WHERE " + + "pk_job IN (SELECT pk_job FROM job WHERE pk_folder=?)", + v, g.getGroupId()); + } + + @Override + public void updateMaxSlots(JobInterface j, int v) { + // Slots are whole counts: -1 unlimited, 0 reject-all, N cap. + if (v < 0) { + v = CueUtil.FEATURE_DISABLED; + } + getJdbcTemplate().update("UPDATE job_resource SET int_max_slots=? WHERE pk_job=?", v, + j.getJobId()); + + if (accountingNotifier.isEnabled() && showDao.isSchedulerManaged(j.getShowId())) { + accountingNotifier.notifyJobMaxSlots(j.getJobId(), v); + } + } + @Override public void updateMinGpus(GroupInterface g, int v) { getJdbcTemplate().update( diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java index cd9047baa8..ea397bedaf 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java @@ -320,9 +320,10 @@ public LayerInterface getLayer(String id) { + "int_gpu_mem_min, " + "str_services, " + "int_timeout," - + "int_timeout_llu " + + "int_timeout_llu, " + + "int_slots_required " + ") " - + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; // spotless:on @Override @@ -332,7 +333,7 @@ public void insertLayerDetail(LayerDetail l) { l.chunkSize, l.dispatchOrder, StringUtils.join(l.tags, " | "), l.type.toString(), l.minimumCores, l.maximumCores, l.isThreadable, l.minimumMemory, l.minimumGpus, l.maximumGpus, l.minimumGpuMemory, StringUtils.join(l.services, ","), l.timeout, - l.timeout_llu); + l.timeout_llu, l.slotsRequired); } @Override diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java index 8f9e6e1aa0..3a19aa7b41 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/NestedWhiteboardDaoJdbc.java @@ -135,6 +135,7 @@ public CachedJobWhiteboardMapper(NestedJobWhiteboardMapper result) { + "job_resource.int_min_cores, " + "job_resource.int_min_gpus, " + "job_resource.int_max_cores, " + + "job_resource.int_max_slots, " + "job_resource.int_max_gpus, " + "job_mem.int_max_rss, " + "job_mem.int_max_pss " diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SubscriptionDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SubscriptionDaoJdbc.java index f04126a865..6546437a11 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SubscriptionDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SubscriptionDaoJdbc.java @@ -250,4 +250,19 @@ public void updateSubscriptionBurst(SubscriptionInterface sub, int size) { size); } } + + @Override + public void updateSubscriptionMaxSlots(SubscriptionInterface sub, int maxSlots) { + // Slots are whole counts: -1 unlimited, 0 reject-all, N cap. + if (maxSlots < 0) { + maxSlots = -1; + } + getJdbcTemplate().update("UPDATE subscription SET int_max_slots=? WHERE pk_subscription=?", + maxSlots, sub.getSubscriptionId()); + + if (accountingNotifier.isEnabled() && showDao.isSchedulerManaged(sub.getShowId())) { + accountingNotifier.notifySubscriptionMaxSlots(sub.getShowId(), sub.getAllocationId(), + maxSlots); + } + } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index ba8aa773c3..5d6c90fc98 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -998,6 +998,7 @@ public static Host.Builder mapHostBuilder(ResultSet rs) throws SQLException { builder.setLockState(LockState.valueOf(SqlUtil.getString(rs, "str_lock_state"))); builder.setHasComment(rs.getBoolean("b_comment")); builder.setThreadMode(ThreadMode.values()[rs.getInt("int_thread_mode")]); + builder.setConcurrentSlotsLimit(rs.getInt("int_concurrent_slots_limit")); builder.setOs(SqlUtil.getString(rs, "str_os")); String tags = SqlUtil.getString(rs, "str_tags"); @@ -1074,7 +1075,7 @@ public Group mapRow(ResultSet rs, int rowNum) throws SQLException { .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) .setMaxGpus(rs.getInt("int_max_gpus")).setMinGpus(rs.getInt("int_min_gpus")) - .setLevel(rs.getInt("int_level")) + .setMaxSlots(rs.getInt("int_max_slots")).setLevel(rs.getInt("int_level")) .setParentId(SqlUtil.getString(rs, "pk_parent_folder")).setGroupStats(stats) .build(); } @@ -1087,6 +1088,7 @@ public Job mapRow(ResultSet rs, int rowNum) throws SQLException { .setMaxCores(Convert.coreUnitsToCores(rs.getInt("int_max_cores"))) .setMinCores(Convert.coreUnitsToCores(rs.getInt("int_min_cores"))) .setMaxGpus(rs.getInt("int_max_gpus")).setMinGpus(rs.getInt("int_min_gpus")) + .setMaxSlots(rs.getInt("int_max_slots")) .setName(SqlUtil.getString(rs, "str_name")) .setPriority(rs.getInt("int_priority")) .setShot(SqlUtil.getString(rs, "str_shot")) @@ -1282,6 +1284,7 @@ public Subscription mapRow(ResultSet rs, int rowNum) throws SQLException { .setBurst(rs.getInt("int_burst")).setName(rs.getString("name")) .setReservedCores(rs.getInt("int_cores")) .setReservedGpus(rs.getInt("int_gpus")).setSize(rs.getInt("int_size")) + .setMaxSlots(rs.getInt("int_max_slots")) .setAllocationName(rs.getString("alloc_name")) .setShowName(rs.getString("show_name")) .setFacility(rs.getString("facility_name")).build(); @@ -1869,6 +1872,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { + "folder.int_job_max_cores," + "folder_resource.int_min_cores," + "folder_resource.int_max_cores," + + "folder_resource.int_max_slots," + "folder.int_job_min_gpus," + "folder.int_job_max_gpus," + "folder_resource.int_min_gpus," @@ -1919,6 +1923,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { + "job.pk_job," + "job.str_log_dir," + "job_resource.int_max_cores," + + "job_resource.int_max_slots," + "job_resource.int_min_cores," + "job_resource.int_max_gpus," + "job_resource.int_min_gpus," @@ -2220,6 +2225,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { + "host.str_lock_state," + "host.b_comment," + "host.int_thread_mode," + + "host.int_concurrent_slots_limit," + "host_stat.str_os," + "host_stat.int_mem_total," + "host_stat.int_mem_free," @@ -2268,6 +2274,7 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { + "subscription.pk_subscription, " + "(alloc.str_name || '.' || show.str_name) AS name, " + "subscription.int_burst, " + + "subscription.int_max_slots, " + "subscription.int_size, " + "subscription.int_cores, " + "subscription.int_gpus, " diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageGroup.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageGroup.java index baad91fb25..0ff8dff69f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageGroup.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageGroup.java @@ -63,6 +63,8 @@ import com.imageworks.spcue.grpc.job.GroupSetGroupResponse; import com.imageworks.spcue.grpc.job.GroupSetMaxCoresRequest; import com.imageworks.spcue.grpc.job.GroupSetMaxCoresResponse; +import com.imageworks.spcue.grpc.job.GroupSetMaxSlotsRequest; +import com.imageworks.spcue.grpc.job.GroupSetMaxSlotsResponse; import com.imageworks.spcue.grpc.job.GroupSetMinCoresRequest; import com.imageworks.spcue.grpc.job.GroupSetMinCoresResponse; import com.imageworks.spcue.grpc.job.GroupSetMaxGpusRequest; @@ -273,6 +275,16 @@ public void setMaxCores(GroupSetMaxCoresRequest request, responseObserver.onCompleted(); } + @Override + public void setMaxSlots(GroupSetMaxSlotsRequest request, + StreamObserver responseObserver) { + GroupInterface group = getGroupInterface(request.getGroup()); + // Slots are whole counts, no core-unit conversion. + groupManager.setGroupMaxSlots(group, request.getMaxSlots()); + responseObserver.onNext(GroupSetMaxSlotsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + @Override public void setMinCores(GroupSetMinCoresRequest request, StreamObserver responseObserver) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageHost.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageHost.java index 5732af62db..93093a361f 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageHost.java @@ -75,6 +75,8 @@ import com.imageworks.spcue.grpc.host.HostSetHardwareStateResponse; import com.imageworks.spcue.grpc.host.HostSetOsRequest; import com.imageworks.spcue.grpc.host.HostSetOsResponse; +import com.imageworks.spcue.grpc.host.HostSetConcurrentSlotsLimitRequest; +import com.imageworks.spcue.grpc.host.HostSetConcurrentSlotsLimitResponse; import com.imageworks.spcue.grpc.host.HostSetThreadModeRequest; import com.imageworks.spcue.grpc.host.HostSetThreadModeResponse; import com.imageworks.spcue.grpc.host.HostUnlockRequest; @@ -264,6 +266,15 @@ public void setThreadMode(HostSetThreadModeRequest request, responseObserver.onCompleted(); } + @Override + public void setConcurrentSlotsLimit(HostSetConcurrentSlotsLimitRequest request, + StreamObserver responseObserver) { + HostInterface host = getHostInterface(request.getHost()); + hostDao.updateConcurrentSlotsLimit(host, request.getConcurrentSlotsLimit()); + responseObserver.onNext(HostSetConcurrentSlotsLimitResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + @Override public void setHardwareState(HostSetHardwareStateRequest request, StreamObserver responseObserver) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java index 96655bb076..330eb1bb0a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageJob.java @@ -128,6 +128,8 @@ import com.imageworks.spcue.grpc.job.JobSetGroupResponse; import com.imageworks.spcue.grpc.job.JobSetMaxCoresRequest; import com.imageworks.spcue.grpc.job.JobSetMaxCoresResponse; +import com.imageworks.spcue.grpc.job.JobSetMaxSlotsRequest; +import com.imageworks.spcue.grpc.job.JobSetMaxSlotsResponse; import com.imageworks.spcue.grpc.job.JobSetMaxGpusRequest; import com.imageworks.spcue.grpc.job.JobSetMaxGpusResponse; import com.imageworks.spcue.grpc.job.JobSetMaxRetriesRequest; @@ -363,6 +365,23 @@ public void setMaxCores(JobSetMaxCoresRequest request, } } + @Override + public void setMaxSlots(JobSetMaxSlotsRequest request, + StreamObserver responseObserver) { + try { + setupJobData(request.getJob()); + if (attemptChange(env, property, jobManager, job, responseObserver)) { + // Slots are whole counts, no core-unit conversion. + jobDao.updateMaxSlots(job, request.getVal()); + responseObserver.onNext(JobSetMaxSlotsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + } catch (EmptyResultDataAccessException e) { + responseObserver.onError( + Status.NOT_FOUND.withDescription("Job not found").asRuntimeException()); + } + } + @Override public void setMinCores(JobSetMinCoresRequest request, StreamObserver responseObserver) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageSubscription.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageSubscription.java index 3d36e4b939..56012830ed 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageSubscription.java +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageSubscription.java @@ -32,6 +32,8 @@ import com.imageworks.spcue.grpc.subscription.SubscriptionInterfaceGrpc; import com.imageworks.spcue.grpc.subscription.SubscriptionSetBurstRequest; import com.imageworks.spcue.grpc.subscription.SubscriptionSetBurstResponse; +import com.imageworks.spcue.grpc.subscription.SubscriptionSetMaxSlotsRequest; +import com.imageworks.spcue.grpc.subscription.SubscriptionSetMaxSlotsResponse; import com.imageworks.spcue.grpc.subscription.SubscriptionSetSizeRequest; import com.imageworks.spcue.grpc.subscription.SubscriptionSetSizeResponse; import com.imageworks.spcue.service.AdminManager; @@ -105,6 +107,15 @@ public void setSize(SubscriptionSetSizeRequest request, responseObserver.onCompleted(); } + @Override + public void setMaxSlots(SubscriptionSetMaxSlotsRequest request, + StreamObserver responseObserver) { + adminManager.setSubscriptionMaxSlots(getSubscriptionDetail(request.getSubscription()), + request.getMaxSlots()); + responseObserver.onNext(SubscriptionSetMaxSlotsResponse.newBuilder().build()); + responseObserver.onCompleted(); + } + public AdminManager getAdminManager() { return adminManager; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java index f00e0e0d28..32df1851ea 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AccountingNotifier.java @@ -198,8 +198,8 @@ public void notifyJobMaxGpus(String jobId, int value) { } /** - * Emit a subscription max-slots cap change. Slots are whole counts (no centicore - * conversion); {@code -1} = unlimited, {@code 0} = reject all slot work. + * Emit a subscription max-slots cap change. Slots are whole counts (no centicore conversion); + * {@code -1} = unlimited, {@code 0} = reject all slot work. */ public void notifySubscriptionMaxSlots(String showId, String allocId, int value) { if (!notifyEnabled) { @@ -226,8 +226,8 @@ public void notifyJobMaxSlots(String jobId, int value) { if (!notifyEnabled) { return; } - String payload = String.format("{\"vertex\":\"job\",\"id\":\"%s\",\"max_slots\":%d}", jobId, - value); + String payload = + String.format("{\"vertex\":\"job\",\"id\":\"%s\",\"max_slots\":%d}", jobId, value); notify(CHANNEL_LIMIT_CHANGE, payload); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java index 54fc73f46d..85d7d353ba 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManager.java @@ -90,6 +90,8 @@ SubscriptionInterface createSubscription(ShowInterface show, AllocationInterface void setSubscriptionBurst(SubscriptionInterface sub, int burst); + void setSubscriptionMaxSlots(SubscriptionInterface sub, int maxSlots); + void setSubscriptionSize(SubscriptionInterface sub, int size); SubscriptionEntity getSubscriptionDetail(String id); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java index dfc997fb48..4eccabede7 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/AdminManagerService.java @@ -173,6 +173,10 @@ public void setSubscriptionBurst(SubscriptionInterface sub, int burst) { subscriptionDao.updateSubscriptionBurst(sub, burst); } + public void setSubscriptionMaxSlots(SubscriptionInterface sub, int maxSlots) { + subscriptionDao.updateSubscriptionMaxSlots(sub, maxSlots); + } + public void setSubscriptionSize(SubscriptionInterface sub, int size) { subscriptionDao.updateSubscriptionSize(sub, size); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/GroupManager.java b/cuebot/src/main/java/com/imageworks/spcue/service/GroupManager.java index 06289ec50c..de5a8dc02c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/GroupManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/GroupManager.java @@ -28,6 +28,8 @@ public interface GroupManager { void setGroupMaxCores(GroupInterface g, int coreUnits); + void setGroupMaxSlots(GroupInterface g, int slots); + void setGroupMinCores(GroupInterface g, int coreUnits); void setGroupDefaultJobMinCores(GroupInterface g, int coreUnits); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/GroupManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/GroupManagerService.java index b969839dee..cc9bfa8c47 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/GroupManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/GroupManagerService.java @@ -69,6 +69,11 @@ public void setGroupMaxCores(GroupInterface g, int coreUnits) { groupDao.updateMaxCores(g, coreUnits); } + @Override + public void setGroupMaxSlots(GroupInterface g, int slots) { + groupDao.updateMaxSlots(g, slots); + } + @Override public void setGroupMinCores(GroupInterface g, int coreUnits) { groupDao.updateMinCores(g, coreUnits); diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index d4ff4e6b5d..bed7fa9033 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -444,6 +444,10 @@ private void handleLayerTags(BuildableJob buildableJob, Element jobTag) { layer.timeout_llu = Integer.parseInt(layerTag.getChildTextTrim("timeout_llu")); } + if (layerTag.getChildTextTrim("slots_required") != null) { + layer.slotsRequired = Integer.parseInt(layerTag.getChildTextTrim("slots_required")); + } + /* * Handle the layer environment */ diff --git a/cuebot/src/main/resources/public/dtd/cjsl-1.16.dtd b/cuebot/src/main/resources/public/dtd/cjsl-1.16.dtd new file mode 100644 index 0000000000..20ebf19c21 --- /dev/null +++ b/cuebot/src/main/resources/public/dtd/cjsl-1.16.dtd @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index 8f5d04e7aa..ff86af2b5c 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -200,6 +200,14 @@ def __init__(self, parent): data=lambda host: ",".join(host.data.tags), tip="The tags applied to the host.\n\n" "On a frame it is the name of the job.") + self.addColumn("Concurrent Slots", 50, id=25, + data=lambda host: ( + host.data.concurrent_slots_limit + if host.data.concurrent_slots_limit >= 0 else "-"), + sort=lambda host: host.data.concurrent_slots_limit, + tip="When >= 0 the host is slot-based: it runs only layers with a " + "slots_required field, up to this many concurrent slots " + "(usually 1 frame = 1 slot). '-' means a regular cores/memory host.") self.hostSearch = opencue.search.HostSearch() @@ -334,6 +342,7 @@ def contextMenuEvent(self, e): self.__menuActions.hosts().addAction(menu, "removeTags") self.__menuActions.hosts().addAction(menu, "renameTag") self.__menuActions.hosts().addAction(menu, "changeAllocation") + self.__menuActions.hosts().addAction(menu, "setConcurrentSlotsLimit") self.__menuActions.hosts().addAction(menu, "delete") self.__menuActions.hosts().addAction(menu, "rebootWhenIdle") self.__menuActions.hosts().addAction(menu, "setRepair") diff --git a/cuegui/cuegui/MenuActions.py b/cuegui/cuegui/MenuActions.py index 34fc063c00..ec55ee1e3d 100644 --- a/cuegui/cuegui/MenuActions.py +++ b/cuegui/cuegui/MenuActions.py @@ -1859,6 +1859,44 @@ def viewProc(self, rpcObjects=None): if hosts: self.app.view_procs.emit(hosts) + setConcurrentSlotsLimit_info = ["Update Slot Limit...", None, "configure"] + + def setConcurrentSlotsLimit(self, rpcObjects=None): + """Set the concurrent slots limit for selected hosts.""" + hosts = self._getOnlyHostObjects(rpcObjects) + if not hosts: + return + + # Current value from the first selected host (proto field, -1 when unset). + current = hosts[0].data.concurrent_slots_limit if len(hosts) == 1 else -1 + + title = "Set Concurrent Slots Limit" + body = "Enter the maximum concurrent slots for this host.\n" \ + "When >= 0 the host becomes slot-based: it only runs layers with a " \ + "slots_required field (regular cores/memory/gpu booking is disabled).\n\n" \ + "-1 = regular host (disable slot mode)\n" \ + " 0 = slot host, but reject all slot work\n" \ + " N = allow up to N concurrent slots" + + (value, choice) = QtWidgets.QInputDialog.getInt( + self._caller, + title, + body, + current, # current value + -1, # minimum value (-1 disables slot mode) + 10000, # maximum value + 1, # step + ) + + if choice: + for host in hosts: + self.cuebotCall( + host.setConcurrentSlotsLimit, + "Set Concurrent Slots Limit on %s Failed" % host.data.name, + int(value), + ) + self._update() + lock_info = ["Lock Host", None, "lock"] def lock(self, rpcObjects=None): diff --git a/proto/src/host.proto b/proto/src/host.proto index 416e473200..91db4516d1 100644 --- a/proto/src/host.proto +++ b/proto/src/host.proto @@ -96,6 +96,11 @@ service HostInterface { // Changes the host's [ThreadMode] rpc SetThreadMode(HostSetThreadModeRequest) returns (HostSetThreadModeResponse); + // Sets the host's concurrent slots limit (-1 disables slot mode; >= 0 makes the host + // slot-based, running only slot layers up to this many concurrent slots) + rpc SetConcurrentSlotsLimit(HostSetConcurrentSlotsLimitRequest) + returns (HostSetConcurrentSlotsLimitResponse); + // Unlocks the host for booking if the proc is in the Locked state. You cannot unlock a NimbyLocked proc. rpc Unlock(HostUnlockRequest) returns (HostUnlockResponse); } @@ -274,6 +279,8 @@ message Host { ThreadMode thread_mode = 27; float gpus = 28; float idle_gpus = 29; + // Max concurrent frames (slots) this host may run. -1 = regular host; >= 0 = slot-based. + int32 concurrent_slots_limit = 30; } message HostSearchCriteria { @@ -638,6 +645,13 @@ message HostSetThreadModeRequest { message HostSetThreadModeResponse {} // Empty +message HostSetConcurrentSlotsLimitRequest { + Host host = 1; + int32 concurrent_slots_limit = 2; +} + +message HostSetConcurrentSlotsLimitResponse {} // Empty + // Unlock message HostUnlockRequest { Host host = 1; diff --git a/proto/src/job.proto b/proto/src/job.proto index 3692b33ae6..cee5b70b90 100644 --- a/proto/src/job.proto +++ b/proto/src/job.proto @@ -126,6 +126,8 @@ service GroupInterface { // Set the group's Max Cores value rpc SetMaxCores(GroupSetMaxCoresRequest) returns (GroupSetMaxCoresResponse); + rpc SetMaxSlots(GroupSetMaxSlotsRequest) returns (GroupSetMaxSlotsResponse); + // Set the groups Min Cores values rpc SetMinCores(GroupSetMinCoresRequest) returns (GroupSetMinCoresResponse); @@ -264,6 +266,8 @@ service JobInterface { // Sets the maximum number of procs that can run on this job rpc SetMaxCores(JobSetMaxCoresRequest) returns (JobSetMaxCoresResponse); + rpc SetMaxSlots(JobSetMaxSlotsRequest) returns (JobSetMaxSlotsResponse); + // Sets the default maximum number of frame retries for the job. One // a frame has retried this many times it will automatically go // to the dead state. The default upper limit on this is 16 retries. @@ -607,6 +611,8 @@ message Group { float default_job_max_gpus = 13; float min_gpus = 14; float max_gpus = 15; + // Max concurrent slots for slot-based layers in this group (-1 unlimited, 0 reject-all). + int32 max_slots = 16; } message GroupSeq { @@ -651,6 +657,8 @@ message Job { float max_gpus = 22; string loki_url = 23; int32 eligible_time = 24; + // Max concurrent slots for slot-based layers in this job (-1 unlimited, 0 reject-all). + int32 max_slots = 25; } // Use to filter the job search. Please note that by searching for non-pending jobs, the output is limited to 200 jobs @@ -727,6 +735,8 @@ message Layer { int32 eligible_time = 23; int32 start_time = 24; int32 stop_time = 25; + // Concurrency slots each frame of this layer requires (0 = not slot-based). + int32 slots_required = 26; } message LayerSeq { @@ -1107,6 +1117,14 @@ message GroupSetMaxCoresRequest { message GroupSetMaxCoresResponse {} // Empty +// SetMaxSlots +message GroupSetMaxSlotsRequest { + Group group = 1; + int32 max_slots = 2; +} + +message GroupSetMaxSlotsResponse {} // Empty + // SetMinCores message GroupSetMinCoresRequest { Group group = 1; @@ -1460,6 +1478,14 @@ message JobSetMaxCoresRequest { message JobSetMaxCoresResponse {} // Empty +// SetMaxSlots +message JobSetMaxSlotsRequest { + Job job = 1; + int32 val = 2; +} + +message JobSetMaxSlotsResponse {} // Empty + // SetMaxGpus message JobSetMaxGpusRequest { Job job = 1; diff --git a/proto/src/subscription.proto b/proto/src/subscription.proto index 399cc1d913..c97d388256 100644 --- a/proto/src/subscription.proto +++ b/proto/src/subscription.proto @@ -29,6 +29,9 @@ service SubscriptionInterface { // Set the size of a subscription rpc SetSize(SubscriptionSetSizeRequest) returns (SubscriptionSetSizeResponse); + + // Set the max concurrent slots for slot-based layers on this subscription + rpc SetMaxSlots(SubscriptionSetMaxSlotsRequest) returns (SubscriptionSetMaxSlotsResponse); } @@ -44,6 +47,8 @@ message Subscription { int32 burst = 7; int32 reserved_cores = 8; int32 reserved_gpus = 9; + // Max concurrent slots for slot-based layers (-1 unlimited, 0 reject-all). + int32 max_slots = 10; } message SubscriptionSeq { @@ -93,3 +98,11 @@ message SubscriptionSetSizeRequest { } message SubscriptionSetSizeResponse {} // Empty + +// SetMaxSlots +message SubscriptionSetMaxSlotsRequest { + Subscription subscription = 1; + int32 max_slots = 2; +} + +message SubscriptionSetMaxSlotsResponse {} // Empty diff --git a/pycue/opencue/wrappers/group.py b/pycue/opencue/wrappers/group.py index a8c6dd3b4d..2bdf22e4e3 100644 --- a/pycue/opencue/wrappers/group.py +++ b/pycue/opencue/wrappers/group.py @@ -58,6 +58,15 @@ def setMaxCores(self, value): self.stub.SetMaxCores(job_pb2.GroupSetMaxCoresRequest(group=self.data, max_cores=value), timeout=Cuebot.Timeout) + def setMaxSlots(self, value): + """Sets the max concurrent slots for slot-based layers in the group. + + :type value: int + :param value: new max slots (-1 unlimited, 0 reject-all, N cap) + """ + self.stub.SetMaxSlots(job_pb2.GroupSetMaxSlotsRequest(group=self.data, max_slots=value), + timeout=Cuebot.Timeout) + def setMinCores(self, value): """Sets the minimum cores of everything the group. diff --git a/pycue/opencue/wrappers/host.py b/pycue/opencue/wrappers/host.py index 56233a764b..37bb6a464f 100644 --- a/pycue/opencue/wrappers/host.py +++ b/pycue/opencue/wrappers/host.py @@ -240,6 +240,18 @@ def setThreadMode(self, mode): self.stub.SetThreadMode(host_pb2.HostSetThreadModeRequest(host=self.data, mode=mode), timeout=Cuebot.Timeout) + def setConcurrentSlotsLimit(self, limit): + """Sets the host concurrent slots limit. + + :type limit: int + :param limit: -1 disables slot mode (regular host); >= 0 makes the host slot-based, + running only slot layers up to this many concurrent slots + """ + self.stub.SetConcurrentSlotsLimit( + host_pb2.HostSetConcurrentSlotsLimitRequest( + host=self.data, concurrent_slots_limit=limit), + timeout=Cuebot.Timeout) + @staticmethod def hasHostRebootedSince(host, start_time): """ diff --git a/pycue/opencue/wrappers/job.py b/pycue/opencue/wrappers/job.py index 85d918e3a1..42434b808d 100644 --- a/pycue/opencue/wrappers/job.py +++ b/pycue/opencue/wrappers/job.py @@ -145,6 +145,15 @@ def setMaxCores(self, maxCores): self.stub.SetMaxCores(job_pb2.JobSetMaxCoresRequest(job=self.data, val=maxCores), timeout=Cuebot.Timeout) + def setMaxSlots(self, maxSlots): + """Sets the max concurrent slots for slot-based layers in the job. + + :type maxSlots: int + :param maxSlots: new max slots (-1 unlimited, 0 reject-all, N cap) + """ + self.stub.SetMaxSlots(job_pb2.JobSetMaxSlotsRequest(job=self.data, val=maxSlots), + timeout=Cuebot.Timeout) + def setMinGpus(self, minGpus): """Sets the minimum procs value :type minGpus: int diff --git a/pycue/opencue/wrappers/subscription.py b/pycue/opencue/wrappers/subscription.py index b77e0b7f51..ce63b5d4a8 100644 --- a/pycue/opencue/wrappers/subscription.py +++ b/pycue/opencue/wrappers/subscription.py @@ -77,6 +77,18 @@ def setBurst(self, burst): subscription_pb2.SubscriptionSetBurstRequest(subscription=self.data, burst=burst), timeout=Cuebot.Timeout) + def setMaxSlots(self, maxSlots): + """Sets the max concurrent slots for slot-based layers on this subscription. + + :type maxSlots: int + :param maxSlots: new max slots (-1 unlimited, 0 reject-all, N cap) + """ + assert (isinstance(maxSlots, int)), "maxSlots is not expected type: int" + self.stub.SetMaxSlots( + subscription_pb2.SubscriptionSetMaxSlotsRequest( + subscription=self.data, max_slots=maxSlots), + timeout=Cuebot.Timeout) + def delete(self): """Deletes a subscription.""" self.stub.Delete( diff --git a/pyoutline/outline/backend/cue.py b/pyoutline/outline/backend/cue.py index 3aa5448563..2ccd5b47f3 100644 --- a/pyoutline/outline/backend/cue.py +++ b/pyoutline/outline/backend/cue.py @@ -384,6 +384,13 @@ def _serialize(launcher, use_pycuerun): else: _warning_spec_version(spec_version, "timeout_llu") + if layer.get_arg("slots_required"): + if spec_version >= Version("1.16"): + sub_element( + spec_layer, "slots_required", "%s" % (layer.get_arg("slots_required"))) + else: + _warning_spec_version(spec_version, "slots_required") + if os.environ.get("OL_TAG_OVERRIDE", False): sub_element(spec_layer, "tags", scrub_tags(os.environ["OL_TAG_OVERRIDE"])) diff --git a/pyoutline/outline/layer.py b/pyoutline/outline/layer.py index 856850ba7c..b16f28ccd5 100644 --- a/pyoutline/outline/layer.py +++ b/pyoutline/outline/layer.py @@ -110,6 +110,8 @@ class _LayerArgs(TypedDict, total=False): # timeout_llu: Timeout for long last update in seconds # before considering a frame hung timeout_llu: int + # slots_required: concurrency slots required per frame (0 = not slot-based) + slots_required: int type: outline.constants.LayerType # The layer type (Render, Util, Post) diff --git a/pyoutline/outline/outline.cfg b/pyoutline/outline/outline.cfg index 55e61d2821..0958a5bd1f 100644 --- a/pyoutline/outline/outline.cfg +++ b/pyoutline/outline/outline.cfg @@ -5,7 +5,7 @@ wrapper_dir = %(home)s/wrappers user_dir = bin_dir = %(home)s/bin backend = cue -spec_version = 1.15 +spec_version = 1.16 facility = local domain = example.com maxretries = 2 From 67d1f1c8e5a3f2f914991bce21e3d92e7f37a4da Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Fri, 3 Jul 2026 15:54:54 -0700 Subject: [PATCH 16/20] [scheduler/docs] Slot gate tests + slot-based-booking concept doc - placement.rs: gate tests covering the full pairing matrix (slot host x slot layer within/over cap, both pairing-mismatch rejections, core/mem floor ignored on slot hosts, constant E-PVM score for slot placements). - docs: slot-based-booking.md covering the two axes, strict pairing, the -1/0/N limit convention, and how slot accounting derives from proc. Slot limit-exceeded observability comes free via the existing accounting_limit_exceeded_total{table} counter (subscription_slots/folder_slots/ job_slots labels). 188 scheduler tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/_docs/concepts/slot-based-booking.md | 95 +++++++++++++++++++ rust/crates/scheduler/src/accounting/store.rs | 1 + .../scheduler/src/pipeline/placement.rs | 54 +++++++++++ 3 files changed, 150 insertions(+) create mode 100644 docs/_docs/concepts/slot-based-booking.md diff --git a/docs/_docs/concepts/slot-based-booking.md b/docs/_docs/concepts/slot-based-booking.md new file mode 100644 index 0000000000..e30c3087ec --- /dev/null +++ b/docs/_docs/concepts/slot-based-booking.md @@ -0,0 +1,95 @@ +--- +title: "Slot-Based Booking" +nav_order: 14 +parent: Concepts +layout: default +linkTitle: "Slot-Based Booking" +date: 2026-07-03 +description: > + Booking frames by concurrency slots instead of cores and memory +--- + +# Slot-Based Booking + +Slot-based booking is an alternative dispatch mode for pipelines whose frames are +limited not by CPU or memory but by some other shared resource — storage bandwidth, +network throughput, a licensed external service, etc. In these pipelines what matters +is bounding *concurrency*, not resource consumption. + +A slot-based frame ignores cores and memory entirely: it reserves **0 cores and 0 +memory** and runs unpinned on the host. The only thing that limits it is a **slot +budget**. + +> **Note:** Slot-based booking is implemented in the standalone Rust scheduler. Cuebot +> does not make slot-based booking decisions; it stores the configuration and publishes +> slot release/limit deltas to the scheduler. + +## The two slot axes + +Slot booking is governed by two independent limits: + +1. **Per-host cap** — `host.concurrent_slots_limit`. A host with this set is a + *slot host*: it runs **only** slot-based layers, up to this many concurrent slots. +2. **Per-hierarchy max** — a `max_slots` limit at the **subscription**, **folder**, and + **job** levels, enforced by the scheduler's accounting store, parallel to (and + independent of) the cores/GPUs limits. + +Both must be satisfied for a slot frame to book. + +## Strict pairing + +Slot hosts and slot layers are strictly paired: + +| Host \ Layer | Regular layer | Slot layer (`slots_required > 0`) | +|---|---|---| +| **Regular host** | books by cores/memory | **rejected** | +| **Slot host** (`concurrent_slots_limit ≥ 0`) | **rejected** | books by slots | + +A slot host never runs a regular layer, and a slot layer never runs on a regular host. + +## Limit values + +Every `max_slots` limit uses the convention: + +- **`-1`** — unlimited (the migration default; a slot layer is bounded only by host + capacity until an admin sets a limit). +- **`0`** — reject all slot work at this level. +- **`N`** — cap at N concurrent slots. + +An unseeded limit is treated as `0` (reject-all) — the scheduler **fails closed** on the +slot axis: a seeding bug manifests as "slot work won't book," never as overrunning a hard +cap. Regular (cores/memory) layers are unaffected by `max_slots`, and slot layers are +unaffected by the cores/GPUs limits. + +## Making a layer slot-based + +Set `slots_required` on the layer in the job spec (spec version **1.16**+). With +PyOutline: + +```python +layer = outline.modules.shell.Shell( + "bandwidth_bound_layer", + command=["my_command"], + slots_required=1, # each frame consumes one slot +) +``` + +A frame may require more than one slot (`slots_required > 1`) — a "heavy" frame counts +N against the host cap and all three hierarchy limits. Slot-based layers are forced +non-threadable. + +## Configuring the limits + +- **Host slot cap** — set a host's concurrent slots limit from CueCommander, or via + PyCue: `host.setConcurrentSlotsLimit(8)` (`-1` disables slot mode). +- **Subscription / folder / job max slots** — set from the GUI or via PyCue + (`subscription.setMaxSlots(...)`, `group.setMaxSlots(...)`, `job.setMaxSlots(...)`), + mirroring how `max_cores` is managed. + +## How it accounts + +`proc.int_slots_reserved` is the single source of truth for slot usage. Per-host slot +counts and per subscription/folder/job counts both derive from `SUM(proc.int_slots_reserved)`, +so RQD does not need to report slots. The scheduler's periodic recompute reconciles the +slot counters from that sum, the same way it reconciles cores/GPUs — a dropped release +leaves a counter reading high (under-book), healed by the next recompute. diff --git a/rust/crates/scheduler/src/accounting/store.rs b/rust/crates/scheduler/src/accounting/store.rs index 48a4a1ffcf..0dec48ca3d 100644 --- a/rust/crates/scheduler/src/accounting/store.rs +++ b/rust/crates/scheduler/src/accounting/store.rs @@ -671,6 +671,7 @@ impl Store { } /// Live booked slots for a job (slot-axis observability). 0 if unseen. + #[cfg(test)] pub fn job_slots_in_use(&self, job_id: Uuid) -> i64 { self.lock().job.get(&job_id).map_or(0, |c| c.slots) } diff --git a/rust/crates/scheduler/src/pipeline/placement.rs b/rust/crates/scheduler/src/pipeline/placement.rs index ab7a10a5b0..acbed56c27 100644 --- a/rust/crates/scheduler/src/pipeline/placement.rs +++ b/rust/crates/scheduler/src/pipeline/placement.rs @@ -757,6 +757,60 @@ mod scoring_tests { assert_eq!(saturation_gate(&h, &l), None); } + // ── Slot-based pairing gate ────────────────────────────────────────────── + + fn slot_host(limit: u32, running: u32) -> Host { + let mut h = host(4, 4, 0, 0); + h.concurrent_slots_limit = Some(limit); + h.running_slots_count = running; + h + } + + fn slot_layer(slots: u32) -> LayerProfile { + let mut l = layer(4, 4, 0, 0); + l.slots_required = slots; + l + } + + #[test] + fn slot_host_accepts_slot_layer_within_cap() { + // 2 running + 2 requested <= 8 cap. + assert_eq!(saturation_gate(&slot_host(8, 2), &slot_layer(2)), Some(0.0)); + } + + #[test] + fn slot_host_rejects_slot_layer_over_cap() { + // 7 running + 2 requested > 8 cap. + assert_eq!(saturation_gate(&slot_host(8, 7), &slot_layer(2)), None); + } + + #[test] + fn slot_host_rejects_regular_layer() { + // Strict pairing: a slot host never runs a non-slot layer. + assert_eq!(saturation_gate(&slot_host(8, 0), &layer(4, 4, 0, 0)), None); + } + + #[test] + fn regular_host_rejects_slot_layer() { + // Strict pairing: a slot layer never runs on a non-slot host. + assert_eq!(saturation_gate(&host(4, 4, 0, 0), &slot_layer(1)), None); + } + + #[test] + fn slot_gate_ignores_core_and_memory_floor() { + // Slot host with zero idle cores/memory still accepts a slot layer. + let mut h = slot_host(8, 0); + h.idle_cores = CoreSize(0); + h.idle_memory = ByteSize::gb(0); + assert_eq!(saturation_gate(&h, &slot_layer(4)), Some(0.0)); + } + + #[test] + fn epvm_gate_scores_slot_placement_constant() { + // Slot placements tie at a constant score (no core stranding to minimize). + assert_eq!(epvm_gate(&slot_host(8, 0), &slot_layer(1)), Some(0.0)); + } + #[test] fn epvm_gate_returns_score_when_valid() { let h = host(64, 64, 0, 0); From 66c5503e8f2968fbb9dd31b6af22fceea4cc6ed5 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Tue, 14 Jul 2026 11:26:11 -0700 Subject: [PATCH 17/20] Update docs/_docs/concepts/slot-based-booking.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/_docs/concepts/slot-based-booking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_docs/concepts/slot-based-booking.md b/docs/_docs/concepts/slot-based-booking.md index e30c3087ec..365a27fa21 100644 --- a/docs/_docs/concepts/slot-based-booking.md +++ b/docs/_docs/concepts/slot-based-booking.md @@ -89,7 +89,7 @@ non-threadable. ## How it accounts `proc.int_slots_reserved` is the single source of truth for slot usage. Per-host slot -counts and per subscription/folder/job counts both derive from `SUM(proc.int_slots_reserved)`, +counts and per-subscription/folder/job counts both derive from `SUM(proc.int_slots_reserved)`, so RQD does not need to report slots. The scheduler's periodic recompute reconciles the slot counters from that sum, the same way it reconciles cores/GPUs — a dropped release leaves a counter reading high (under-book), healed by the next recompute. From 5188f4562fa33cadf83a7406dfe5fd966873b532 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Tue, 14 Jul 2026 14:39:13 -0700 Subject: [PATCH 18/20] Fix review findings --- .../imageworks/spcue/dao/postgres/ProcDaoJdbc.java | 3 ++- .../spcue/dao/postgres/WhiteboardDaoJdbc.java | 6 ++++++ .../java/com/imageworks/spcue/service/JobSpec.java | 5 +++-- cuegui/cuegui/HostMonitorTree.py | 9 ++++++++- proto/src/host.proto | 4 ++++ rust/crates/rqd/src/frame/manager.rs | 5 +++-- rust/crates/scheduler/src/dao/proc_dao.rs | 2 +- .../scheduler/src/pipeline/dispatcher/actor.rs | 14 ++++++++++++-- 8 files changed, 39 insertions(+), 9 deletions(-) diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index c3dccb6733..9b2a936ec2 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -102,7 +102,7 @@ public boolean verifyRunningProc(String procId, String frameId) { + "WHERE " + "pk_proc=? " + "RETURNING int_cores_reserved, int_mem_reserved, " - + "int_gpus_reserved, int_gpu_mem_reserved"; + + "int_gpus_reserved, int_gpu_mem_reserved, int_slots_reserved"; // spotless:on public boolean deleteVirtualProc(VirtualProc proc) { @@ -117,6 +117,7 @@ public boolean deleteVirtualProc(VirtualProc proc) { proc.memoryReserved = ((Number) result.get("int_mem_reserved")).longValue(); proc.gpusReserved = ((Number) result.get("int_gpus_reserved")).intValue(); proc.gpuMemoryReserved = ((Number) result.get("int_gpu_mem_reserved")).longValue(); + proc.slotsReserved = ((Number) result.get("int_slots_reserved")).intValue(); } catch (EmptyResultDataAccessException e) { logger.info("failed to delete " + proc + " , proc does not exist."); return false; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java index 5d6c90fc98..4a118bd86b 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/WhiteboardDaoJdbc.java @@ -999,6 +999,7 @@ public static Host.Builder mapHostBuilder(ResultSet rs) throws SQLException { builder.setHasComment(rs.getBoolean("b_comment")); builder.setThreadMode(ThreadMode.values()[rs.getInt("int_thread_mode")]); builder.setConcurrentSlotsLimit(rs.getInt("int_concurrent_slots_limit")); + builder.setIdleSlots(rs.getInt("int_slots_idle")); builder.setOs(SqlUtil.getString(rs, "str_os")); String tags = SqlUtil.getString(rs, "str_tags"); @@ -2226,6 +2227,11 @@ public Show mapRow(ResultSet rs, int rowNum) throws SQLException { + "host.b_comment," + "host.int_thread_mode," + "host.int_concurrent_slots_limit," + + "CASE WHEN host.int_concurrent_slots_limit > 0 THEN " + + "(host.int_concurrent_slots_limit - COALESCE(" + + "(SELECT SUM(proc.int_slots_reserved) FROM proc " + + "WHERE proc.pk_host = host.pk_host), 0)) " + + "ELSE -1 END AS int_slots_idle," + "host_stat.str_os," + "host_stat.int_mem_total," + "host_stat.int_mem_free," diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index bed7fa9033..e3ab8864cb 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -444,8 +444,9 @@ private void handleLayerTags(BuildableJob buildableJob, Element jobTag) { layer.timeout_llu = Integer.parseInt(layerTag.getChildTextTrim("timeout_llu")); } - if (layerTag.getChildTextTrim("slots_required") != null) { - layer.slotsRequired = Integer.parseInt(layerTag.getChildTextTrim("slots_required")); + String slotsRequired = layerTag.getChildTextTrim("slots_required"); + if (slotsRequired != null && !slotsRequired.isEmpty()) { + layer.slotsRequired = Integer.parseInt(slotsRequired); } /* diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index ff86af2b5c..2c0dbe6963 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -200,7 +200,7 @@ def __init__(self, parent): data=lambda host: ",".join(host.data.tags), tip="The tags applied to the host.\n\n" "On a frame it is the name of the job.") - self.addColumn("Concurrent Slots", 50, id=25, + self.addColumn("Slots", 50, id=25, data=lambda host: ( host.data.concurrent_slots_limit if host.data.concurrent_slots_limit >= 0 else "-"), @@ -208,6 +208,13 @@ def __init__(self, parent): tip="When >= 0 the host is slot-based: it runs only layers with a " "slots_required field, up to this many concurrent slots " "(usually 1 frame = 1 slot). '-' means a regular cores/memory host.") + self.addColumn("Slots Idle", 60, id=26, + data=lambda host: ( + host.data.idle_slots + if host.data.concurrent_slots_limit > 0 else "-"), + sort=lambda host: host.data.idle_slots, + tip="The number of slots that are not reserved by running frames.\n" + "'-' means a regular cores/memory host.") self.hostSearch = opencue.search.HostSearch() diff --git a/proto/src/host.proto b/proto/src/host.proto index 91db4516d1..2c85abb903 100644 --- a/proto/src/host.proto +++ b/proto/src/host.proto @@ -281,6 +281,10 @@ message Host { float idle_gpus = 29; // Max concurrent frames (slots) this host may run. -1 = regular host; >= 0 = slot-based. int32 concurrent_slots_limit = 30; + // Number of slots not currently reserved (concurrent_slots_limit minus the slots + // reserved by running frames). Only computed for slot-based hosts + // (concurrent_slots_limit > 0); -1 otherwise. + int32 idle_slots = 31; } message HostSearchCriteria { diff --git a/rust/crates/rqd/src/frame/manager.rs b/rust/crates/rqd/src/frame/manager.rs index 133a1f22f5..6aa5853074 100644 --- a/rust/crates/rqd/src/frame/manager.rs +++ b/rust/crates/rqd/src/frame/manager.rs @@ -318,8 +318,9 @@ impl FrameManager { } // Invalid number of cores. Slot-based frames legitimately reserve 0 // cores (they run unpinned, capped by concurrency slots instead), so - // only reject non-positive cores for regular frames. - if run_frame.num_cores <= 0 && run_frame.slots_required <= 0 { + // allow exactly 0 cores for them; negative cores are always invalid and + // regular frames must reserve a positive number of cores. + if run_frame.num_cores < 0 || (run_frame.num_cores == 0 && run_frame.slots_required <= 0) { Err(FrameManagerError::InvalidArgument( "Not launching, num_cores must be positive".to_string(), ))? diff --git a/rust/crates/scheduler/src/dao/proc_dao.rs b/rust/crates/scheduler/src/dao/proc_dao.rs index 33af460d06..0acdd85f1c 100644 --- a/rust/crates/scheduler/src/dao/proc_dao.rs +++ b/rust/crates/scheduler/src/dao/proc_dao.rs @@ -170,7 +170,7 @@ impl ProcDao { .bind((virtual_proc.gpu_memory_reserved.0 / KB) as i64) .bind(0) .bind(virtual_proc.is_local_dispatch) - .bind(virtual_proc.slots_required as i64) + .bind(virtual_proc.slots_required as i32) .execute(&mut **transaction) .await .map_err(|err| ProcDaoError::DbFailure { diff --git a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs index 950884a4d0..3f05c0aafc 100644 --- a/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs +++ b/rust/crates/scheduler/src/pipeline/dispatcher/actor.rs @@ -1041,9 +1041,19 @@ impl RqdDispatcherService { // Slot-based hosts ignore cores/memory entirely: a slot frame reserves 0 // cores and 0 memory and only consumes the host's concurrency slots. The - // per-host slot cap was already enforced by the placement gate at - // check-out; here we just record the reservation. + // placement gate enforced the per-host slot cap at check-out, but several + // slot frames booked onto this host in the same dispatch pass accumulate + // `running_slots_count`, so re-check the cap here and stop dispatching to + // the host once another frame would exceed it. if host.is_slot_host() { + if let Some(limit) = host.concurrent_slots_limit { + if host.running_slots_count + frame.slots_required > limit { + return Err(VirtualProcError::HostResourcesExtinguished(format!( + "slot host {} at capacity ({} + {} > {})", + host.name, host.running_slots_count, frame.slots_required, limit + ))); + } + } host.running_slots_count += frame.slots_required; host.last_updated = Utc::now(); return Ok(( From 995199dea0b0ef335ec38b875150dc138b7d18be Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Tue, 11 Aug 2026 08:06:46 -0700 Subject: [PATCH 19/20] [cuebot] Implement slot based dispatching on Cuebot --- VERSION.in | 2 +- .../com/imageworks/spcue/DispatchFrame.java | 5 + .../com/imageworks/spcue/DispatchHost.java | 21 + .../com/imageworks/spcue/VirtualProc.java | 39 ++ .../imageworks/spcue/dao/DispatcherDao.java | 34 ++ .../spcue/dao/postgres/DispatcherDaoJdbc.java | 95 ++++- .../spcue/dao/postgres/HostDaoJdbc.java | 8 + .../spcue/dao/postgres/ProcDaoJdbc.java | 8 +- .../spcue/dao/postgres/SlotDispatchQuery.java | 352 ++++++++++++++++ .../spcue/dispatcher/DispatchSupport.java | 30 ++ .../dispatcher/DispatchSupportService.java | 24 +- .../dispatcher/FrameCompleteHandler.java | 37 +- .../spcue/dispatcher/HostReportHandler.java | 71 ++++ .../spcue/dispatcher/RedirectManager.java | 41 ++ .../spcue/dispatcher/SlotDispatcher.java | 339 ++++++++++++++++ .../commands/DispatchBookHostSlots.java | 52 +++ .../spcue/service/BookingManagerService.java | 15 + .../com/imageworks/spcue/service/JobSpec.java | 12 + .../V50__Enforce_slot_proc_booking.sql | 31 ++ .../spring/applicationContext-service.xml | 8 + .../test/dispatcher/SlotDispatcherTests.java | 377 ++++++++++++++++++ .../test/service/AccountingNotifierTests.java | 18 +- .../src/test/resources/conf/dtd/cjsl-1.16.dtd | 106 +++++ .../conf/jobspec/jobspec_slot_test.xml | 69 ++++ docs/_docs/concepts/slot-based-booking.md | 22 +- rust/crates/rqd/src/system/machine.rs | 18 +- rust/crates/scheduler/src/dao/host_dao.rs | 2 +- 27 files changed, 1808 insertions(+), 28 deletions(-) create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SlotDispatchQuery.java create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dispatcher/SlotDispatcher.java create mode 100644 cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostSlots.java create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V50__Enforce_slot_proc_booking.sql create mode 100644 cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/SlotDispatcherTests.java create mode 100644 cuebot/src/test/resources/conf/dtd/cjsl-1.16.dtd create mode 100644 cuebot/src/test/resources/conf/jobspec/jobspec_slot_test.xml diff --git a/VERSION.in b/VERSION.in index e0250ea3c7..57432cbd95 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -1.28 +1.30 diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java index b73cd75c3c..5e0498d0b9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java @@ -44,6 +44,11 @@ public class DispatchFrame extends FrameEntity implements FrameInterface { public int maxGpus; public long minGpuMemory; + // Concurrency slots each frame of this layer requires (slot-based scheduling). + // 0 = not slot-based. Only populated by the slot dispatch queries; the generic + // dispatch queries never return slot-based frames. + public int slotsRequired; + // A comma separated list of services public String services; diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchHost.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchHost.java index a4bee8e755..b09d4de0d3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DispatchHost.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DispatchHost.java @@ -43,6 +43,19 @@ public class DispatchHost extends Entity // Basically an 0 = auto, 1 = all. public int threadMode; + /** + * Max concurrent frames (slots) this host may run. -1 marks a regular (cores/memory) host; >= 0 + * marks a slot-based host that only runs slot-based layers, capped at this many concurrent + * slots. + */ + public int concurrentSlotsLimit = -1; + + /** + * Number of slots not currently reserved on this host (concurrentSlotsLimit minus + * SUM(proc.int_slots_reserved)). Only meaningful for slot-based hosts; -1 otherwise. + */ + public int idleSlots = -1; + public long memory; public long idleMemory; public long gpuMemory; @@ -69,6 +82,14 @@ public String getHostId() { return id; } + /** + * True when this host is slot-based: it only runs slot-based layers, capped by + * concurrentSlotsLimit, and ignores cores/memory for booking decisions. + */ + public boolean isSlotHost() { + return concurrentSlotsLimit >= 0; + } + public String getAllocationId() { return allocationId; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java index 1f4739ff4e..27122c5a1c 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java @@ -210,6 +210,45 @@ && containsSelfishService(frame.services.split(","), selfishServices)) { return proc; } + /** + * Build a proc for a slot-based booking. Slot procs reserve 0 cores, 0 memory and 0 gpus - the + * only resource they consume is the frame's slot requirement, which counts against the host's + * concurrent slots limit and the subscription/folder/job max_slots caps. + * + * @param host a slot-based host (isSlotHost() == true) + * @param frame a slot-based frame (slotsRequired > 0) + * @return + */ + public static final VirtualProc buildSlotProc(DispatchHost host, DispatchFrame frame) { + if (frame.slotsRequired <= 0) { + throw new EntityException( + "Cannot build a slot proc for a frame that requires no slots."); + } + + VirtualProc proc = new VirtualProc(); + proc.allocationId = host.getAllocationId(); + proc.hostId = host.getHostId(); + proc.frameId = null; + proc.layerId = frame.getLayerId(); + proc.jobId = frame.getJobId(); + proc.showId = frame.getShowId(); + proc.facilityId = frame.getFacilityId(); + proc.os = frame.os; + + proc.hostName = host.getName(); + proc.unbooked = false; + proc.isLocalDispatch = false; + proc.canHandleNegativeCoresRequest = false; + + proc.coresReserved = 0; + proc.memoryReserved = 0; + proc.gpusReserved = 0; + proc.gpuMemoryReserved = 0; + proc.slotsReserved = frame.slotsRequired; + + return proc; + } + private static final boolean containsSelfishService(String[] frameServices, String[] selfishServices) { for (String frameService : frameServices) { diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java b/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java index b751a23cad..599a7620de 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/DispatcherDao.java @@ -153,6 +153,40 @@ public interface DispatcherDao { */ List findNextDispatchFrames(LayerInterface layer, DispatchHost host, int limit); + /** + * Return a list of jobs with pending slot-based work that could book on the specified + * slot-based host. Jobs over their job/folder/subscription max_slots caps are excluded. + * + * @param host a slot-based host (concurrentSlotsLimit >= 0) + * @param numJobs + * @return a set of unique job ids ordered by priority + */ + Set findSlotDispatchJobs(DispatchHost host, int numJobs); + + /** + * Return the next slot-based frames in the given job bookable on the specified slot-based host. + * Only frames of slot-based layers whose slot requirement fits the host's idle slots and whose + * job/folder/subscription max_slots caps allow another booking are returned. + * + * @param job + * @param host a slot-based host (concurrentSlotsLimit >= 0) + * @param limit + * @return + */ + List findNextSlotDispatchFrames(JobInterface job, DispatchHost host, int limit); + + /** + * Return how many more slots the given job may book on the given host's allocation: the + * smallest remaining allowance across the job, folder and subscription max_slots caps. + * Integer.MAX_VALUE when all three caps are unlimited (-1); 0 when the job's show has no + * subscription to the host's allocation. + * + * @param job + * @param host + * @return + */ + int getSlotCapacityRemaining(JobInterface job, DispatchHost host); + /** * Return Scheduling Mode selected * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java index a7f9285f9a..532d9aef78 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/DispatcherDaoJdbc.java @@ -363,11 +363,13 @@ public List findNextDispatchFrames(JobInterface job, VirtualProc long lastTime = System.currentTimeMillis(); List frames; if (proc.isLocalDispatch) { - frames = getJdbcTemplate().query(FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC, + frames = getJdbcTemplate().query( + SlotDispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC_EXCLUDE_SLOT, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, proc.memoryReserved, proc.gpuMemoryReserved, job.getJobId(), limit); } else { - frames = getJdbcTemplate().query(FIND_DISPATCH_FRAME_BY_JOB_AND_PROC, + frames = getJdbcTemplate().query( + SlotDispatchQuery.FIND_DISPATCH_FRAME_BY_JOB_AND_PROC_EXCLUDE_SLOT, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, proc.coresReserved, proc.memoryReserved, proc.gpusReserved, (proc.gpuMemoryReserved > 0) ? 1 : 0, proc.gpuMemoryReserved, job.getJobId(), proc.hostName, job.getJobId(), limit); @@ -386,12 +388,14 @@ public List findNextDispatchFrames(JobInterface job, DispatchHost List frames; if (host.isLocalDispatch) { - frames = getJdbcTemplate().query(FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST, + frames = getJdbcTemplate().query( + SlotDispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST_EXCLUDE_SLOT, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, host.idleMemory, host.idleGpuMemory, job.getJobId(), limit); } else { - frames = getJdbcTemplate().query(FIND_DISPATCH_FRAME_BY_JOB_AND_HOST, + frames = getJdbcTemplate().query( + SlotDispatchQuery.FIND_DISPATCH_FRAME_BY_JOB_AND_HOST_EXCLUDE_SLOT, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, host.idleCores, host.idleMemory, threadMode(host.threadMode), host.idleGpus, (host.idleGpuMemory > 0) ? 1 : 0, host.idleGpuMemory, job.getJobId(), host.getName(), job.getJobId(), limit); @@ -409,11 +413,13 @@ public List findNextDispatchFrames(LayerInterface layer, VirtualP List frames; if (proc.isLocalDispatch) { - frames = getJdbcTemplate().query(FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC, + frames = getJdbcTemplate().query( + SlotDispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC_EXCLUDE_SLOT, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, proc.memoryReserved, proc.gpuMemoryReserved, layer.getLayerId(), limit); } else { - frames = getJdbcTemplate().query(FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC, + frames = getJdbcTemplate().query( + SlotDispatchQuery.FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC_EXCLUDE_SLOT, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, proc.coresReserved, proc.memoryReserved, proc.gpusReserved, proc.gpuMemoryReserved, layer.getLayerId(), layer.getLayerId(), proc.hostName, limit); @@ -432,12 +438,14 @@ public List findNextDispatchFrames(LayerInterface layer, Dispatch List frames; if (host.isLocalDispatch) { - frames = getJdbcTemplate().query(FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST, + frames = getJdbcTemplate().query( + SlotDispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST_EXCLUDE_SLOT, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, host.idleMemory, host.idleGpuMemory, layer.getLayerId(), limit); } else { - frames = getJdbcTemplate().query(FIND_DISPATCH_FRAME_BY_LAYER_AND_HOST, + frames = getJdbcTemplate().query( + SlotDispatchQuery.FIND_DISPATCH_FRAME_BY_LAYER_AND_HOST_EXCLUDE_SLOT, FrameDaoJdbc.DISPATCH_FRAME_MAPPER, host.idleCores, host.idleMemory, threadMode(host.threadMode), host.idleGpus, host.idleGpuMemory, layer.getLayerId(), layer.getLayerId(), host.getName(), limit); @@ -449,6 +457,77 @@ public List findNextDispatchFrames(LayerInterface layer, Dispatch return frames; } + /** + * Maps rows of the slot dispatch frame queries; delegates to the generic mapper and adds the + * slot requirement column. + */ + public static final RowMapper SLOT_DISPATCH_FRAME_MAPPER = + new RowMapper() { + public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { + DispatchFrame frame = FrameDaoJdbc.DISPATCH_FRAME_MAPPER.mapRow(rs, rowNum); + frame.slotsRequired = rs.getInt("int_slots_required"); + return frame; + } + }; + + @Override + public Set findSlotDispatchJobs(DispatchHost host, int numJobs) { + long lastTime = System.currentTimeMillis(); + LinkedHashSet result = new LinkedHashSet(); + + result.addAll(getJdbcTemplate().query(new PreparedStatementCreator() { + @Override + public PreparedStatement createPreparedStatement(Connection conn) throws SQLException { + String query = handleInClause("str_os", SlotDispatchQuery.FIND_SLOT_DISPATCH_JOBS, + host.getOs().length); + PreparedStatement findJobsStmt = conn.prepareStatement(query); + int index = 1; + findJobsStmt.setString(index++, host.getFacilityId()); + for (String item : host.getOs()) { + findJobsStmt.setString(index++, item); + } + findJobsStmt.setInt(index++, host.idleSlots); + findJobsStmt.setString(index++, host.getName()); + findJobsStmt.setString(index++, host.getAllocationId()); + findJobsStmt.setInt(index++, numJobs); + return findJobsStmt; + } + }, PKJOB_MAPPER)); + + prometheusMetrics.setBookingDurationMetric("findSlotDispatchJobs query", + System.currentTimeMillis() - lastTime); + return result; + } + + @Override + public List findNextSlotDispatchFrames(JobInterface job, DispatchHost host, + int limit) { + long lastTime = System.currentTimeMillis(); + List frames = + getJdbcTemplate().query(SlotDispatchQuery.FIND_SLOT_DISPATCH_FRAMES_BY_JOB_AND_HOST, + SLOT_DISPATCH_FRAME_MAPPER, host.idleSlots, job.getJobId(), host.getName(), + job.getJobId(), host.getAllocationId(), limit); + prometheusMetrics.setBookingDurationMetric("findNextSlotDispatchFrames query", + System.currentTimeMillis() - lastTime); + return frames; + } + + @Override + public int getSlotCapacityRemaining(JobInterface job, DispatchHost host) { + try { + Integer capacity = + getJdbcTemplate().queryForObject(SlotDispatchQuery.GET_SLOT_CAPACITY_REMAINING, + Integer.class, host.getAllocationId(), job.getJobId()); + if (capacity == null) { + return Integer.MAX_VALUE; + } + return Math.max(capacity, 0); + } catch (org.springframework.dao.EmptyResultDataAccessException e) { + // No subscription between the job's show and the host's allocation. + return 0; + } + } + @Override public DispatchFrame findNextDispatchFrame(JobInterface job, VirtualProc proc) { return findNextDispatchFrames(job, proc, 1).get(0); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java index d74eae703a..db0c4ec339 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java @@ -231,6 +231,8 @@ public DispatchHost mapRow(ResultSet rs, int rowNum) throws SQLException { host.idleGpus = rs.getInt("int_gpus_idle"); host.isNimby = rs.getBoolean("b_nimby"); host.threadMode = rs.getInt("int_thread_mode"); + host.concurrentSlotsLimit = rs.getInt("int_concurrent_slots_limit"); + host.idleSlots = rs.getInt("int_slots_idle"); host.tags = rs.getString("str_tags"); host.setOs(rs.getString("str_os")); host.hardwareState = HardwareState.valueOf(rs.getString("str_state")); @@ -256,6 +258,12 @@ public DispatchHost mapRow(ResultSet rs, int rowNum) throws SQLException { + " host.int_gpu_mem_idle, " + " host.b_nimby, " + " host.int_thread_mode, " + + " host.int_concurrent_slots_limit, " + + " CASE WHEN host.int_concurrent_slots_limit >= 0 THEN " + + " (host.int_concurrent_slots_limit - COALESCE(" + + " (SELECT SUM(proc.int_slots_reserved) FROM proc " + + " WHERE proc.pk_host = host.pk_host), 0)) " + + " ELSE -1 END AS int_slots_idle, " + " host.str_tags, " + " host_stat.str_os, " + " host_stat.str_state, " diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java index 9b2a936ec2..1b39a58e64 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java @@ -146,9 +146,10 @@ public boolean deleteVirtualProc(VirtualProc proc) { + "int_gpu_mem_reserved, " + "int_gpu_mem_pre_reserved, " + "int_gpu_mem_used, " - + "b_local " + + "b_local, " + + "int_slots_reserved " + ") " - + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "; + + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "; // spotless:on public void insertVirtualProc(VirtualProc proc) { @@ -163,7 +164,8 @@ public void insertVirtualProc(VirtualProc proc) { proc.getHostId(), proc.getShowId(), proc.getLayerId(), proc.getJobId(), proc.getFrameId(), proc.coresReserved, proc.memoryReserved, proc.memoryReserved, memReservedMin, proc.gpusReserved, proc.gpuMemoryReserved, - proc.gpuMemoryReserved, memGpuReservedMin, proc.isLocalDispatch); + proc.gpuMemoryReserved, memGpuReservedMin, proc.isLocalDispatch, + proc.slotsReserved); // Update all of the resource counts procCreated(proc); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SlotDispatchQuery.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SlotDispatchQuery.java new file mode 100644 index 0000000000..b2fefe1100 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/SlotDispatchQuery.java @@ -0,0 +1,352 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dao.postgres; + +/** + * SQL for slot-based dispatching. + * + * Slot-based booking is an alternative dispatch mode for hosts marked with a concurrent slots limit + * ({@code host.int_concurrent_slots_limit >= 0}). Slot hosts run only slot-based layers + * ({@code layer.int_slots_required > 0}); slot layers run only on slot hosts. A slot booking + * reserves 0 cores and 0 memory - the only constraints are the per-host slot cap and the + * {@code int_max_slots} limits at the subscription, folder and job levels (-1 = unlimited, 0 = + * reject-all, N = cap at N concurrent slots). {@code proc.int_slots_reserved} is the single source + * of truth for slot usage at every level. + * + * These queries are intentionally kept separate from {@link DispatchQuery}: the generic + * cores/memory dispatch pipeline is performance sensitive and must not be affected by the slot + * axis. The only interaction with the generic path is the {@code *_EXCLUDE_SLOT} variants below, + * which append a single equality predicate so regular hosts and procs never pick up slot-based + * frames (strict pairing). + */ +public class SlotDispatchQuery { + + /** + * Marker present in every generic frame-dispatch query; used to append the slot-layer exclusion + * predicate without touching {@link DispatchQuery} itself. + */ + private static final String WAITING_MARKER = "frame.str_state='WAITING' "; + + private static String excludeSlotLayers(String query) { + if (!query.contains(WAITING_MARKER)) { + throw new IllegalStateException( + "Generic dispatch query no longer contains the WAITING marker; " + + "the slot-layer exclusion cannot be applied."); + } + return query.replace(WAITING_MARKER, + "frame.str_state='WAITING' AND layer.int_slots_required = 0 "); + } + + /** + * Generic frame-dispatch queries with slot-based layers excluded. Used by the regular dispatch + * path so a slot layer can never book onto a regular (cores/memory) host or proc. + */ + public static final String FIND_DISPATCH_FRAME_BY_JOB_AND_PROC_EXCLUDE_SLOT = + excludeSlotLayers(DispatchQuery.FIND_DISPATCH_FRAME_BY_JOB_AND_PROC); + + public static final String FIND_DISPATCH_FRAME_BY_JOB_AND_HOST_EXCLUDE_SLOT = + excludeSlotLayers(DispatchQuery.FIND_DISPATCH_FRAME_BY_JOB_AND_HOST); + + public static final String FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC_EXCLUDE_SLOT = + excludeSlotLayers(DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_PROC); + + public static final String FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST_EXCLUDE_SLOT = + excludeSlotLayers(DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_JOB_AND_HOST); + + public static final String FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC_EXCLUDE_SLOT = + excludeSlotLayers(DispatchQuery.FIND_DISPATCH_FRAME_BY_LAYER_AND_PROC); + + public static final String FIND_DISPATCH_FRAME_BY_LAYER_AND_HOST_EXCLUDE_SLOT = + excludeSlotLayers(DispatchQuery.FIND_DISPATCH_FRAME_BY_LAYER_AND_HOST); + + public static final String FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC_EXCLUDE_SLOT = + excludeSlotLayers(DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_PROC); + + public static final String FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST_EXCLUDE_SLOT = + excludeSlotLayers(DispatchQuery.FIND_LOCAL_DISPATCH_FRAME_BY_LAYER_AND_HOST); + + /** + * Find jobs with pending slot-based work bookable on a slot host. + * + * A job qualifies when it is PENDING, matches the host's facility/OS, has at least one + * slot-based layer with waiting frames whose tags match the host and whose slot requirement + * fits the host's idle slots, and is under the job/folder max_slots caps. The show must also + * subscribe to the host's allocation and be under the subscription max_slots cap. Slot usage at + * every level derives from SUM(proc.int_slots_reserved). + * + * Binds: facility, os..., idle_slots, host_name, alloc, limit + */ + // spotless:off + public static final String FIND_SLOT_DISPATCH_JOBS = + "/* FIND_SLOT_DISPATCH_JOBS */ " + + "SELECT pk_job, int_priority, rank FROM ( " + + "SELECT " + + "ROW_NUMBER() OVER (ORDER BY job_resource.int_priority DESC) AS rank, " + + "job.pk_job, " + + "job_resource.int_priority " + + "FROM " + + "job " + + "JOIN job_resource ON job_resource.pk_job = job.pk_job " + + "JOIN folder_resource ON folder_resource.pk_folder = job.pk_folder " + + "JOIN show ON show.pk_show = job.pk_show " + + "WHERE " + /* Scheduler-managed shows are dispatched by the standalone Rust + * scheduler, including their slot-based work. */ + + "show.b_scheduler_managed = false " + + "AND " + + "job.str_state = 'PENDING' " + + "AND " + + "job.b_paused = false " + + "AND " + + "job.pk_facility = ? " + + "AND " + + "(job.str_os IS NULL OR job.str_os = '' OR job.str_os IN ?) " + + "AND EXISTS ( " + + "SELECT 1 " + + "FROM " + + "layer " + + "JOIN layer_stat ON layer_stat.pk_layer = layer.pk_layer " + + "JOIN host ON host.str_tags ~* ('(?x)' || layer.str_tags || '\\y') " + + "WHERE " + + "layer.pk_job = job.pk_job " + + "AND " + + "layer.int_slots_required > 0 " + + "AND " + + "layer.int_slots_required <= ? " + + "AND " + + "layer_stat.int_waiting_count > 0 " + + "AND " + + "host.str_name = ? " + + ") " + + "AND " + + "(job_resource.int_max_slots = -1 OR " + + "(SELECT COALESCE(SUM(proc.int_slots_reserved), 0) FROM proc " + + "WHERE proc.pk_job = job.pk_job) " + + "< job_resource.int_max_slots) " + + "AND " + + "(folder_resource.int_max_slots = -1 OR " + + "(SELECT COALESCE(SUM(proc.int_slots_reserved), 0) " + + "FROM proc JOIN job j ON j.pk_job = proc.pk_job " + + "WHERE j.pk_folder = job.pk_folder) " + + "< folder_resource.int_max_slots) " + + "AND EXISTS ( " + + "SELECT 1 FROM subscription s " + + "WHERE " + + "s.pk_show = job.pk_show " + + "AND " + + "s.pk_alloc = ? " + + "AND " + + "(s.int_max_slots = -1 OR " + + "(SELECT COALESCE(SUM(p.int_slots_reserved), 0) " + + "FROM proc p JOIN host h ON h.pk_host = p.pk_host " + + "WHERE p.pk_show = s.pk_show AND h.pk_alloc = s.pk_alloc) " + + "< s.int_max_slots) " + + ") " + /* rank < ? returns numJobs - 1 rows; kept as-is for parity with the generic + * FIND_JOBS queries, which use the same bound. */ + + ") AS t1 WHERE rank < ?"; + // spotless:on + + /** + * Compute how many more slots the given job may book, taking the smallest remaining allowance + * across the job, folder and subscription max_slots caps. NULL means unlimited (all three caps + * are -1). Usage at every level derives from SUM(proc.int_slots_reserved). + * + * Binds: alloc, job + */ + // spotless:off + public static final String GET_SLOT_CAPACITY_REMAINING = + "/* GET_SLOT_CAPACITY_REMAINING */ " + + "SELECT LEAST( " + + "CASE WHEN job_resource.int_max_slots = -1 THEN NULL " + + "ELSE job_resource.int_max_slots - " + + "(SELECT COALESCE(SUM(p.int_slots_reserved), 0) FROM proc p " + + "WHERE p.pk_job = job.pk_job) " + + "END, " + + "CASE WHEN folder_resource.int_max_slots = -1 THEN NULL " + + "ELSE folder_resource.int_max_slots - " + + "(SELECT COALESCE(SUM(p.int_slots_reserved), 0) " + + "FROM proc p JOIN job j2 ON j2.pk_job = p.pk_job " + + "WHERE j2.pk_folder = job.pk_folder) " + + "END, " + + "CASE WHEN s.int_max_slots = -1 THEN NULL " + + "ELSE s.int_max_slots - " + + "(SELECT COALESCE(SUM(p.int_slots_reserved), 0) " + + "FROM proc p JOIN host h ON h.pk_host = p.pk_host " + + "WHERE p.pk_show = job.pk_show AND h.pk_alloc = s.pk_alloc) " + + "END " + + ") AS int_slot_capacity " + + "FROM job " + + "JOIN job_resource ON job_resource.pk_job = job.pk_job " + + "JOIN folder_resource ON folder_resource.pk_folder = job.pk_folder " + + "JOIN subscription s ON s.pk_show = job.pk_show AND s.pk_alloc = ? " + + "WHERE job.pk_job = ?"; + // spotless:on + + /** + * Find the next slot-based frames in a job for a slot host. + * + * Selects WAITING frames of slot-based layers whose slot requirement fits the host's idle + * slots, whose tags match the host, that are under the layer limits (limit_record) and under + * the job/folder/subscription max_slots caps counting this layer's requirement. + * + * Binds: idle_slots, job, host_name, job, alloc, limit + */ + // spotless:off + public static final String FIND_SLOT_DISPATCH_FRAMES_BY_JOB_AND_HOST = + "/* FIND_SLOT_DISPATCH_FRAMES_BY_JOB_AND_HOST */ " + + "SELECT " + + "show_name, " + + "job_name, " + + "pk_job, " + + "pk_show, " + + "pk_facility, " + + "str_name, " + + "str_shot, " + + "str_user, " + + "int_uid, " + + "str_log_dir, " + + "COALESCE(str_os, '') AS str_os, " + + "COALESCE(str_loki_url, '') AS str_loki_url, " + + "frame_name, " + + "frame_state, " + + "pk_frame, " + + "pk_layer, " + + "int_retries, " + + "int_version, " + + "layer_name, " + + "layer_type, " + + "b_threadable, " + + "int_cores_min, " + + "int_cores_max, " + + "int_mem_min, " + + "int_gpus_min, " + + "int_gpus_max, " + + "int_gpu_mem_min, " + + "int_slots_required, " + + "str_cmd, " + + "str_range, " + + "int_chunk_size, " + + "str_services " + + "FROM ( " + + "SELECT " + + "ROW_NUMBER() OVER ( ORDER BY " + + "frame.int_dispatch_order ASC, " + + "frame.int_layer_order ASC " + + ") AS LINENUM, " + + "job.str_show AS show_name, " + + "job.str_name AS job_name, " + + "job.pk_job, " + + "job.pk_show, " + + "job.pk_facility, " + + "job.str_name, " + + "job.str_shot, " + + "job.str_user, " + + "job.int_uid, " + + "job.str_log_dir, " + + "job.str_os, " + + "job.str_loki_url, " + + "frame.str_name AS frame_name, " + + "frame.str_state AS frame_state, " + + "frame.pk_frame, " + + "frame.pk_layer, " + + "frame.int_retries, " + + "frame.int_version, " + + "layer.str_name AS layer_name, " + + "layer.str_type AS layer_type, " + + "layer.b_threadable, " + + "layer.int_cores_min, " + + "layer.int_cores_max, " + + "layer.int_mem_min, " + + "layer.int_gpus_min, " + + "layer.int_gpus_max, " + + "layer.int_gpu_mem_min, " + + "layer.int_slots_required, " + + "layer.str_cmd, " + + "layer.str_range, " + + "layer.int_chunk_size, " + + "layer.str_services " + + "FROM " + + "job, " + + "frame, " + + "layer " + + "WHERE " + + "frame.pk_layer = layer.pk_layer " + + "AND " + + "layer.pk_job = job.pk_job " + + "AND " + + "frame.str_state='WAITING' " + + "AND " + + "layer.int_slots_required > 0 " + + "AND " + + "layer.int_slots_required <= ? " + + "AND " + + "job.pk_job = ? " + + "AND " + + "layer.pk_layer IN ( " + + "SELECT " + + "l.pk_layer " + + "FROM " + + "layer l " + + "JOIN job j ON j.pk_job = l.pk_job " + + "JOIN job_resource ON job_resource.pk_job = j.pk_job " + + "JOIN folder_resource ON folder_resource.pk_folder = j.pk_folder " + + "JOIN host h ON (h.str_tags ~* ('(?x)' || l.str_tags || '\\y') AND h.str_name = ?) " + + "LEFT JOIN layer_limit ON layer_limit.pk_layer = l.pk_layer " + + "LEFT JOIN limit_record ON limit_record.pk_limit_record = layer_limit.pk_limit_record " + + "LEFT JOIN (" + + "SELECT " + + "limit_record.pk_limit_record, " + + "SUM(layer_stat.int_running_count) AS int_sum_running " + + "FROM " + + "layer_limit " + + "LEFT JOIN limit_record ON layer_limit.pk_limit_record = limit_record.pk_limit_record " + + "LEFT JOIN layer_stat ON layer_stat.pk_layer = layer_limit.pk_layer " + + "GROUP BY limit_record.pk_limit_record) AS sum_running " + + "ON limit_record.pk_limit_record = sum_running.pk_limit_record " + + "WHERE " + + "l.pk_job = ? " + + "AND " + + "(sum_running.int_sum_running < limit_record.int_max_value " + + "OR sum_running.int_sum_running IS NULL) " + + "AND " + + "(job_resource.int_max_slots = -1 OR " + + "(SELECT COALESCE(SUM(proc.int_slots_reserved), 0) FROM proc " + + "WHERE proc.pk_job = j.pk_job) " + + "+ l.int_slots_required <= job_resource.int_max_slots) " + + "AND " + + "(folder_resource.int_max_slots = -1 OR " + + "(SELECT COALESCE(SUM(proc.int_slots_reserved), 0) " + + "FROM proc JOIN job j2 ON j2.pk_job = proc.pk_job " + + "WHERE j2.pk_folder = j.pk_folder) " + + "+ l.int_slots_required <= folder_resource.int_max_slots) " + + "AND EXISTS ( " + + "SELECT 1 FROM subscription s " + + "WHERE " + + "s.pk_show = j.pk_show " + + "AND " + + "s.pk_alloc = ? " + + "AND " + + "(s.int_max_slots = -1 OR " + + "(SELECT COALESCE(SUM(p.int_slots_reserved), 0) " + + "FROM proc p JOIN host h2 ON h2.pk_host = p.pk_host " + + "WHERE p.pk_show = s.pk_show AND h2.pk_alloc = s.pk_alloc) " + + "+ l.int_slots_required <= s.int_max_slots) " + + ") " + + ") " + + ") AS t1 WHERE LINENUM <= ?"; + // spotless:on +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java index 88ca36ace4..ed4f0cd901 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupport.java @@ -509,6 +509,36 @@ void updateProcMemoryUsage(FrameInterface frame, long rss, long maxRss, long pss */ Set findDispatchJobs(DispatchHost host, ShowInterface show, int numJobs); + /** + * Return a set of job IDs with pending slot-based work bookable on the given slot-based host. + * + * @param host a slot-based host (concurrentSlotsLimit >= 0) + * @param numJobs + * @return + */ + Set findSlotDispatchJobs(DispatchHost host, int numJobs); + + /** + * Return the next slot-based frames of the given job bookable on the given slot-based host. + * + * @param job + * @param host a slot-based host (concurrentSlotsLimit >= 0) + * @param limit + * @return + */ + List findNextSlotDispatchFrames(JobInterface job, DispatchHost host, int limit); + + /** + * Return how many more slots the given job may book on the given host's allocation (smallest + * remaining allowance across the job/folder/subscription max_slots caps; Integer.MAX_VALUE = + * unlimited). + * + * @param job + * @param host + * @return + */ + int getSlotCapacityRemaining(JobInterface job, DispatchHost host); + /** * Return true of the job has pending frames. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 75d9f43c81..0bfbb05eab 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -187,6 +187,25 @@ public Set findDispatchJobs(DispatchHost host, ShowInterface show, int n return dispatcherDao.findDispatchJobs(host, show, numJobs); } + @Override + @Transactional(readOnly = true) + public Set findSlotDispatchJobs(DispatchHost host, int numJobs) { + return dispatcherDao.findSlotDispatchJobs(host, numJobs); + } + + @Override + @Transactional(readOnly = true) + public List findNextSlotDispatchFrames(JobInterface job, DispatchHost host, + int limit) { + return dispatcherDao.findNextSlotDispatchFrames(job, host, limit); + } + + @Override + @Transactional(readOnly = true) + public int getSlotCapacityRemaining(JobInterface job, DispatchHost host) { + return dispatcherDao.getSlotCapacityRemaining(job, host); + } + @Transactional(propagation = Propagation.REQUIRED) public boolean increaseReservedMemory(ProcInterface p, long value) { return procDao.increaseReservedMemory(p, value); @@ -395,8 +414,9 @@ public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { .setJobName(frame.jobName).setFrameId(frame.id).setFrameName(frame.name) .setLayerId(frame.getLayerId()).setResourceId(proc.getProcId()) .setNumCores(proc.coresReserved).setNumGpus(proc.gpusReserved) - .setStartTime(System.currentTimeMillis()).setIgnoreNimby(proc.isLocalDispatch) - .setOs(proc.os).setSoftMemoryLimit(frame.softMemoryLimit).setLokiUrl(frame.lokiURL) + .setSlotsRequired(proc.slotsReserved).setStartTime(System.currentTimeMillis()) + .setIgnoreNimby(proc.isLocalDispatch).setOs(proc.os) + .setSoftMemoryLimit(frame.softMemoryLimit).setLokiUrl(frame.lokiURL) .setHardMemoryLimit(frame.hardMemoryLimit) .putAllEnvironment(jobDao.getEnvironment(frame)) .putAllEnvironment(layerDao.getLayerEnvironment(frame)).putEnvironment("CUE3", "1") diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index ce745f6186..212c74d178 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -37,6 +37,7 @@ import com.imageworks.spcue.Source; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; +import com.imageworks.spcue.dispatcher.commands.DispatchBookHostSlots; import com.imageworks.spcue.dispatcher.commands.DispatchNextFrame; import com.imageworks.spcue.dispatcher.commands.KeyRunnable; import com.imageworks.spcue.grpc.host.LockState; @@ -86,6 +87,7 @@ public class FrameCompleteHandler { private BookingQueue bookingQueue; private Dispatcher dispatcher; private Dispatcher localDispatcher; + private Dispatcher slotDispatcher; private JobManagerSupport jobManagerSupport; private DispatchSupport dispatchSupport; private JmsMover jsmMover; @@ -347,10 +349,16 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, FrameCompleteRep * memory issue and should be retried. In this case, disable the optimizer and raise the * memory by what is specified in the show's service override, service or 2GB. */ - if (report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE + if (proc.slotsReserved == 0 && (report + .getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE || frameDetail.exitStatus == Dispatcher.EXIT_STATUS_MEMORY_FAILURE - || report.getExitStatus() == Dispatcher.DOCKER_EXIT_STATUS_MEMORY_FAILURE) { + || report.getExitStatus() == Dispatcher.DOCKER_EXIT_STATUS_MEMORY_FAILURE)) { + /* + * Slot-based procs are excluded: they reserve 0 memory by design, so raising the + * layer's memory requirement would only corrupt the layer record (slot dispatch + * ignores memory entirely). + */ long increase = CueUtil.GB2; // since there can be multiple services, just going for the @@ -529,6 +537,23 @@ else if (report.getHost().getNimbyLocked()) { if (newFrameState.equals(FrameState.WAITING) || newFrameState.equals(FrameState.SUCCEEDED)) { + /* + * Slot-based procs are never reused for the next frame: the generic + * proc-to-next-frame path books by cores/memory, which slot procs don't hold. + * Release the slots and requeue the host through the slot dispatcher so it can pick + * up its next slot frame immediately. + * + * Non-happy-path releases (DEAD/EATEN frames, failed launches, the generic unbook + * branches above) release the slots without this immediate requeue; the host's next + * report picks the slots back up. + */ + if (proc.slotsReserved > 0) { + dispatchSupport.unbookProc(proc, "slot-based proc, releasing slots"); + bookingQueue.execute(new DispatchBookHostSlots( + hostManager.getDispatchHost(proc.getHostId()), slotDispatcher)); + return; + } + /* * Scheduler-managed shows: the standalone Rust scheduler owns dispatch. Don't * reuse/rebook the proc here (that races the scheduler and strands pk_frame=NULL @@ -787,6 +812,14 @@ public Dispatcher getLocalDispatcher() { return localDispatcher; } + public Dispatcher getSlotDispatcher() { + return slotDispatcher; + } + + public void setSlotDispatcher(Dispatcher slotDispatcher) { + this.slotDispatcher = slotDispatcher; + } + public void setLocalDispatcher(Dispatcher localDispatcher) { this.localDispatcher = localDispatcher; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 6277b5de34..d9c9103911 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -34,6 +34,7 @@ import com.imageworks.spcue.dao.LayerDao; import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; import com.imageworks.spcue.dispatcher.commands.DispatchBookHostLocal; +import com.imageworks.spcue.dispatcher.commands.DispatchBookHostSlots; import com.imageworks.spcue.dispatcher.commands.DispatchHandleHostReport; import com.imageworks.spcue.dispatcher.commands.DispatchRqdKillFrame; import com.imageworks.spcue.dispatcher.commands.DispatchRqdKillFrameMemory; @@ -85,6 +86,7 @@ public class HostReportHandler { private DispatchSupport dispatchSupport; private Dispatcher dispatcher; private Dispatcher localDispatcher; + private Dispatcher slotDispatcher; private RqdClient rqdClient; private JobManager jobManager; private JobDao jobDao; @@ -270,6 +272,17 @@ public void handleHostReport(HostReport report, boolean isBoot) { */ handleMemoryUsage(host, report.getHost(), runningFrames); + /* + * Slot-based hosts (concurrentSlotsLimit >= 0) only run slot-based layers and book + * purely by concurrency slots. Their booking flow is deviated to the slot dispatcher + * and never enters the generic cores/memory dispatch pipeline. Cores/memory checks are + * intentionally skipped: slot bookings reserve neither. + */ + if (host.isSlotHost()) { + handleSlotHostBooking(host, report); + return; + } + /* * The checks are done in order of least CPU intensive to most CPU intensive, saving * checks that hit the DB for last. @@ -365,6 +378,44 @@ public void handleHostReport(HostReport report, boolean isBoot) { } } + /** + * Queue a slot-based host for booking through the slot dispatcher if it passes the slot host + * bookability checks: enough temp storage, hardware UP, not locked or NIMBY locked, at least + * one idle slot and pending work on the cue. Cores/memory checks are skipped by design - slot + * bookings reserve neither. + * + * @param host a slot-based host (isSlotHost() == true) + * @param report + */ + private void handleSlotHostBooking(DispatchHost host, HostReport report) { + String msg = null; + if (!isTempDirStorageEnough(report.getHost().getTotalMcp(), report.getHost().getFreeMcp(), + host.getOs())) { + msg = String.format( + "%s doesn't have enough free space in the temporary directory (mcp), %dMB", + host.name, (report.getHost().getFreeMcp() / 1024)); + } else if (!host.hardwareState.equals(HardwareState.UP)) { + msg = host + " is not in the Up state."; + } else if (host.lockState.equals(LockState.LOCKED)) { + msg = host + " is locked."; + } else if (report.getHost().getNimbyLocked()) { + msg = host + " is NIMBY locked."; + } else if (host.idleSlots <= 0) { + msg = String.format("%s doesn't have any idle slots, %d of %d in use.", host.name, + host.concurrentSlotsLimit - host.idleSlots, host.concurrentSlotsLimit); + } else if (!dispatchSupport.isCueBookable(host)) { + msg = "The cue has no pending jobs"; + } + + if (msg != null) { + logger.trace(msg); + } else if (env.getProperty("dispatcher.turn_off_booking", Boolean.class, false)) { + logger.debug("Booking has been turned off on Cuebot's configuration"); + } else { + bookingQueue.execute(new DispatchBookHostSlots(host, slotDispatcher)); + } + } + /** * Check if a reported temp storage size and availability is enough for running a job * @@ -777,6 +828,12 @@ private boolean isFrameOverboard(final RunningFrameInfo frame) { try { VirtualProc proc = hostManager.getVirtualProc(frame.getResourceId()); + + // Slot-based procs reserve 0 memory by design; the overboard check is meaningless + // for them and would kill every slot frame. + if (proc.slotsReserved > 0) { + return false; + } double reserved = (double) proc.memoryReserved; // Last memory report is higher than the threshold @@ -820,6 +877,12 @@ private void handleMemoryReservations(final RunningFrameInfo frame) { return; } + // Slot-based procs reserve 0 memory by design; don't grow their reservation (that + // would corrupt the host's idle memory accounting). + if (proc.slotsReserved > 0) { + return; + } + if (dispatchSupport.increaseReservedMemory(proc, frame.getRss())) { proc.memoryReserved = frame.getRss(); logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() @@ -1207,6 +1270,14 @@ public void setLocalDispatcher(Dispatcher localDispatcher) { this.localDispatcher = localDispatcher; } + public Dispatcher getSlotDispatcher() { + return slotDispatcher; + } + + public void setSlotDispatcher(Dispatcher slotDispatcher) { + this.slotDispatcher = slotDispatcher; + } + public ThreadPoolExecutor getKillQueue() { return killQueue; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java index 7fe75c1517..c50a3be2e3 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/RedirectManager.java @@ -115,6 +115,10 @@ public List addRedirect(ProcSearchCriteria criteria, GroupInterface search.notGroups(groups); List procs = hostManager.findBookedVirtualProcs(search); + + // Slot-based procs cannot be redirected; see addRedirect(ProcSearchCriteria, List...). + procs.removeIf(proc -> proc.slotsReserved > 0); + if (procs.size() == 0) { return procs; } @@ -153,6 +157,13 @@ public List addRedirect(ProcSearchCriteria criteria, List procs = hostManager.findBookedVirtualProcs(procSearch); + + /* + * Slot-based procs cannot be redirected: redirecting rebooks the host through the generic + * cores/memory dispatcher, which would break the strict slot host / slot layer pairing. + */ + procs.removeIf(proc -> proc.slotsReserved > 0); + if (procs.size() == 0) { return procs; } @@ -194,6 +205,10 @@ public boolean addRedirect(List procs, JobInterface job, Source sou String redirectGroupId = SqlUtil.genKeyRandom(); for (VirtualProc proc : procs) { + // Slot-based procs cannot be redirected; see addRedirect(ProcSearchCriteria...). + if (proc.slotsReserved > 0) { + continue; + } Redirect r = new Redirect(redirectGroupId, job); if (procDao.setRedirectTarget(proc, r)) { redirectService.put(proc.getProcId(), r); @@ -201,6 +216,9 @@ public boolean addRedirect(List procs, JobInterface job, Source sou } for (VirtualProc proc : procs) { + if (proc.slotsReserved > 0) { + continue; + } jobManagerSupport.kill(proc, source); } @@ -218,6 +236,12 @@ public boolean addRedirect(List procs, JobInterface job, Source sou */ public boolean addRedirect(VirtualProc proc, JobInterface job, boolean kill, Source source) { + // Slot-based procs cannot be redirected; see addRedirect(ProcSearchCriteria...). + if (proc.slotsReserved > 0) { + logger.info("Ignoring redirect of slot-based proc " + proc); + return false; + } + if (dispatchSupport.findNextDispatchFrames(job, proc, 1).size() < 1) { return false; } @@ -246,6 +270,12 @@ public boolean addRedirect(VirtualProc proc, JobInterface job, boolean kill, Sou public boolean addRedirect(VirtualProc proc, GroupInterface group, boolean kill, Source source) { + // Slot-based procs cannot be redirected; see addRedirect(ProcSearchCriteria...). + if (proc.slotsReserved > 0) { + logger.info("Ignoring redirect of slot-based proc " + proc); + return false; + } + // Test a dispatch DispatchHost host = hostManager.getDispatchHost(proc.getHostId()); host.idleCores = proc.coresReserved; @@ -280,6 +310,17 @@ public boolean redirect(VirtualProc proc) { try { + /* + * Defensive: slot procs never get redirect targets (see the addRedirect guards), but if + * one slips through, release it instead of rebooking the slot host through the generic + * cores/memory dispatcher. + */ + if (proc.slotsReserved > 0) { + redirectService.remove(proc.getProcId()); + dispatchSupport.unbookProc(proc, "slot-based proc cannot be redirected"); + return false; + } + Redirect r = redirectService.remove(proc.getProcId()); if (r == null) { logger.info("Failed to find redirect for proc " + proc); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/SlotDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/SlotDispatcher.java new file mode 100644 index 0000000000..4f1d91ea6d --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/SlotDispatcher.java @@ -0,0 +1,339 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; + +import com.imageworks.spcue.DispatchFrame; +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.DispatchJob; +import com.imageworks.spcue.FrameInterface; +import com.imageworks.spcue.GroupInterface; +import com.imageworks.spcue.JobInterface; +import com.imageworks.spcue.LayerInterface; +import com.imageworks.spcue.ShowInterface; +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.rqd.RqdClient; +import com.imageworks.spcue.rqd.RqdClientException; +import com.imageworks.spcue.service.JobManager; + +/** + * The Slot Dispatcher. + * + * Books slot-based layers (layer.slotsRequired > 0) onto slot-based hosts + * (host.concurrentSlotsLimit >= 0). Slot bookings reserve 0 cores and 0 memory; the only + * constraints are the host's concurrent slots limit and the subscription/folder/job max_slots caps. + * This dispatcher is completely independent from the generic cores/memory pipeline + * ({@link CoreUnitDispatcher}): host reports for slot-based hosts are deviated here by + * {@link HostReportHandler} and never enter the generic dispatch queries. + * + * The dispatch pipeline mirrors the generic one: + * + * 1. Find jobs with pending slot work bookable on this host. + * + * 2. For each job, find slot frames that fit the host's idle slots and the max_slots caps. + * + * 3. Reserve the frame (WAITING to RUNNING), insert the proc (the before_insert_proc database + * trigger revalidates the host's slot cap under a host row lock, so concurrent bookings can never + * exceed the cap), and launch on RQD. + */ +public class SlotDispatcher implements Dispatcher { + + private static final Logger logger = LogManager.getLogger(SlotDispatcher.class); + + private DispatchSupport dispatchSupport; + + private JobManager jobManager; + + private RqdClient rqdClient; + + public boolean testMode = false; + + private Environment env; + + @Autowired + public SlotDispatcher(Environment env) { + this.env = env; + } + + private int getIntProperty(String property) { + return env.getRequiredProperty(property, Integer.class); + } + + /* + * Keeps a map of unique job IDs that should be skipped over for booking until the record has + * expired, so concurrent slot hosts don't all pile onto the same top-priority job. Mirrors + * CoreUnitDispatcher's job lock. + */ + private Cache jobLock; + + private Cache getOrCreateJobLock() { + if (jobLock == null) { + this.jobLock = CacheBuilder.newBuilder() + .concurrencyLevel(getIntProperty("dispatcher.job_lock_concurrency_level")) + .expireAfterWrite(getIntProperty("dispatcher.job_lock_expire_seconds"), + TimeUnit.SECONDS) + .build(); + } + return jobLock; + } + + @Override + public List dispatchHostToAllShows(DispatchHost host) { + return dispatchHost(host); + } + + @Override + public List dispatchHost(DispatchHost host) { + List procs = new ArrayList(); + + if (!host.isSlotHost()) { + logger.warn(host.getName() + " is not a slot-based host, skipping slot dispatch."); + return procs; + } + + Set jobs = dispatchSupport.findSlotDispatchJobs(host, + getIntProperty("dispatcher.job_query_max")); + + try { + for (String jobId : jobs) { + if (host.idleSlots <= 0) { + break; + } + + if (procs.size() >= getIntProperty("dispatcher.host_frame_dispatch_max")) { + break; + } + + if (getIntProperty("dispatcher.job_lock_expire_seconds") > 0) { + if (getOrCreateJobLock().getIfPresent(jobId) != null) { + continue; + } + jobLock.put(jobId, jobId); + } + + DispatchJob job = jobManager.getDispatchJob(jobId); + try { + procs.addAll(dispatchHost(host, job)); + } catch (DispatcherException e) { + /* + * Something is wrong with the host itself (e.g. RQD is unreachable); stop + * dispatching to it instead of churning through every remaining job. + */ + throw e; + } catch (Exception e) { + logger.info("slot job dispatch exception, " + e); + } + } + } catch (DispatcherException e) { + logger.info(host.name + " slot dispatcher exception, " + e); + } + + return procs; + } + + @Override + public List dispatchHost(DispatchHost host, JobInterface job) { + List procs = new ArrayList(); + + if (!host.isSlotHost() || host.idleSlots <= 0) { + return procs; + } + + List frames = dispatchSupport.findNextSlotDispatchFrames(job, host, + getIntProperty("dispatcher.frame_query_max")); + + logger.info("Slot frames found: " + frames.size() + " for host " + host.getName() + " " + + host.idleSlots + "/" + host.concurrentSlotsLimit + " idle slots on job " + + job.getName()); + + /* + * The frame query checks the job/folder/subscription max_slots caps against usage at query + * time; bookings made in this pass are not reflected there, so track them here to never + * exceed a cap within the pass. + */ + int slotCapacityRemaining = dispatchSupport.getSlotCapacityRemaining(job, host); + int slotsBooked = 0; + + for (DispatchFrame frame : frames) { + + if (frame.slotsRequired <= 0 || frame.slotsRequired > host.idleSlots) { + continue; + } + + if (slotsBooked + frame.slotsRequired > slotCapacityRemaining) { + // The job/folder/subscription caps have no room for another booking of this + // job in this pass. + break; + } + + VirtualProc proc = VirtualProc.buildSlotProc(host, frame); + + try { + dispatch(frame, proc); + dispatchSummary(proc, frame); + } catch (FrameReservationException fre) { + /* + * Another thread got the frame first, move on to the next frame. + */ + DispatchSupport.bookingRetries.incrementAndGet(); + logger.info("slot frame reservation error, " + "failed to book next frame, " + fre); + continue; + } catch (ResourceDuplicationFailureException rdfe) { + /* + * The frame already has a proc assigned; fix it and move on. + */ + DispatchSupport.bookingErrors.incrementAndGet(); + dispatchSupport.fixFrame(frame); + logger.info("slot proc update error, " + proc + " already assigned " + + "to another frame, " + rdfe); + continue; + } catch (ResourceReservationFailureException rrfe) { + /* + * The host slot cap was exceeded (enforced by the before_insert_proc trigger) or + * the proc insert failed. Clear the frame back to WAITING and stop dispatching to + * this host; its slot capacity is spent. + */ + DispatchSupport.bookingErrors.incrementAndGet(); + dispatchSupport.clearFrame(frame); + logger.info("slot reservation error, failed to allocate slots on " + host.getName() + + ", " + rrfe); + break; + } catch (Exception e) { + /* + * The frame/host records may have been updated but something else failed. Unbook + * the proc, clear the frame and stop dispatching this host. Also send a kill just + * in case the frame actually launched. + */ + DispatchSupport.bookingErrors.incrementAndGet(); + logger.warn("slot dispatch failed booking proc " + proc + " on job " + job, e); + dispatchSupport.unbookProc(proc); + dispatchSupport.clearFrame(frame); + try { + rqdClient.killFrame(proc, + "An accounting error occured when booking this frame."); + } catch (RqdClientException rqde) { + // Expected to fail unless the frame actually launched. + } + throw new DispatcherException( + "stopped slot dispatching " + host.getName() + ", " + e); + } + + procs.add(proc); + DispatchSupport.bookedProcs.getAndIncrement(); + host.idleSlots = host.idleSlots - frame.slotsRequired; + slotsBooked = slotsBooked + frame.slotsRequired; + + if (host.idleSlots <= 0) { + break; + } + if (procs.size() >= getIntProperty("dispatcher.job_frame_dispatch_max")) { + break; + } + if (procs.size() >= getIntProperty("dispatcher.host_frame_dispatch_max")) { + break; + } + } + + return procs; + } + + @Override + public void dispatch(DispatchFrame frame, VirtualProc proc) { + // Allocate frame on the database + dispatchSupport.startFrameAndProc(proc, frame); + + // Communicate with RQD to run the frame. + if (!testMode) { + dispatchSupport.runFrame(proc, frame); + } + } + + @Override + public List dispatchHost(DispatchHost host, ShowInterface show) { + throw new RuntimeException("not implemented"); + } + + @Override + public List dispatchHost(DispatchHost host, GroupInterface group) { + throw new RuntimeException("not implemented"); + } + + @Override + public List dispatchHost(DispatchHost host, LayerInterface layer) { + throw new RuntimeException("not implemented"); + } + + @Override + public List dispatchHost(DispatchHost host, FrameInterface frame) { + throw new RuntimeException("not implemented"); + } + + @Override + public void dispatchProcToJob(VirtualProc proc, JobInterface job) { + throw new RuntimeException("not implemented, slot procs are unbooked on frame completion"); + } + + @Override + public boolean isTestMode() { + return testMode; + } + + @Override + public void setTestMode(boolean enabled) { + testMode = enabled; + } + + private void dispatchSummary(VirtualProc p, DispatchFrame f) { + logger.trace("Slot booking summary: " + p.slotsReserved + " slots " + p.getName() + " to " + + f.show + "/" + f.shot); + } + + public DispatchSupport getDispatchSupport() { + return dispatchSupport; + } + + public void setDispatchSupport(DispatchSupport dispatchSupport) { + this.dispatchSupport = dispatchSupport; + } + + public JobManager getJobManager() { + return jobManager; + } + + public void setJobManager(JobManager jobManager) { + this.jobManager = jobManager; + } + + public RqdClient getRqdClient() { + return rqdClient; + } + + public void setRqdClient(RqdClient rqdClient) { + this.rqdClient = rqdClient; + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostSlots.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostSlots.java new file mode 100644 index 0000000000..d6e47db6f0 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostSlots.java @@ -0,0 +1,52 @@ +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher.commands; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.dispatcher.Dispatcher; + +/** + * A command for booking a slot-based host through the slot dispatcher. + * + * Slot-based hosts (host.concurrentSlotsLimit >= 0) only run slot-based layers, so their booking + * never enters the generic cores/memory dispatch pipeline. + * + * @category command + */ +public class DispatchBookHostSlots extends KeyRunnable { + + private DispatchHost host; + private Dispatcher dispatcher; + + public DispatchBookHostSlots(DispatchHost host, Dispatcher d) { + super(host.getId() + "_slots"); + this.host = host; + this.dispatcher = d; + } + + public DispatchHost getDispatchHost() { + return host; + } + + public void run() { + new DispatchCommandTemplate() { + public void wrapDispatchCommand() { + if (host.idleSlots > 0) { + dispatcher.dispatchHost(host); + } + } + }.execute(); + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java index 405e4e5b28..0000df273d 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/BookingManagerService.java @@ -24,6 +24,7 @@ import org.springframework.transaction.annotation.Transactional; import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.EntityException; import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.HostInterface; import com.imageworks.spcue.JobDetail; @@ -151,21 +152,35 @@ public LocalHostAssignment getLocalHostAssignment(String hostId, String jobId) { * Create LocalHostAssignments */ + /** + * Slot-based hosts only run slot-based layers; a local (cores/memory) booking on one would + * break the strict slot host / slot layer pairing. + */ + private void checkNotSlotHost(DispatchHost host) { + if (host.isSlotHost()) { + throw new EntityException("Cannot create a local host assignment on slot-based host " + + host.getName() + ", slot-based hosts only run slot-based layers."); + } + } + @Override public void createLocalHostAssignment(DispatchHost host, JobInterface job, LocalHostAssignment lja) { + checkNotSlotHost(host); bookingDao.insertLocalHostAssignment(host, job, lja); } @Override public void createLocalHostAssignment(DispatchHost host, LayerInterface layer, LocalHostAssignment lja) { + checkNotSlotHost(host); bookingDao.insertLocalHostAssignment(host, layer, lja); } @Override public void createLocalHostAssignment(DispatchHost host, FrameInterface frame, LocalHostAssignment lja) { + checkNotSlotHost(host); bookingDao.insertLocalHostAssignment(host, frame, lja); } diff --git a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java index e3ab8864cb..6dc7c61c26 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java +++ b/cuebot/src/main/java/com/imageworks/spcue/service/JobSpec.java @@ -447,6 +447,18 @@ private void handleLayerTags(BuildableJob buildableJob, Element jobTag) { String slotsRequired = layerTag.getChildTextTrim("slots_required"); if (slotsRequired != null && !slotsRequired.isEmpty()) { layer.slotsRequired = Integer.parseInt(slotsRequired); + if (layer.slotsRequired < 0) { + // A negative value would match neither the generic nor the slot dispatch + // queries, silently stranding every frame of the layer. + throw new SpecBuilderException("Error, invalid slots_required " + + layer.slotsRequired + " on layer " + layer.name + + ", must be 0 (not slot-based) or a positive slot count."); + } + if (layer.slotsRequired > 0) { + // Slot-based layers are forced non-threadable: a slot frame reserves + // exactly its declared slots, never a variable core count. + layer.isThreadable = false; + } } /* diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V50__Enforce_slot_proc_booking.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V50__Enforce_slot_proc_booking.sql new file mode 100644 index 0000000000..41db5f61e0 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V50__Enforce_slot_proc_booking.sql @@ -0,0 +1,31 @@ +-- Slot-based procs legitimately reserve 0 cores (they book by concurrency slots +-- instead), so relax the 0-core guard for them. At the same time make the database +-- the hard enforcement point for the per-host concurrent slots cap: the insert takes +-- a row lock on the host, which serializes slot bookings per host, then verifies +-- SUM(proc.int_slots_reserved) + NEW.int_slots_reserved stays within +-- host.int_concurrent_slots_limit. This also guarantees strict pairing on the host +-- side: a slot proc can only ever be inserted for a slot-based host. +CREATE OR REPLACE FUNCTION trigger__before_insert_proc() +RETURNS TRIGGER AS $body$ +DECLARE + slot_limit INT; + slots_in_use INT; +BEGIN + IF NEW.int_slots_reserved > 0 THEN + SELECT int_concurrent_slots_limit INTO slot_limit FROM host + WHERE pk_host = NEW.pk_host FOR UPDATE; + IF slot_limit IS NULL OR slot_limit < 0 THEN + RAISE EXCEPTION 'failed to allocate slots, host is not slot-based'; + END IF; + SELECT COALESCE(SUM(int_slots_reserved), 0) INTO slots_in_use FROM proc + WHERE pk_host = NEW.pk_host; + IF slots_in_use + NEW.int_slots_reserved > slot_limit THEN + RAISE EXCEPTION 'failed to allocate slots, host is at its concurrent slots limit'; + END IF; + ELSIF NEW.int_cores_reserved <= 0 THEN + RAISE EXCEPTION 'failed to allocate proc, tried to allocate 0 cores'; + END IF; + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 18be8179f9..3bd65c8ad1 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -361,6 +361,12 @@ + + + + + + @@ -388,6 +394,7 @@ + @@ -405,6 +412,7 @@ + diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/SlotDispatcherTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/SlotDispatcherTests.java new file mode 100644 index 0000000000..893663b3b1 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/SlotDispatcherTests.java @@ -0,0 +1,377 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.test.dispatcher; + +import java.io.File; +import java.util.List; +import javax.annotation.Resource; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.test.annotation.Rollback; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.transaction.annotation.Transactional; + +import com.imageworks.spcue.DispatchFrame; +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.FrameDetail; +import com.imageworks.spcue.JobDetail; +import com.imageworks.spcue.VirtualProc; +import com.imageworks.spcue.dao.FrameDao; +import com.imageworks.spcue.dao.HostDao; +import com.imageworks.spcue.dao.JobDao; +import com.imageworks.spcue.dao.ProcDao; +import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.dispatcher.DispatchSupport; +import com.imageworks.spcue.dispatcher.ResourceReservationFailureException; +import com.imageworks.spcue.grpc.host.HardwareState; +import com.imageworks.spcue.grpc.job.FrameState; +import com.imageworks.spcue.grpc.report.RenderHost; +import com.imageworks.spcue.service.AdminManager; +import com.imageworks.spcue.service.HostManager; +import com.imageworks.spcue.service.JobLauncher; +import com.imageworks.spcue.service.JobManager; +import com.imageworks.spcue.test.TransactionalTest; +import com.imageworks.spcue.util.CueUtil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@ContextConfiguration +public class SlotDispatcherTests extends TransactionalTest { + + @Resource + JobManager jobManager; + + @Resource + JobLauncher jobLauncher; + + @Resource + HostManager hostManager; + + @Resource + AdminManager adminManager; + + @Resource + HostDao hostDao; + + @Resource + JobDao jobDao; + + @Resource + ProcDao procDao; + + @Resource + FrameDao frameDao; + + @Resource + Dispatcher slotDispatcher; + + @Resource + Dispatcher dispatcher; + + @Resource + DispatchSupport dispatchSupport; + + private static final String HOSTNAME = "slot_beta"; + + private static final String SLOT_JOB = "pipe-dev.cue-testuser_slot_test_v1"; + + private static final String HEAVY_SLOT_JOB = "pipe-dev.cue-testuser_slot_test_heavy"; + + private static final String REGULAR_JOB = "pipe-dev.cue-testuser_shell_dispatch_test_v1"; + + @Before + public void launchJob() { + jobLauncher.testMode = true; + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_slot_test.xml")); + jobLauncher.launch(new File("src/test/resources/conf/jobspec/jobspec_dispatch_test.xml")); + } + + @Before + public void setTestMode() { + slotDispatcher.setTestMode(true); + dispatcher.setTestMode(true); + } + + @Before + public void createHost() { + RenderHost host = RenderHost.newBuilder().setName(HOSTNAME).setBootTime(1192369572) + // The minimum amount of free space in the temporary directory to book a host. + .setFreeMcp(CueUtil.GB).setFreeMem(53500).setFreeSwap(20760).setLoad(1) + .setTotalMcp(CueUtil.GB4).setTotalMem(8173264).setTotalSwap(20960) + .setNimbyEnabled(false).setNumProcs(1).setCoresPerProc(100).addTags("test") + .setState(HardwareState.UP).setFacility("spi").putAttributes("SP_OS", "Linux") + .build(); + + hostManager.createHost(host, adminManager.findAllocationDetail("spi", "general")); + } + + private DispatchHost getSlotHost(int concurrentSlotsLimit) { + DispatchHost host = hostManager.findDispatchHost(HOSTNAME); + hostDao.updateConcurrentSlotsLimit(host, concurrentSlotsLimit); + return hostManager.findDispatchHost(HOSTNAME); + } + + private JobDetail getSlotJob() { + return jobManager.findJobDetail(SLOT_JOB); + } + + private JobDetail getHeavySlotJob() { + return jobManager.findJobDetail(HEAVY_SLOT_JOB); + } + + private JobDetail getRegularJob() { + return jobManager.findJobDetail(REGULAR_JOB); + } + + @Test + @Transactional + @Rollback(true) + public void testSlotHostFieldsMapped() { + DispatchHost host = getSlotHost(4); + assertTrue(host.isSlotHost()); + assertEquals(4, host.concurrentSlotsLimit); + assertEquals(4, host.idleSlots); + + hostDao.updateConcurrentSlotsLimit(host, -1); + DispatchHost regularHost = hostManager.findDispatchHost(HOSTNAME); + assertFalse(regularHost.isSlotHost()); + assertEquals(-1, regularHost.idleSlots); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchSlotHostToJob() { + DispatchHost host = getSlotHost(4); + JobDetail job = getSlotJob(); + + // job_frame_dispatch_max is 3 in the test properties, the host has 4 idle slots. + List procs = slotDispatcher.dispatchHost(host, job); + assertEquals(3, procs.size()); + + for (VirtualProc proc : procs) { + assertEquals(1, proc.slotsReserved); + assertEquals(0, proc.coresReserved); + assertEquals(0, proc.memoryReserved); + assertEquals(0, proc.gpusReserved); + + VirtualProc stored = procDao.getVirtualProc(proc.getProcId()); + assertEquals(1, stored.slotsReserved); + assertEquals(0, stored.coresReserved); + + FrameDetail frame = frameDao.getFrameDetail(proc.getFrameId()); + assertEquals(FrameState.RUNNING, frame.state); + } + + // In-memory accounting was decremented and the database derives the same value + // from SUM(proc.int_slots_reserved). + assertEquals(1, host.idleSlots); + assertEquals(1, hostManager.findDispatchHost(HOSTNAME).idleSlots); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchSlotHostCapsAtIdleSlots() { + DispatchHost host = getSlotHost(2); + JobDetail job = getSlotJob(); + + List procs = slotDispatcher.dispatchHost(host, job); + assertEquals(2, procs.size()); + assertEquals(0, host.idleSlots); + + // The host is full; another dispatch books nothing. + DispatchHost fullHost = hostManager.findDispatchHost(HOSTNAME); + assertEquals(0, fullHost.idleSlots); + assertEquals(0, slotDispatcher.dispatchHost(fullHost, job).size()); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchHeavySlotFrames() { + DispatchHost host = getSlotHost(4); + JobDetail job = getHeavySlotJob(); + + // Each frame requires 3 slots; only one fits in 4. + List procs = slotDispatcher.dispatchHost(host, job); + assertEquals(1, procs.size()); + assertEquals(3, procs.get(0).slotsReserved); + assertEquals(1, host.idleSlots); + + // A second frame (3 slots) no longer fits the remaining 1 slot. + DispatchHost refreshed = hostManager.findDispatchHost(HOSTNAME); + assertEquals(1, refreshed.idleSlots); + assertEquals(0, slotDispatcher.dispatchHost(refreshed, job).size()); + } + + @Test + @Transactional + @Rollback(true) + public void testDispatchSlotHost() { + DispatchHost host = getSlotHost(4); + + // The full host dispatch finds slot jobs on its own; the regular job is + // never considered. + List procs = slotDispatcher.dispatchHost(host); + assertTrue(procs.size() > 0); + for (VirtualProc proc : procs) { + assertTrue(proc.slotsReserved > 0); + assertEquals(0, proc.coresReserved); + assertFalse(getRegularJob().getJobId().equals(proc.getJobId())); + } + } + + @Test + @Transactional + @Rollback(true) + public void testSlotHostDoesNotBookRegularJob() { + DispatchHost host = getSlotHost(4); + JobDetail regularJob = getRegularJob(); + + // Strict pairing: a slot host never runs a regular (cores/memory) layer. + assertEquals(0, slotDispatcher.dispatchHost(host, regularJob).size()); + } + + @Test + @Transactional + @Rollback(true) + public void testRegularHostDoesNotBookSlotJob() { + // The host stays a regular host (no slot limit). + DispatchHost host = hostManager.findDispatchHost(HOSTNAME); + JobDetail slotJob = getSlotJob(); + + // Strict pairing: the generic dispatcher never books slot-based layers. + assertEquals(0, dispatcher.dispatchHost(host, slotJob).size()); + + // And the generic full-host dispatch only books the regular job. + List procs = dispatcher.dispatchHost(host); + for (VirtualProc proc : procs) { + assertEquals(0, proc.slotsReserved); + assertFalse(getSlotJob().getJobId().equals(proc.getJobId())); + assertFalse(getHeavySlotJob().getJobId().equals(proc.getJobId())); + } + } + + @Test + @Transactional + @Rollback(true) + public void testJobMaxSlotsCap() { + DispatchHost host = getSlotHost(4); + JobDetail job = getSlotJob(); + + jobDao.updateMaxSlots(job, 2); + List procs = slotDispatcher.dispatchHost(host, job); + assertEquals(2, procs.size()); + + // The job is at its cap; nothing else books. + DispatchHost refreshed = hostManager.findDispatchHost(HOSTNAME); + assertEquals(0, slotDispatcher.dispatchHost(refreshed, job).size()); + } + + @Test + @Transactional + @Rollback(true) + public void testJobMaxSlotsRejectAll() { + DispatchHost host = getSlotHost(4); + JobDetail job = getSlotJob(); + + jobDao.updateMaxSlots(job, 0); + assertEquals(0, slotDispatcher.dispatchHost(host, job).size()); + } + + @Test + @Transactional + @Rollback(true) + public void testFolderMaxSlotsCap() { + DispatchHost host = getSlotHost(4); + JobDetail job = getSlotJob(); + + jdbcTemplate.update("UPDATE folder_resource SET int_max_slots=2 WHERE " + + "pk_folder=(SELECT pk_folder FROM job WHERE pk_job=?)", job.getJobId()); + + List procs = slotDispatcher.dispatchHost(host, job); + assertEquals(2, procs.size()); + + DispatchHost refreshed = hostManager.findDispatchHost(HOSTNAME); + assertEquals(0, slotDispatcher.dispatchHost(refreshed, job).size()); + } + + @Test + @Transactional + @Rollback(true) + public void testSubscriptionMaxSlotsCap() { + DispatchHost host = getSlotHost(4); + JobDetail job = getSlotJob(); + + jdbcTemplate.update( + "UPDATE subscription SET int_max_slots=2 WHERE " + + "pk_show=(SELECT pk_show FROM job WHERE pk_job=?) AND pk_alloc=?", + job.getJobId(), host.getAllocationId()); + + List procs = slotDispatcher.dispatchHost(host, job); + assertEquals(2, procs.size()); + + DispatchHost refreshed = hostManager.findDispatchHost(HOSTNAME); + assertEquals(0, slotDispatcher.dispatchHost(refreshed, job).size()); + } + + @Test + @Transactional + @Rollback(true) + public void testTriggerRejectsBookingOverHostCap() { + DispatchHost host = getSlotHost(1); + JobDetail job = getSlotJob(); + + List frames = dispatchSupport.findNextSlotDispatchFrames(job, host, 10); + assertTrue(frames.size() >= 2); + + slotDispatcher.dispatch(frames.get(0), VirtualProc.buildSlotProc(host, frames.get(0))); + + /* + * The host cap is spent; a direct insert bypassing the dispatcher's in-memory accounting + * must be rejected by the before_insert_proc trigger. The raised database exception aborts + * the test transaction, so this is the last operation of the test. + */ + VirtualProc over = VirtualProc.buildSlotProc(host, frames.get(1)); + over.frameId = frames.get(1).getFrameId(); + try { + procDao.insertVirtualProc(over); + fail("Expected the before_insert_proc trigger to reject booking over the host cap."); + } catch (ResourceReservationFailureException expected) { + // Expected: host is at its concurrent slots limit. + } + } + + @Test + @Transactional + @Rollback(true) + public void testUnbookSlotProcReturnsSlots() { + DispatchHost host = getSlotHost(2); + JobDetail job = getSlotJob(); + + List procs = slotDispatcher.dispatchHost(host, job); + assertEquals(2, procs.size()); + assertEquals(0, hostManager.findDispatchHost(HOSTNAME).idleSlots); + + dispatchSupport.unbookProc(procs.get(0)); + assertEquals(1, hostManager.findDispatchHost(HOSTNAME).idleSlots); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/service/AccountingNotifierTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/service/AccountingNotifierTests.java index eee5fd5216..019040c535 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/service/AccountingNotifierTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/service/AccountingNotifierTests.java @@ -93,9 +93,25 @@ public void notifyReleaseEmitsNegatedDelta() { notifier.notifyRelease(proc); // Cores converted to cores (400/100=4) then negated; gpus pass through then negated. + // Regular procs hold no slots, so the slot delta is 0. String expected = "{\"show\":\"show-uuid\",\"alloc\":\"alloc-uuid\"," + "\"folder\":\"folder-uuid\",\"job\":\"job-uuid\",\"layer\":\"layer-uuid\"," - + "\"dept\":\"dept-uuid\",\"cores\":-4,\"gpus\":-2}"; + + "\"dept\":\"dept-uuid\",\"cores\":-4,\"gpus\":-2,\"slots\":0}"; + verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_RELEASE), eq(expected)); + } + + @Test + public void notifyReleaseEmitsNegatedSlotDelta() { + // A slot-based proc reserves 0 cores/gpus and a positive slot count. + proc.coresReserved = 0; + proc.gpusReserved = 0; + proc.slotsReserved = 3; + + notifier.notifyRelease(proc); + + String expected = "{\"show\":\"show-uuid\",\"alloc\":\"alloc-uuid\"," + + "\"folder\":\"folder-uuid\",\"job\":\"job-uuid\",\"layer\":\"layer-uuid\"," + + "\"dept\":\"dept-uuid\",\"cores\":0,\"gpus\":0,\"slots\":-3}"; verify(jdbcTemplate).queryForList(eq(NOTIFY_SQL), eq(CHANNEL_RELEASE), eq(expected)); } diff --git a/cuebot/src/test/resources/conf/dtd/cjsl-1.16.dtd b/cuebot/src/test/resources/conf/dtd/cjsl-1.16.dtd new file mode 100644 index 0000000000..20ebf19c21 --- /dev/null +++ b/cuebot/src/test/resources/conf/dtd/cjsl-1.16.dtd @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/cuebot/src/test/resources/conf/jobspec/jobspec_slot_test.xml b/cuebot/src/test/resources/conf/jobspec/jobspec_slot_test.xml new file mode 100644 index 0000000000..bd59ee7ae6 --- /dev/null +++ b/cuebot/src/test/resources/conf/jobspec/jobspec_slot_test.xml @@ -0,0 +1,69 @@ + + + + + + + + + + spi + Lighting + pipe + dev.cue + testuser + 9860 + + + false + Linux + + + + /shots/pipe/usr_testuser/logs/help.py + 1-10 + 1 + 1 + general + + shell + + + + + + + + false + Linux + + + + /shots/pipe/usr_testuser/logs/help.py + 1-10 + 1 + 3 + general + + shell + + + + + + + diff --git a/docs/_docs/concepts/slot-based-booking.md b/docs/_docs/concepts/slot-based-booking.md index 365a27fa21..da831f5ab8 100644 --- a/docs/_docs/concepts/slot-based-booking.md +++ b/docs/_docs/concepts/slot-based-booking.md @@ -20,9 +20,12 @@ A slot-based frame ignores cores and memory entirely: it reserves **0 cores and memory** and runs unpinned on the host. The only thing that limits it is a **slot budget**. -> **Note:** Slot-based booking is implemented in the standalone Rust scheduler. Cuebot -> does not make slot-based booking decisions; it stores the configuration and publishes -> slot release/limit deltas to the scheduler. +> **Note:** Slot-based booking is implemented both in the standalone Rust scheduler and +> in Cuebot's dispatcher. In Cuebot, host reports from slot-based hosts are deviated to a +> dedicated slot dispatcher that books by slots only; the generic cores/memory dispatch +> queries are untouched and never see slot-based layers. On shows managed by the Rust +> scheduler, the scheduler owns slot booking and Cuebot only stores the configuration and +> publishes slot release/limit deltas. ## The two slot axes @@ -61,6 +64,19 @@ slot axis: a seeding bug manifests as "slot work won't book," never as overrunni cap. Regular (cores/memory) layers are unaffected by `max_slots`, and slot layers are unaffected by the cores/GPUs limits. +### Enforcement strength (Cuebot dispatcher) + +When Cuebot dispatches slot work, the two axes are enforced with different strength: + +- **Per-host cap** — **hard**. A database trigger on proc insert takes a row lock on the + host and verifies `SUM(proc.int_slots_reserved) + new slots <= concurrent_slots_limit`, + so concurrent bookings (including from the Rust scheduler) can never exceed a host's cap. +- **Subscription / folder / job `max_slots`** — **best-effort**. The caps are checked in + the dispatch queries at query time and re-checked against the bookings made within the + same dispatch pass, but two Cuebot instances (or two hosts dispatched concurrently) can + jointly overrun a cap by up to one pass's bookings each. The overrun self-corrects as + frames complete; treat these caps as strong steering, not invariants. + ## Making a layer slot-based Set `slots_required` on the layer in the job spec (spec version **1.16**+). With diff --git a/rust/crates/rqd/src/system/machine.rs b/rust/crates/rqd/src/system/machine.rs index 0856a9fa04..e05f850abf 100644 --- a/rust/crates/rqd/src/system/machine.rs +++ b/rust/crates/rqd/src/system/machine.rs @@ -561,13 +561,17 @@ impl MachineMonitor { let frame_report = frame.clone_into_running_frame_info(); info!("Sending frame complete report: {}", frame); - if let Err(err) = self.release_cores(&frame.request.resource_id()).await { - warn!( - "Failed to release cores reserved by {}: {}", - frame.request.resource_id(), - err - ); - }; + // Slot-based frames reserve 0 cores and therefore have no core booking to + // release; skip the release to avoid a spurious ReservationNotFound warning. + if frame.request.num_cores > 0 { + if let Err(err) = self.release_cores(&frame.request.resource_id()).await { + warn!( + "Failed to release cores reserved by {}: {}", + frame.request.resource_id(), + err + ); + }; + } // Send complete report if let Err(err) = self diff --git a/rust/crates/scheduler/src/dao/host_dao.rs b/rust/crates/scheduler/src/dao/host_dao.rs index e016f2f637..71cfba65bf 100644 --- a/rust/crates/scheduler/src/dao/host_dao.rs +++ b/rust/crates/scheduler/src/dao/host_dao.rs @@ -170,7 +170,7 @@ SELECT DISTINCT a.pk_alloc, a.str_name as str_alloc_name, hs.ts_ping, - h.int_concurrent_slots_limit, + h.int_concurrent_slots_limit::bigint as int_concurrent_slots_limit, COALESCE(( SELECT SUM(p.int_slots_reserved) FROM proc p WHERE p.pk_host = h.pk_host ), 0)::bigint as int_running_slots From c05c733dd1dec099d9ef24d96f9eaf740488a2a9 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Tue, 11 Aug 2026 08:34:08 -0700 Subject: [PATCH 20/20] [cuebot] Consolidate slot-booking migrations into a single V47 Upstream master took V46 (orphaned frame check task lock), which collided with this branch's V46__Add_layer_slots_required. Fold the five slot migrations (layer slots_required, host concurrent_slots_limit, proc slots_reserved, sub/folder/job max_slots, slot-aware proc insert trigger) into one V47__Add_slot_based_booking migration. Co-Authored-By: Claude Fable 5 --- .../V46__Add_layer_slots_required.sql | 4 -- .../V47__Add_host_concurrent_slots_limit.sql | 5 -- .../V47__Add_slot_based_booking.sql | 65 +++++++++++++++++++ .../V48__Add_proc_slots_reserved.sql | 6 -- .../migrations/V49__Add_slot_limits.sql | 12 ---- .../V50__Enforce_slot_proc_booking.sql | 31 --------- 6 files changed, 65 insertions(+), 58 deletions(-) delete mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_slots_required.sql delete mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_host_concurrent_slots_limit.sql create mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_slot_based_booking.sql delete mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V48__Add_proc_slots_reserved.sql delete mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V49__Add_slot_limits.sql delete mode 100644 cuebot/src/main/resources/conf/ddl/postgres/migrations/V50__Enforce_slot_proc_booking.sql diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_slots_required.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_slots_required.sql deleted file mode 100644 index 9310fc6615..0000000000 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_slots_required.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Mark a layer as slot-based by requiring at least this many concurrency slots per frame. --- 0 means the layer is not slot-based and books by cores/memory as usual. -alter table layer - add int_slots_required INT DEFAULT 0 NOT NULL; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_host_concurrent_slots_limit.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_host_concurrent_slots_limit.sql deleted file mode 100644 index 4f46634db5..0000000000 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_host_concurrent_slots_limit.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Limit the max number of concurrent frames a host may run (slot-based host). --- -1 means the host is not slot-based and books by cores/memory as usual. --- When >= 0 the host only runs slot-based layers, capped at this many concurrent slots. -alter table host - add int_concurrent_slots_limit INT DEFAULT -1 NOT NULL; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_slot_based_booking.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_slot_based_booking.sql new file mode 100644 index 0000000000..aec686ead3 --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V47__Add_slot_based_booking.sql @@ -0,0 +1,65 @@ +-- Slot-based booking: layers and hosts opt into concurrency-slot dispatch, where frames +-- book a whole-count slot instead of cores/memory. + +-- Mark a layer as slot-based by requiring at least this many concurrency slots per frame. +-- 0 means the layer is not slot-based and books by cores/memory as usual. +alter table layer + add int_slots_required INT DEFAULT 0 NOT NULL; + +-- Limit the max number of concurrent frames a host may run (slot-based host). +-- -1 means the host is not slot-based and books by cores/memory as usual. +-- When >= 0 the host only runs slot-based layers, capped at this many concurrent slots. +alter table host + add int_concurrent_slots_limit INT DEFAULT -1 NOT NULL; + +-- Number of concurrency slots reserved by a proc (booked frame) on a slot-based host. +-- 0 for regular (cores/memory) procs. This is the single source of truth for slot +-- accounting: per-host and per subscription/folder/job slot usage both derive from +-- SUM(proc.int_slots_reserved). +alter table proc + add int_slots_reserved INT DEFAULT 0 NOT NULL; + +-- Per-hierarchy hard limit on concurrent slots for slot-based layers, parallel to the +-- cores/gpus limits. Enforced by the scheduler accounting store at subscription, folder +-- and job level. -1 means unlimited; 0 means reject all slot work; N caps at N slots. +-- Regular (cores/memory) layers are unaffected by this limit. +alter table subscription + add int_max_slots INT DEFAULT -1 NOT NULL; + +alter table folder_resource + add int_max_slots INT DEFAULT -1 NOT NULL; + +alter table job_resource + add int_max_slots INT DEFAULT -1 NOT NULL; + +-- Slot-based procs legitimately reserve 0 cores (they book by concurrency slots +-- instead), so relax the 0-core guard for them. At the same time make the database +-- the hard enforcement point for the per-host concurrent slots cap: the insert takes +-- a row lock on the host, which serializes slot bookings per host, then verifies +-- SUM(proc.int_slots_reserved) + NEW.int_slots_reserved stays within +-- host.int_concurrent_slots_limit. This also guarantees strict pairing on the host +-- side: a slot proc can only ever be inserted for a slot-based host. +CREATE OR REPLACE FUNCTION trigger__before_insert_proc() +RETURNS TRIGGER AS $body$ +DECLARE + slot_limit INT; + slots_in_use INT; +BEGIN + IF NEW.int_slots_reserved > 0 THEN + SELECT int_concurrent_slots_limit INTO slot_limit FROM host + WHERE pk_host = NEW.pk_host FOR UPDATE; + IF slot_limit IS NULL OR slot_limit < 0 THEN + RAISE EXCEPTION 'failed to allocate slots, host is not slot-based'; + END IF; + SELECT COALESCE(SUM(int_slots_reserved), 0) INTO slots_in_use FROM proc + WHERE pk_host = NEW.pk_host; + IF slots_in_use + NEW.int_slots_reserved > slot_limit THEN + RAISE EXCEPTION 'failed to allocate slots, host is at its concurrent slots limit'; + END IF; + ELSIF NEW.int_cores_reserved <= 0 THEN + RAISE EXCEPTION 'failed to allocate proc, tried to allocate 0 cores'; + END IF; + RETURN NEW; +END; +$body$ +LANGUAGE PLPGSQL; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V48__Add_proc_slots_reserved.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V48__Add_proc_slots_reserved.sql deleted file mode 100644 index a667f3192b..0000000000 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V48__Add_proc_slots_reserved.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Number of concurrency slots reserved by a proc (booked frame) on a slot-based host. --- 0 for regular (cores/memory) procs. This is the single source of truth for slot --- accounting: per-host and per subscription/folder/job slot usage both derive from --- SUM(proc.int_slots_reserved). -alter table proc - add int_slots_reserved INT DEFAULT 0 NOT NULL; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V49__Add_slot_limits.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V49__Add_slot_limits.sql deleted file mode 100644 index 806dfe2676..0000000000 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V49__Add_slot_limits.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Per-hierarchy hard limit on concurrent slots for slot-based layers, parallel to the --- cores/gpus limits. Enforced by the scheduler accounting store at subscription, folder --- and job level. -1 means unlimited; 0 means reject all slot work; N caps at N slots. --- Regular (cores/memory) layers are unaffected by this limit. -alter table subscription - add int_max_slots INT DEFAULT -1 NOT NULL; - -alter table folder_resource - add int_max_slots INT DEFAULT -1 NOT NULL; - -alter table job_resource - add int_max_slots INT DEFAULT -1 NOT NULL; diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V50__Enforce_slot_proc_booking.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V50__Enforce_slot_proc_booking.sql deleted file mode 100644 index 41db5f61e0..0000000000 --- a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V50__Enforce_slot_proc_booking.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Slot-based procs legitimately reserve 0 cores (they book by concurrency slots --- instead), so relax the 0-core guard for them. At the same time make the database --- the hard enforcement point for the per-host concurrent slots cap: the insert takes --- a row lock on the host, which serializes slot bookings per host, then verifies --- SUM(proc.int_slots_reserved) + NEW.int_slots_reserved stays within --- host.int_concurrent_slots_limit. This also guarantees strict pairing on the host --- side: a slot proc can only ever be inserted for a slot-based host. -CREATE OR REPLACE FUNCTION trigger__before_insert_proc() -RETURNS TRIGGER AS $body$ -DECLARE - slot_limit INT; - slots_in_use INT; -BEGIN - IF NEW.int_slots_reserved > 0 THEN - SELECT int_concurrent_slots_limit INTO slot_limit FROM host - WHERE pk_host = NEW.pk_host FOR UPDATE; - IF slot_limit IS NULL OR slot_limit < 0 THEN - RAISE EXCEPTION 'failed to allocate slots, host is not slot-based'; - END IF; - SELECT COALESCE(SUM(int_slots_reserved), 0) INTO slots_in_use FROM proc - WHERE pk_host = NEW.pk_host; - IF slots_in_use + NEW.int_slots_reserved > slot_limit THEN - RAISE EXCEPTION 'failed to allocate slots, host is at its concurrent slots limit'; - END IF; - ELSIF NEW.int_cores_reserved <= 0 THEN - RAISE EXCEPTION 'failed to allocate proc, tried to allocate 0 cores'; - END IF; - RETURN NEW; -END; -$body$ -LANGUAGE PLPGSQL;