diff --git a/VERSION.in b/VERSION.in index e0250ea3c7..9de53f1932 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -1.28 +1.29 diff --git a/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java b/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java index ab03ec6a8c..6a9f86124c 100644 --- a/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java +++ b/cuebot/src/main/java/com/imageworks/common/spring/remoting/GrpcServer.java @@ -28,6 +28,7 @@ import com.imageworks.spcue.servant.ManageHost; import com.imageworks.spcue.servant.ManageJob; import com.imageworks.spcue.servant.ManageLayer; +import com.imageworks.spcue.servant.ManageLicense; import com.imageworks.spcue.servant.ManageLimit; import com.imageworks.spcue.servant.ManageMatcher; import com.imageworks.spcue.servant.ManageOwner; @@ -110,6 +111,7 @@ public void start() throws IOException { .addService(applicationContext.getBean("manageHost", ManageHost.class)) .addService(applicationContext.getBean("manageJob", ManageJob.class)) .addService(applicationContext.getBean("manageLayer", ManageLayer.class)) + .addService(applicationContext.getBean("manageLicense", ManageLicense.class)) .addService(applicationContext.getBean("manageLimit", ManageLimit.class)) .addService(applicationContext.getBean("manageMatcher", ManageMatcher.class)) .addService(applicationContext.getBean("manageOwner", ManageOwner.class)) diff --git a/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java b/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java index f5bfcc347f..1c0c512b4a 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java +++ b/cuebot/src/main/java/com/imageworks/spcue/config/AppConfig.java @@ -28,6 +28,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.context.annotation.ImportResource; import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.PropertySource; @@ -42,6 +43,7 @@ "classpath:conf/spring/applicationContext-monitoring.xml", "classpath:conf/spring/applicationContext-accounting.xml"}) @EnableConfigurationProperties +@Import(LicenseConfig.class) @PropertySource({"classpath:opencue.properties"}) public class AppConfig { diff --git a/cuebot/src/main/java/com/imageworks/spcue/config/LicenseConfig.java b/cuebot/src/main/java/com/imageworks/spcue/config/LicenseConfig.java new file mode 100644 index 0000000000..a9c320f6e2 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/config/LicenseConfig.java @@ -0,0 +1,52 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.config; + +import javax.sql.DataSource; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.imageworks.spcue.dispatcher.LicenseBookingGate; +import com.imageworks.spcue.dispatcher.LicenseSource; + +/** + * Live application licensing (CUE_LICENSES) beans, shared between the main and test contexts so the + * XML-defined dispatcher beans can reference them in both. Everything here is inert unless + * scheduler.license.provider is configured AND a layer declares licenses in its environment. + */ +@Configuration +public class LicenseConfig { + + /** + * Live view of floating application licenses. The poller is a no-op unless + * scheduler.license.provider is configured; a layer declaring licenses in its environment is + * what turns gating on. + */ + @Bean(initMethod = "start", destroyMethod = "stop") + public LicenseSource licenseSource(Environment env, DataSource cueDataSource) { + return new LicenseSource(env, new JdbcTemplate(cueDataSource)); + } + + /** Applies the license budgets to the legacy booking path. */ + @Bean + public LicenseBookingGate licenseBookingGate(LicenseSource licenseSource, + DataSource cueDataSource) { + return new LicenseBookingGate(new JdbcTemplate(cueDataSource), licenseSource); + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java index 7b3cd911fa..0bb117d434 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java @@ -85,6 +85,12 @@ public class CoreUnitDispatcher implements Dispatcher { private HostManager hostManager; + /** + * Gates dispatch on live application-license budgets (CUE_LICENSES). Optional: when unset, + * dispatch behaves exactly as before. + */ + private LicenseBookingGate licenseBookingGate; + public boolean testMode = false; private final long MEM_RESERVED_MIN; @@ -135,6 +141,10 @@ private Cache getOrCreateJobLock() { private List dispatchJobs(DispatchHost host, Set jobs) { List procs = new ArrayList(); + // One license session for the whole booking pass: its budget snapshot is + // read at most once and its pass-local accounting spans every job below. + LicenseBookingGate.Session licenseSession = + licenseBookingGate == null ? null : licenseBookingGate.newSession(host.getName()); try { for (String jobid : jobs) { @@ -157,7 +167,7 @@ private List dispatchJobs(DispatchHost host, Set jobs) { DispatchJob job = jobManager.getDispatchJob(jobid); try { - procs.addAll(dispatchHost(host, job)); + procs.addAll(dispatchHost(host, job, licenseSession)); } catch (JobDispatchException e) { logger.info("job dispatch exception," + e); } @@ -241,6 +251,12 @@ public List dispatchHost(DispatchHost host, GroupInterface group) { @Override public List dispatchHost(DispatchHost host, JobInterface job) { + return dispatchHost(host, job, + licenseBookingGate == null ? null : licenseBookingGate.newSession(host.getName())); + } + + private List dispatchHost(DispatchHost host, JobInterface job, + LicenseBookingGate.Session licenseSession) { List procs = new ArrayList(); @@ -258,6 +274,14 @@ public List dispatchHost(DispatchHost host, JobInterface job) { env.getProperty("dispatcher.frame.selfish.services", "").split(","); for (DispatchFrame frame : frames) { + // Hold frames whose layer needs an application license with no free + // seat; other layers of the job may still book, so skip, not break. + if (licenseSession != null && !licenseSession.canBook(frame.getLayerId())) { + logger.debug("Cannot dispatch frame " + frame.getName() + + ", no application license seat free."); + continue; + } + VirtualProc proc = VirtualProc.build(host, frame, selfishServices); if (frame.minCores <= 0 && !proc.canHandleNegativeCoresRequest) { @@ -291,6 +315,9 @@ public void wrapDispatchFrame() { }.execute(); if (success) { + if (licenseSession != null) { + licenseSession.booked(frame.getLayerId()); + } procs.add(proc); DispatchSupport.bookedProcs.getAndIncrement(); @@ -322,9 +349,14 @@ public void wrapDispatchFrame() { public void dispatchProcToJob(VirtualProc proc, JobInterface job) { + LicenseBookingGate.Session licenseSession = + licenseBookingGate == null ? null : licenseBookingGate.newSession(proc.hostName); // Do not throttle this method for (DispatchFrame frame : dispatchSupport.findNextDispatchFrames(job, proc, getIntProperty("dispatcher.frame_query_max"))) { + if (licenseSession != null && !licenseSession.canBook(frame.getLayerId())) { + continue; + } try { boolean success = new DispatchFrameTemplate(proc, job, frame, true) { public void wrapDispatchFrame() { @@ -422,6 +454,14 @@ public void setRqdClient(RqdClient rqdClient) { this.rqdClient = rqdClient; } + public LicenseBookingGate getLicenseBookingGate() { + return licenseBookingGate; + } + + public void setLicenseBookingGate(LicenseBookingGate licenseBookingGate) { + this.licenseBookingGate = licenseBookingGate; + } + private abstract class DispatchFrameTemplate { protected VirtualProc proc; protected JobInterface job; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index ce745f6186..16a1cd65e9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -16,10 +16,16 @@ package com.imageworks.spcue.dispatcher; import java.sql.Timestamp; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashSet; import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; @@ -129,11 +135,95 @@ public void setSatisfyDependOnlyOnFrameSuccess(boolean satisfyDependOnlyOnFrameS this.satisfyDependOnlyOnFrameSuccess = satisfyDependOnlyOnFrameSuccess; } + /** + * Exit statuses that mean "the application could not get a license", from + * {@code scheduler.license.denied_exit_statuses}. Vendor specific, so it is a site setting; + * EMPTY by default, which leaves frame-completion behaviour exactly as it was. + * + * Static because {@link #determineFrameState} is static and is the natural place for the + * decision. Written once when this bean is constructed, long before any report can arrive, and + * only read afterwards. + */ + private static volatile Set licenseDeniedStatuses = Collections.emptySet(); + + /** + * How many times one frame may be requeued for a license-denied exit without spending a retry, + * from {@code scheduler.license.denied_requeue_limit}. The booking gate holds licensed layers + * while their pool is full, so genuine denials are rare races; a frame denied over and over is + * misconfigured (wrong feature name, bad local license setup) and without a bound it would + * requeue forever, never marching to DEAD. Past the limit the vendor's own exit status is + * persisted again, so ordinary retry accounting takes over. Zero or negative means unbounded + * (the pre-limit behaviour). + */ + private static volatile int licenseDeniedRequeueLimit = 10; + + /** + * License-denied requeues per frame id, backing the limit above. Static for the same reason as + * {@link #licenseDeniedStatuses}. Entries expire so a frame retried much later starts fresh, + * and the size bound keeps a misbehaving farm from growing it without limit. + */ + private static final Cache licenseDeniedRequeues = CacheBuilder.newBuilder() + .maximumSize(50_000).expireAfterWrite(12, TimeUnit.HOURS).build(); + @Autowired public FrameCompleteHandler(Environment env) { this.env = env; satisfyDependOnlyOnFrameSuccess = env.getProperty("depend.satisfy_only_on_frame_success", Boolean.class, true); + licenseDeniedStatuses = + parseStatuses(env.getProperty("scheduler.license.denied_exit_statuses", "")); + licenseDeniedRequeueLimit = + env.getProperty("scheduler.license.denied_requeue_limit", Integer.class, 10); + if (!licenseDeniedStatuses.isEmpty()) { + logger.info("license-denied exit statuses (requeued without spending a retry, up to " + + licenseDeniedRequeueLimit + " times per frame): " + licenseDeniedStatuses); + } + } + + /** Parse a comma separated list of exit statuses, ignoring blanks and junk. */ + private static Set parseStatuses(String csv) { + if (csv == null || csv.trim().isEmpty()) + return Collections.emptySet(); + Set out = new HashSet<>(); + for (String part : csv.split(",")) { + String s = part.trim(); + if (s.isEmpty()) + continue; + try { + out.add(Integer.valueOf(s)); + } catch (NumberFormatException e) { + logger.warn("ignoring non-numeric scheduler.license.denied_exit_statuses entry '" + + s + "'"); + } + } + return out; + } + + /** + * Did this frame exit because no application license was free? Always false unless the site + * configured the statuses, so this cannot change behaviour on its own. + */ + private static boolean isLicenseDenied(int exitStatus) { + return licenseDeniedStatuses.contains(exitStatus); + } + + /** + * Does this frame still have license-denied requeue budget left? Read by + * {@link #determineFrameState} and by the exit-status persistence, both against the same count; + * {@link #countLicenseDeniedRequeue} moves the count afterwards, once per report. + */ + static boolean underLicenseDeniedLimit(String frameId) { + if (licenseDeniedRequeueLimit <= 0) { + return true; + } + Long count = licenseDeniedRequeues.getIfPresent(frameId); + return count == null || count < licenseDeniedRequeueLimit; + } + + /** Record one license-denied requeue for this frame. */ + static void countLicenseDeniedRequeue(String frameId) { + Long count = licenseDeniedRequeues.getIfPresent(frameId); + licenseDeniedRequeues.put(frameId, count == null ? 1L : count + 1); } /** @@ -170,7 +260,23 @@ public void handleFrameCompleteReport(final FrameCompleteReport report) { // not able to override what has been set by the previous logic. int exitStatus = report.getExitStatus(); if (frameDetail.exitStatus == Dispatcher.EXIT_STATUS_MEMORY_FAILURE) { + // The OOM pre-mark wins over everything, including a license-denied + // exit code: the frame was killed for memory, and storing + // MEMORY_FAILURE is what drives the retry-with-more-memory logic. exitStatus = frameDetail.exitStatus; + } else if (isLicenseDenied(exitStatus) + && underLicenseDeniedLimit(report.getFrame().getFrameId())) { + // A frame that died because no application license was free is + // requeued by determineFrameState above. Persist it as SKIP_RETRY so + // the retry counter is not incremented when it runs again (the + // increment reads the frame's STORED exit status, so recording the + // vendor's own code here would spend a retry on a queue wait). + // Bounded per frame: past the limit the vendor status is stored and + // ordinary retry accounting resumes (see licenseDeniedRequeueLimit). + logger.info("frame " + frame.getName() + " could not get a license (exit " + + exitStatus + "); requeueing without spending a retry"); + countLicenseDeniedRequeue(report.getFrame().getFrameId()); + exitStatus = FrameExitStatus.SKIP_RETRY_VALUE; } if (dispatchSupport.stopFrame(frame, newFrameState, exitStatus, @@ -623,7 +729,20 @@ else if (frame.state.equals(FrameState.DEAD)) { long lastUpdate = (r - report.getFrame().getLluTime()) / 60; FrameState newState = FrameState.WAITING; - if (report.getExitStatus() == FrameExitStatus.SKIP_RETRY_VALUE + if (isLicenseDenied(report.getExitStatus()) + && underLicenseDeniedLimit(frame.getFrameId())) { + // The application could not get a license. That is a resource + // being contended, not a broken frame: requeue it and do NOT + // burn a retry (the caller persists SKIP_RETRY for that), or a + // busy license pool would march every frame to DEAD. Headroom + // and the dispatcher's live gate are what avoid this; this + // catches the race they cannot -- an artist taking the last seat + // between the license sample and the actual checkout. Bounded + // per frame (underLicenseDeniedLimit): a frame denied over and + // over is misconfigured, not queue-unlucky, and falls back to + // ordinary retry accounting below. + newState = FrameState.WAITING; + } else if (report.getExitStatus() == FrameExitStatus.SKIP_RETRY_VALUE || (job.maxRetries != 0 && report.getExitSignal() == 119)) { report = FrameCompleteReport.newBuilder(report) .setExitStatus(FrameExitStatus.SKIP_RETRY_VALUE).build(); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 6277b5de34..bdaba0b973 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -28,11 +28,13 @@ import com.imageworks.spcue.LayerEntity; import com.imageworks.spcue.LocalHostAssignment; import com.imageworks.spcue.PrometheusMetricsCollector; +import com.imageworks.spcue.ShowInterface; import com.imageworks.spcue.Source; import com.imageworks.spcue.VirtualProc; import com.imageworks.spcue.dao.JobDao; import com.imageworks.spcue.dao.LayerDao; import com.imageworks.spcue.dispatcher.commands.DispatchBookHost; +import com.imageworks.spcue.dispatcher.commands.DispatchBookHostLicensePack; import com.imageworks.spcue.dispatcher.commands.DispatchBookHostLocal; import com.imageworks.spcue.dispatcher.commands.DispatchHandleHostReport; import com.imageworks.spcue.dispatcher.commands.DispatchRqdKillFrame; @@ -57,6 +59,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -91,6 +94,7 @@ public class HostReportHandler { private LayerDao layerDao; private KafkaEventPublisher kafkaEventPublisher; private MonitoringEventBuilder monitoringEventBuilder; + private LicenseBookingGate licenseBookingGate; @Autowired private Environment env; @@ -341,6 +345,26 @@ public void handleHostReport(HostReport report, boolean isBoot) { return; } + /* + * License packing: when this host is already running frames that hold host-based + * application licenses, try to book some frames which require the same license to + * keep them from booking a new host and consuming more licenses. Only the cheap + * query runs on this thread (cached layer lookups); the budget snapshot, the + * pack-job query and the dispatching all happen on a booking thread. The pack + * command ends with the exact booking the host would have received here (preferred + * show included), so nothing is lost if packing finds no work. + */ + if (licenseBookingGate != null + && env.getProperty("scheduler.license.pack_jobs_max", Integer.class, 5) > 0 + && licenseBookingGate.anyLicensedLayers(runningFrames)) { + ShowInterface preferredShow = + hostManager.isPreferShow(host) ? hostManager.getPreferredShow(host) + : null; + bookingQueue.execute(new DispatchBookHostLicensePack(host, runningFrames, + preferredShow, licenseBookingGate, jobManager, dispatcher, env)); + return; + } + /* * Check if the host prefers a show. If it does , dispatch to that show first. */ @@ -1227,6 +1251,14 @@ public void setMonitoringEventBuilder(MonitoringEventBuilder monitoringEventBuil this.monitoringEventBuilder = monitoringEventBuilder; } + public LicenseBookingGate getLicenseBookingGate() { + return licenseBookingGate; + } + + public void setLicenseBookingGate(LicenseBookingGate licenseBookingGate) { + this.licenseBookingGate = licenseBookingGate; + } + /** * Publishes a host state change event to Kafka for monitoring purposes. */ diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LicenseBookingGate.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LicenseBookingGate.java new file mode 100644 index 0000000000..cb5f54f823 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LicenseBookingGate.java @@ -0,0 +1,315 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.grpc.report.RunningFrameInfo; + +/** + * Applies {@link LicenseSource} budgets to the legacy (Cuebot) booking path. + * + * The dispatch queries in {@code DispatchQuery} are deliberately not touched: license state lives + * in this process (a poll of the license server), not in the database, so the gate runs in Java on + * the candidate frames those queries return. A frame whose layer declares licenses (via the + * {@code CUE_LICENSES} layer environment) is only dispatched while every pool it lists has a free + * seat; everything else passes through untouched and pays only a cache lookup. + * + * Two entry points: + * + *
    + *
  • {@link Session}: per dispatch pass, filters candidate {@code DispatchFrame}s against a budget + * snapshot, with pass-local accounting so one pass cannot book fifty frames against ten seats. + * Cross-Cuebot and cross-host races are absorbed by the in-flight correction inside + * {@link LicenseSource} (booked frames are RUNNING in the DB immediately) plus headroom, and the + * license-denied requeue in {@code FrameCompleteHandler} catches the remainder.
    • + *
  • the packing helpers ({@link #hostBasedLicensesRunning}, {@link #findPackableJobs}), used by + * {@code HostReportHandler} to steer layers that need a host-based license onto hosts already + * holding that license.
    • + *
+ */ +public class LicenseBookingGate { + + private static final Logger logger = LogManager.getLogger(LicenseBookingGate.class); + + /** + * How long a layer's license declaration is remembered. Layer environments are written at + * launch and effectively immutable afterwards, so this only bounds memory, not correctness. + */ + private static final long LAYER_CACHE_EXPIRE_MINUTES = 10; + private static final long LAYER_CACHE_MAX_SIZE = 100_000; + + /** Hard bound on rows the pack-job query may return, regardless of pending backlog. */ + private static final int MAX_PACK_QUERY_ROWS = 500; + + private final JdbcTemplate jdbc; + private final LicenseSource licenseSource; + + /** layerId -> license names it declares; an empty list means unlicensed (the common case). */ + private final Cache> layerLicenses = + CacheBuilder.newBuilder().maximumSize(LAYER_CACHE_MAX_SIZE) + .expireAfterWrite(LAYER_CACHE_EXPIRE_MINUTES, TimeUnit.MINUTES).build(); + + public LicenseBookingGate(JdbcTemplate jdbc, LicenseSource licenseSource) { + this.jdbc = jdbc; + this.licenseSource = licenseSource; + } + + /** + * License names the given layer declares, lowercased; empty when the layer is unlicensed. + * + * Returns null when the lookup itself failed, which callers must treat as "hold the frame": + * booking a layer whose requirements could not be read is the blind run this gate exists to + * prevent. A DB failure here means dispatch is failing anyway. + */ + public List licensesForLayer(String layerId) { + try { + return layerLicenses.get(layerId, () -> { + List values = jdbc.queryForList( + "SELECT str_value FROM layer_env WHERE pk_layer = ? AND str_key = ?", + String.class, layerId, licenseSource.getEnvKey()); + if (values.isEmpty()) { + return Collections.emptyList(); + } + return LicenseSource.splitNames(values.get(0)); + }); + } catch (Exception e) { + logger.warn("LicenseBookingGate: failed to read licenses for layer " + layerId + + ", holding its frames: " + e); + return null; + } + } + + /** A gate session for one dispatch pass onto one host. */ + public Session newSession(String hostname) { + return new Session(hostname); + } + + /** + * Pass-local license accounting for one dispatch pass onto one host. + * + * The budget snapshot is taken lazily, on the first licensed frame the pass meets, so passes + * over unlicensed work never touch {@link LicenseSource}. Within the pass, floating seats are + * decremented per booked frame and host-based seats taken on this host are remembered, so the + * pass stays inside the budget it started with. + */ + public final class Session { + + /** Lowercased, to match {@link LicenseSource}'s host normalization. */ + private final String host; + + /** Budgets fetched so far; grows as new license names are encountered. */ + private final Map budgets = new HashMap<>(); + + /** Floating seats consumed by frames booked in this pass, per license. */ + private final Map floatingUsed = new HashMap<>(); + + /** Working copy of each host-based license's seat set, including seats this pass took. */ + private final Map> seats = new HashMap<>(); + + private Session(String hostname) { + this.host = hostname == null ? "" : hostname.toLowerCase(); + } + + /** + * May a frame of this layer be booked on this session's host right now? True for unlicensed + * layers. A licensed layer needs a seat in EVERY pool it lists; a stale or unknown pool + * holds it (fail closed). + */ + public boolean canBook(String layerId) { + List names = licensesForLayer(layerId); + if (names == null) { + return false; + } + if (names.isEmpty()) { + return true; + } + ensureBudgets(names); + for (String name : names) { + LicenseSource.LicenseBudget budget = budgets.get(name); + if (budget == null || budget.stale) { + return false; + } + if (budget.hostBased) { + // Free on a host already holding the license; a fresh host + // must fit under the seat cap. + Set seated = seats.get(name); + if (!seated.contains(host) && seated.size() >= budget.seatCap) { + return false; + } + } else { + if (budget.usable - floatingUsed.getOrDefault(name, 0) < 1) { + return false; + } + } + } + return true; + } + + /** + * Record a successful booking of a frame of this layer, consuming this pass's budget: + * floating pools lose a seat per frame, host-based pools gain this host as a seat. + */ + public void booked(String layerId) { + List names = licensesForLayer(layerId); + if (names == null || names.isEmpty()) { + return; + } + for (String name : names) { + LicenseSource.LicenseBudget budget = budgets.get(name); + if (budget == null || budget.stale) { + continue; + } + if (budget.hostBased) { + seats.get(name).add(host); + } else { + floatingUsed.merge(name, 1, Integer::sum); + } + } + } + + private void ensureBudgets(List names) { + Set missing = new HashSet<>(); + for (String name : names) { + if (!budgets.containsKey(name)) { + missing.add(name); + } + } + if (missing.isEmpty()) { + return; + } + Map fetched = + licenseSource.snapshotBudgets(missing); + budgets.putAll(fetched); + for (LicenseSource.LicenseBudget budget : fetched.values()) { + if (budget.hostBased) { + seats.put(budget.name, new HashSet<>(budget.seats)); + } + } + } + } + + // ---- packing (HostReportHandler) -------------------------------------- + + /** + * Cheap packing pre-check for the report thread: does any of these running frames declare + * licenses at all? Costs only layer-cache lookups (an indexed single-row query per layer per + * cache period), never a budget snapshot. False when no provider is configured, because then + * there is nothing to pack against. + */ + public boolean anyLicensedLayers(List runningFrames) { + if (!licenseSource.hasProvider()) { + return false; + } + for (RunningFrameInfo frame : runningFrames) { + List names = licensesForLayer(frame.getLayerId()); + if (names != null && !names.isEmpty()) { + return true; + } + } + return false; + } + + /** + * Host-based licenses held by any of the given running frames, per the current sample. This is + * the packing trigger: a host already holding such a license can run more frames that need it + * for free. Floating licenses are excluded (a seat is spent per frame, nothing to pack), as are + * stale pools (the gate would hold their layers anyway). + */ + public Set hostBasedLicensesRunning(List runningFrames) { + if (!licenseSource.hasProvider() || runningFrames.isEmpty()) { + return Collections.emptySet(); + } + Set names = new HashSet<>(); + for (RunningFrameInfo frame : runningFrames) { + List layerNames = licensesForLayer(frame.getLayerId()); + if (layerNames != null) { + names.addAll(layerNames); + } + } + if (names.isEmpty()) { + return Collections.emptySet(); + } + Set out = new HashSet<>(); + for (LicenseSource.LicenseBudget budget : licenseSource.snapshotBudgets(names).values()) { + if (budget.hostBased && !budget.stale) { + out.add(budget.name); + } + } + return out; + } + + /** + * Ids of pending jobs with waiting frames on layers that need one of the given licenses, + * highest priority first, filtered to jobs this host could run (facility and OS; tags, + * resources, subscription burst and the license budgets themselves are all re-checked by the + * normal dispatch path). License matching is done in Java on the layer's declaration -- + * deliberately no new predicates in the sensitive dispatch queries. + */ + public List findPackableJobs(Set licenses, DispatchHost host, int limit) { + if (licenses.isEmpty() || limit <= 0) { + return Collections.emptyList(); + } + Set hostOs = new HashSet<>(Arrays.asList(host.getOs())); + List jobIds = new ArrayList<>(); + // The SQL LIMIT is a hard bound on rows examined (one row per licensed + // layer with waiting frames), generous because OS/license filtering + // happens in Java below; `limit` caps the jobs actually returned. + jdbc.query("SELECT DISTINCT job.pk_job, job.str_os, le.str_value, jr.int_priority " + + "FROM layer_env le " + "JOIN layer l ON l.pk_layer = le.pk_layer " + + "JOIN layer_stat lst ON lst.pk_layer = le.pk_layer " + + "JOIN job ON job.pk_job = l.pk_job " + + "JOIN job_resource jr ON jr.pk_job = job.pk_job " + "WHERE le.str_key = ? " + + "AND lst.int_waiting_count > 0 " + "AND job.str_state = 'PENDING' " + + "AND job.b_paused = false " + "AND job.pk_facility = ? " + + "ORDER BY jr.int_priority DESC LIMIT " + MAX_PACK_QUERY_ROWS, rs -> { + if (jobIds.size() >= limit) { + return; + } + String jobOs = rs.getString("str_os"); + if (jobOs != null && !jobOs.isEmpty() && !hostOs.contains(jobOs)) { + return; + } + boolean wantsOne = false; + for (String name : LicenseSource.splitNames(rs.getString("str_value"))) { + if (licenses.contains(name)) { + wantsOne = true; + break; + } + } + String jobId = rs.getString("pk_job"); + if (wantsOne && !jobIds.contains(jobId)) { + jobIds.add(jobId); + } + }, licenseSource.getEnvKey(), host.getFacilityId()); + return jobIds; + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LicenseSource.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LicenseSource.java new file mode 100644 index 0000000000..3a0b163a47 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LicenseSource.java @@ -0,0 +1,804 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.core.env.Environment; +import org.springframework.jdbc.core.JdbcTemplate; + +/** + * Live view of floating application licenses (Houdini Engine, Katana, Maya, ...) for the planner. + * + *

Why this exists

A {@code limit_record} holds a static number an admin typed. That works + * for an internal throttle but not for a real license pool, because the pool is also drawn on from + * outside the cue: artist workstations, CI, other farms. A fixed cap of 100 means nothing when 60 + * seats are already out to people. The only authority on how many seats are free is the license + * server, so this class polls it and the planner gates against the live number. + * + *

Shape

A background daemon thread asks a site-provided endpoint for every license we care + * about and keeps the answer in memory. The planner never touches the endpoint on the hot path: + * once per tick it asks for {@link #snapshotBudgets} and gets plain numbers. The provider is either + * an {@code http://} endpoint or {@code script:} wrapping a vendor CLI such as + * {@code sesictrl}; either way it returns the same JSON: + * + * 
+ * {"queried_at": 1690000000,
+ *  "licenses": [{"name": "hengine", "feature": "Houdini Engine", "total": 800,
+ *                "available": 794, "host_based": false,
+ *                "hosts": [{"host": "wolf1018", "count": 1}]}]}
+ *
+ * + * {@code available} is server truth and has already netted out every consumer, ours included. + * {@code queried_at} (epoch seconds, when the numbers were true) is REQUIRED: a response without a + * usable timestamp is rejected and the previous sample keeps aging toward stale, because a provider + * re-serving a cached payload with no timestamp would otherwise look fresh forever. {@code hosts} + * is optional; when present it enables seat counting for host-based licenses and shows which render + * nodes are dual-used as workstations. + * + *

Staleness is the whole difficulty

The planner always acts on a sample that is already + * old, and between the sample and a real checkout seats move. Two terms correct for that: + * + *
    + *
  • in-flight -- frames WE booked since the sample was taken. The server has not seen them + * yet, so they are not in {@code available} and must be subtracted, otherwise one tick happily + * books fifty frames against ten free seats. It is derived from the database rather than from a + * counter in this process, so a Cuebot that takes over after a failover computes the same number as + * the one it replaced.
    • + *
  • headroom -- seats deliberately left for interactive users, per license.
    • + *
+ * + * When the sample goes older than {@code stale_seconds} this fails CLOSED: budgets go to zero and + * licensed layers stop being placed. Over-booking a license we cannot see would fail frames on + * checkout on the farm; holding them costs throughput on licensed layers only, and is recoverable. + * + *

Every Cuebot polls

Not just the planning leader. The poll is a cheap read and it keeps + * standbys warm, so a Cuebot promoted by failover already holds a fresh sample instead of gating + * all licensed work until its first poll lands. + */ +public class LicenseSource { + + private static final Logger logger = LogManager.getLogger(LicenseSource.class); + + /** Cap on a provider response, so a broken endpoint cannot exhaust the heap. */ + private static final int MAX_RESPONSE_BYTES = 4 * 1024 * 1024; + + private final Environment env; + private final JdbcTemplate jdbc; + + private final String provider; + private final String envKey; + private final int pollSeconds; + private final int timeoutSeconds; + private final int staleSeconds; + private final int defaultHeadroom; + private final int inflightPadSeconds; + + private volatile Thread poller; + private volatile boolean running; + + /** Last good sample, or null before the first successful poll. */ + private volatile Sample sample = null; + + /** Consecutive failed polls; drives the log-once-then-quiet warning. */ + private volatile int consecutiveFailures = 0; + + /** Throttle for the "holding licensed layers" warning, which is per tick otherwise. */ + private static final long STALE_WARN_INTERVAL_MS = 60_000; + private volatile long lastStaleWarnMs = 0; + private volatile long lastNoProviderWarnMs = 0; + + public LicenseSource(Environment env, JdbcTemplate jdbc) { + this.env = env; + this.jdbc = jdbc; + // Trimmed: a properties file with a trailing space would otherwise count as + // "configured" and start a poller that fails forever, reporting a parse + // error instead of the truth, which is that nobody set a provider. + this.provider = env.getProperty("scheduler.license.provider", "").trim(); + this.envKey = env.getProperty("scheduler.license.env_key", "CUE_LICENSES"); + this.pollSeconds = env.getProperty("scheduler.license.poll_seconds", Integer.class, 20); + this.timeoutSeconds = + env.getProperty("scheduler.license.timeout_seconds", Integer.class, 10); + this.staleSeconds = env.getProperty("scheduler.license.stale_seconds", Integer.class, 300); + this.defaultHeadroom = + env.getProperty("scheduler.license.headroom.default", Integer.class, 0); + this.inflightPadSeconds = + env.getProperty("scheduler.license.inflight_pad_seconds", Integer.class, 5); + } + + /** + * Is a provider configured, i.e. can we obtain live numbers at all? + * + * There is deliberately NO separate on/off switch. What turns licensing on is a layer asking + * for a license, because that is a hard requirement of the work, not a site preference: a flag + * that had to be remembered would sooner or later be forgotten, and the farm would then quietly + * book straight through a license pool. If no provider is configured and a layer asks for a + * license anyway, that layer is HELD rather than run blind (see {@link #snapshotBudgets}). + */ + public boolean hasProvider() { + return !provider.isEmpty(); + } + + /** The layer environment key that binds a layer to its licenses, e.g. {@code CUE_LICENSES}. */ + public String getEnvKey() { + return envKey; + } + + // ---- lifecycle -------------------------------------------------------- + + /** + * Start the background poll. Polls once immediately (in the new thread, so startup is never + * blocked by a slow license server) and then every {@code poll_seconds}. Until that first + * sample lands, licensed layers are held: no sample means no authority to book. + */ + public synchronized void start() { + if (!hasProvider() || poller != null) + return; + running = true; + Thread t = new Thread(() -> { + while (running) { + try { + poll(); + } catch (RuntimeException e) { + logger.warn("LicenseSource: poll failed: " + e); + } + try { + Thread.sleep(1000L * pollSeconds); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + }); + t.setName("Scheduler-license-poll"); + t.setDaemon(true); + poller = t; + t.start(); + logger.info("LicenseSource: polling " + provider + " every " + pollSeconds + "s" + + " (env key " + envKey + ", stale after " + staleSeconds + "s)"); + } + + public synchronized void stop() { + running = false; + Thread t = poller; + poller = null; + if (t != null) + t.interrupt(); + } + + // ---- the sample ------------------------------------------------------ + + /** One license as the provider reported it. */ + public static final class LicenseState { + public final String name; + public final String feature; + public final int total; + public final int available; + public final boolean hostBased; + /** Hosts the provider says hold this license (any consumer, not just ours). Never null. */ + public final Set hosts; + + LicenseState(String name, String feature, int total, int available, boolean hostBased, + Set hosts) { + this.name = name; + this.feature = feature; + this.total = total; + this.available = available; + this.hostBased = hostBased; + this.hosts = hosts; + } + } + + /** + * A provider response plus the two clock facts needed to age it: when we received it, and how + * old it already was on arrival ({@code lagSeconds}, from the provider's own + * {@code queried_at}). + */ + private static final class Sample { + final Map licenses; + final long receivedAtMs; + final long lagSeconds; + + Sample(Map licenses, long receivedAtMs, long lagSeconds) { + this.licenses = licenses; + this.receivedAtMs = receivedAtMs; + this.lagSeconds = lagSeconds; + } + } + + /** + * What the planner may book for one license this tick. + * + * For a floating license only {@link #usable} matters: frames still bookable right now. For a + * host-based license the cap counts distinct hosts, so the planner needs the seat set + * ({@link #seats}, hosts that already hold one, including workstations outside the cue) and + * {@link #seatCap}, the most seats it may let exist. + */ + public static final class LicenseBudget { + public final String name; + public final boolean hostBased; + public final int usable; + public final int seatCap; + public final Set seats; + /** Sample too old (or missing) to act on: everything is held. */ + public final boolean stale; + + LicenseBudget(String name, boolean hostBased, int usable, int seatCap, Set seats, + boolean stale) { + this.name = name; + this.hostBased = hostBased; + this.usable = usable; + this.seatCap = seatCap; + this.seats = seats; + this.stale = stale; + } + } + + // ---- per-tick read --------------------------------------------------- + + + /** + * Per-license budgets for this tick: {@code available - in-flight - headroom} for floating + * licenses, and the seat set plus seat cap for host-based ones. + * + * Returns an empty map when licensing is off. When the sample is missing or stale every license + * the planner asks about comes back {@code stale}, which holds licensed layers rather than + * guessing. + * + * @param wanted license names the current candidate set actually needs; nothing else is + * computed. + */ + public Map snapshotBudgets(Set wanted) { + if (wanted.isEmpty()) + return Collections.emptyMap(); + if (!hasProvider()) { + // A layer asked for a license and this Cuebot has no way to find out + // how many are free. Hold it: running it blind would book straight + // through somebody's license pool. Loud, because it is a + // misconfiguration, not a transient condition. + long nowMs = System.currentTimeMillis(); + if (nowMs - lastNoProviderWarnMs > STALE_WARN_INTERVAL_MS) { + lastNoProviderWarnMs = nowMs; + logger.warn("LicenseSource: layers ask for licenses " + wanted + + " but scheduler.license.provider is not configured;" + + " holding that work. Configure a provider or remove" + " " + envKey + + " from those layers."); + } + Map held = new HashMap<>(); + for (String name : wanted) + held.put(name, new LicenseBudget(name, false, 0, 0, Collections.emptySet(), true)); + return held; + } + + Sample s = sample; + long ageSeconds = (s == null) ? Long.MAX_VALUE + : s.lagSeconds + (System.currentTimeMillis() - s.receivedAtMs) / 1000L; + boolean stale = (s == null) || ageSeconds > staleSeconds; + + Map out = new HashMap<>(); + if (stale) { + // Throttled: this fires per tick while a provider is down, which is + // once every few seconds on every Cuebot. One line a minute is enough + // to see the outage without burying the log that would explain it. + long nowMs = System.currentTimeMillis(); + if ((s != null || consecutiveFailures > 0) + && nowMs - lastStaleWarnMs > STALE_WARN_INTERVAL_MS) { + lastStaleWarnMs = nowMs; + logger.warn("LicenseSource: " + + (s == null ? "no sample yet from " + provider + : "sample is " + ageSeconds + "s old, stale after " + staleSeconds + + "s") + + "; holding licensed layers (" + wanted.size() + " pool(s))"); + } + for (String name : wanted) { + out.put(name, new LicenseBudget(name, false, 0, 0, Collections.emptySet(), true)); + } + return out; + } + + // In-flight: what we booked since the sample was taken, so it is not yet + // reflected in `available`. Ask the DB for it (any Cuebot then agrees) and + // ask by AGE rather than by absolute instant, so a clock offset between + // Cuebot and Postgres cannot skew the window. + // + // The window is padded, because a provider's timestamp is only as honest + // as the provider. Collecting the numbers takes time (a vendor CLI, a + // query, an exporter scrape), and one that stamps the result when it + // FINISHES advertises a sample fresher than it is. Frames we booked during + // that collection then fall outside the window and outside `available`, + // and go uncounted twice over -- which is exactly how a tick over-books. + // Widening the window can only over-count in-flight, whose consequence is + // booking slightly less; getting it wrong the other way fails frames at + // checkout on the farm. + InFlight inFlight = readInFlight(ageSeconds + inflightPadSeconds, wanted); + + for (String name : wanted) { + LicenseState st = s.licenses.get(name); + if (st == null) { + // The layer asks for a license the provider does not report. We + // have no authority on it, so hold rather than assume it is free. + logger.warn("LicenseSource: no data for license '" + name + + "' requested by a layer; holding its frames"); + out.put(name, new LicenseBudget(name, false, 0, 0, Collections.emptySet(), true)); + continue; + } + int headroom = env.getProperty("scheduler.license.headroom." + name, Integer.class, + defaultHeadroom); + if (st.hostBased) { + // Machines already holding this license: the ones the provider + // reports, UNION every host of ours currently running it. The + // union never double counts a dual-use machine, and a frame + // placed on any of them is free, because it shares that + // machine's one checkout. + Set seats = new HashSet<>(st.hosts); + seats.addAll(inFlight.hostsByLicense.getOrDefault(name, Collections.emptySet())); + // How many machines may hold this license in total. + // + // Bounded via `available` rather than total - headroom, because + // `hosts` is optional: a provider that reports none leaves us + // blind to the machines outside the cue holding seats, and + // capping at total - headroom would then let us open every seat + // the license has while artists already hold some. `available` is + // server truth and has those holders netted out. + // + // The subtraction of hostsRecent is what makes it STABLE. seats is + // current while `available` is from the sample, so adding the two + // alone would count our own growth twice: every machine we seated + // since the sample would raise the cap by one and immediately + // justify another. Netting out the machines seated inside the + // sample window removes that feedback, exactly as the in-flight + // frame count does for a floating license. + // + // When the provider DOES report hosts the terms cancel to + // total - headroom, so one expression covers both cases. + int recentHosts = inFlight.recentHostsByLicense + .getOrDefault(name, Collections.emptySet()).size(); + int seatCap = seats.size() + Math.max(0, st.available - headroom - recentHosts); + out.put(name, new LicenseBudget(name, true, 0, seatCap, seats, false)); + } else { + int booked = inFlight.framesByLicense.getOrDefault(name, 0); + int usable = st.available - booked - headroom; + if (usable < 0) + usable = 0; + out.put(name, + new LicenseBudget(name, false, usable, 0, Collections.emptySet(), false)); + } + } + return out; + } + + /** + * Frames booked since the sample (floating), and hosts running licensed frames (host-based). + */ + private static final class InFlight { + final Map framesByLicense = new HashMap<>(); + final Map> hostsByLicense = new HashMap<>(); + /** Hosts that started running the license INSIDE the sample window. */ + final Map> recentHostsByLicense = new HashMap<>(); + } + + /** + * The in-flight terms, from the database so every Cuebot derives the same numbers. + * + * Two different windows, because the two license kinds count different things: + *
    + *
  • floating: frames started WITHIN the sample's age -- exactly the bookings the license + * server has not observed yet.
    • + *
  • host-based: ALL hosts currently running the license, any age. Seats are a set, so + * unioning our full host list with the provider's is idempotent and covers providers that + * report no hosts at all.
    • + *
+ */ + private InFlight readInFlight(long ageSeconds, Set wanted) { + InFlight f = new InFlight(); + // One pass over running licensed frames. Driven from layer_env (indexed on + // the key) so a farm whose licensed layers are a small slice of the whole + // does not pay for the rest. + jdbc.query("SELECT le.str_value AS lic, f.str_host AS host, " + + " (f.ts_started > now() - CAST(? AS INTERVAL)) AS recent " + "FROM layer_env le " + + "JOIN frame f ON f.pk_layer = le.pk_layer " + + "WHERE le.str_key = ? AND f.str_state = 'RUNNING'", rs -> { + boolean recent = rs.getBoolean("recent"); + String host = rs.getString("host"); + for (String name : splitNames(rs.getString("lic"))) { + if (!wanted.contains(name)) + continue; + if (recent) + f.framesByLicense.merge(name, 1, Integer::sum); + if (host != null && !host.isEmpty()) { + String h = host.toLowerCase(); + f.hostsByLicense.computeIfAbsent(name, k -> new HashSet<>()).add(h); + if (recent) { + f.recentHostsByLicense.computeIfAbsent(name, k -> new HashSet<>()) + .add(h); + } + } + } + }, ageSeconds + " seconds", envKey); + return f; + } + + // ---- read-only view (ManageLicense servant) -------------------------- + + /** One license for display: the provider's numbers plus this cluster's own usage. */ + public static final class LicenseInfo { + public final LicenseState state; + /** Seats withheld for interactive users ({@code scheduler.license.headroom.}). */ + public final int headroom; + /** Frames currently RUNNING here whose layer declares this license. */ + public final int runningFrames; + /** Distinct hosts those running frames occupy. */ + public final int runningHosts; + + LicenseInfo(LicenseState state, int headroom, int runningFrames, int runningHosts) { + this.state = state; + this.headroom = headroom; + this.runningFrames = runningFrames; + this.runningHosts = runningHosts; + } + } + + /** Poller state plus every license in the current sample, for display. */ + public static final class SourceStatus { + public final boolean configured; + public final String provider; + public final String envKey; + public final int pollSeconds; + public final int staleSeconds; + public final boolean hasSample; + /** Age of the sample including the provider's own lag; 0 without a sample. */ + public final long ageSeconds; + public final boolean stale; + /** Sorted by name. Empty without a sample. */ + public final List licenses; + + SourceStatus(boolean configured, String provider, String envKey, int pollSeconds, + int staleSeconds, boolean hasSample, long ageSeconds, boolean stale, + List licenses) { + this.configured = configured; + this.provider = provider; + this.envKey = envKey; + this.pollSeconds = pollSeconds; + this.staleSeconds = staleSeconds; + this.hasSample = hasSample; + this.ageSeconds = ageSeconds; + this.stale = stale; + this.licenses = licenses; + } + } + + /** + * The current sample and poller state, for display (the read-only Licenses view in CueGUI). + * + * Unlike {@link #snapshotBudgets} this reports every license in the sample, and the usage + * numbers count ALL of our running licensed frames rather than only the recent ones inside the + * in-flight window: an operator wants "what is the farm holding", not the booking correction. + * The database read is one indexed pass over running licensed frames, the same shape the + * booking path already pays per tick, so a GUI refresh is cheap. A database failure propagates: + * this is a live inspection, and a wrong answer dressed as a real one would send an operator + * chasing numbers that mean nothing. + */ + public SourceStatus describe() { + Sample s = sample; + long ageSeconds = (s == null) ? 0 + : s.lagSeconds + (System.currentTimeMillis() - s.receivedAtMs) / 1000L; + boolean stale = (s == null) || ageSeconds > staleSeconds; + List licenses = new ArrayList<>(); + if (s != null && !s.licenses.isEmpty()) { + RunningUse use = readRunningUse(s.licenses.keySet()); + List states = new ArrayList<>(s.licenses.values()); + states.sort(Comparator.comparing(st -> st.name)); + for (LicenseState st : states) { + int headroom = env.getProperty("scheduler.license.headroom." + st.name, + Integer.class, defaultHeadroom); + licenses.add(new LicenseInfo(st, headroom, use.frames.getOrDefault(st.name, 0), + use.hosts.getOrDefault(st.name, Collections.emptySet()).size())); + } + } + return new SourceStatus(hasProvider(), redactProvider(provider), envKey, pollSeconds, + staleSeconds, s != null, ageSeconds, stale, licenses); + } + + /** + * The provider string safe for display. The status travels to every CueGUI, and sites embed + * credentials in http URLs (userinfo, query tokens), so both are redacted while the host and + * path stay visible -- an operator debugging still sees WHERE the numbers come from. A + * {@code script:} command line is shown as configured; keep secrets out of it. + */ + static String redactProvider(String provider) { + if (!provider.startsWith("http:") && !provider.startsWith("https:")) { + return provider; + } + // scheme://user:secret@host -> scheme://host. Greedy to the LAST @ + // before the path/query, because Java's URL tolerates a raw @ in the + // password and a lazy match would leak its tail; bounded at /?# so an + // @ inside the query or path is never mistaken for userinfo. + String out = provider.replaceFirst("^(https?://)[^/?#]*@", "$1"); + // ?token=abc&x=y -> ?token=****&x=**** + return out.replaceAll("([?&][^=&#]*)=[^&#]*", "$1=****"); + } + + /** Running licensed frames and their hosts, per license, no age window. */ + private static final class RunningUse { + final Map frames = new HashMap<>(); + final Map> hosts = new HashMap<>(); + } + + private RunningUse readRunningUse(Set wanted) { + RunningUse use = new RunningUse(); + jdbc.query("SELECT le.str_value AS lic, f.str_host AS host " + "FROM layer_env le " + + "JOIN frame f ON f.pk_layer = le.pk_layer " + + "WHERE le.str_key = ? AND f.str_state = 'RUNNING'", rs -> { + String host = rs.getString("host"); + for (String name : splitNames(rs.getString("lic"))) { + if (!wanted.contains(name)) + continue; + use.frames.merge(name, 1, Integer::sum); + if (host != null && !host.isEmpty()) { + use.hosts.computeIfAbsent(name, k -> new HashSet<>()) + .add(host.toLowerCase()); + } + } + }, envKey); + return use; + } + + // ---- polling --------------------------------------------------------- + + /** Fetch and parse one sample. Keeps the previous sample on failure (staleness handles it). */ + void poll() { + long t0 = System.currentTimeMillis(); + String body; + try { + body = fetch(); + } catch (Exception e) { + consecutiveFailures++; + if (consecutiveFailures == 1 || consecutiveFailures % 10 == 0) { + logger.warn("LicenseSource: provider " + provider + " failed (" + + consecutiveFailures + " in a row): " + e); + } + return; + } + try { + JsonObject root = JsonParser.parseString(body).getAsJsonObject(); + Map parsed = new HashMap<>(); + JsonArray arr = root.getAsJsonArray("licenses"); + if (arr != null) { + for (JsonElement el : arr) { + JsonObject o = el.getAsJsonObject(); + String name = str(o, "name", null); + if (name == null || name.isEmpty()) + continue; + Set hosts = new HashSet<>(); + JsonArray ha = o.getAsJsonArray("hosts"); + if (ha != null) { + for (JsonElement he : ha) { + String h = he.isJsonObject() ? str(he.getAsJsonObject(), "host", null) + : he.getAsString(); + if (h != null && !h.isEmpty()) + hosts.add(h.toLowerCase()); + } + } + parsed.put(name.toLowerCase(), + new LicenseState(name.toLowerCase(), str(o, "feature", name), + num(o, "total", 0), num(o, "available", 0), + bool(o, "host_based", false), hosts)); + } + } + long queriedAt = lng(root, "queried_at", 0L); + long now = System.currentTimeMillis(); + // queried_at is REQUIRED: without it a provider re-serving a stale + // cached payload would look fresh forever. Reject the poll and let + // the previous sample keep aging toward stale (fail closed). + if (queriedAt <= 0L) { + consecutiveFailures++; + if (consecutiveFailures == 1 || consecutiveFailures % 10 == 0) { + logger.warn("LicenseSource: provider response has no usable queried_at " + + "(got " + queriedAt + "); rejecting sample (" + consecutiveFailures + + " in a row)"); + } + return; + } + // How stale the sample already was when it reached us. Clamped at 0: + // a provider clock running ahead must not make a sample look fresher + // than it is. + long lag = Math.max(0L, now / 1000L - queriedAt); + sample = new Sample(parsed, now, lag); + if (consecutiveFailures > 0) { + logger.info("LicenseSource: provider recovered after " + consecutiveFailures + + " failed polls"); + } + consecutiveFailures = 0; + if (logger.isDebugEnabled()) { + StringBuilder sb = new StringBuilder(); + for (LicenseState st : parsed.values()) { + sb.append(' ').append(st.name).append('=').append(st.available).append('/') + .append(st.total); + if (st.hostBased) + sb.append("(hosts:").append(st.hosts.size()).append(')'); + } + logger.debug( + "LicenseSource: sample in " + (now - t0) + "ms, lag " + lag + "s:" + sb); + } + } catch (RuntimeException e) { + consecutiveFailures++; + logger.warn("LicenseSource: unparseable provider response: " + e); + } + } + + /** {@code http:URL} or {@code script:/path [args]}. */ + private String fetch() throws IOException, InterruptedException { + if (provider.startsWith("http:") || provider.startsWith("https:")) { + return fetchHttp(provider); + } + if (provider.startsWith("script:")) { + return fetchScript(provider.substring("script:".length()).trim()); + } + throw new IOException( + "provider must start with http:, https: or script: -- got " + provider); + } + + private String fetchHttp(String url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setConnectTimeout(timeoutSeconds * 1000); + conn.setReadTimeout(timeoutSeconds * 1000); + conn.setRequestProperty("Accept", "application/json"); + try { + int code = conn.getResponseCode(); + if (code != 200) + throw new IOException("HTTP " + code); + try (InputStream in = conn.getInputStream()) { + return readBounded(in); + } + } finally { + conn.disconnect(); + } + } + + /** + * Run a site script and take its stdout. + * + * A vendor CLI can hang, so the process gets a hard deadline and is killed on it. stdout is + * drained on a helper thread rather than after {@code waitFor}: a script that fills the pipe + * buffer while we wait would deadlock, and killing it is what unblocks the reader. + */ + private String fetchScript(String cmd) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder("/bin/sh", "-c", cmd); + pb.redirectErrorStream(false); + Process p = pb.start(); + StringBuilder out = new StringBuilder(); + StringBuilder err = new StringBuilder(); + IOException[] readError = new IOException[1]; + Thread reader = new Thread(() -> { + try (InputStream in = p.getInputStream()) { + out.append(readBounded(in)); + } catch (IOException e) { + readError[0] = e; + } + }); + reader.setName("Scheduler-license-script"); + reader.setDaemon(true); + reader.start(); + // Drain stderr too, or a chatty script blocks on a full stderr pipe + // (~64KB) and never exits, turning a good answer into a fake timeout. + // Kept bounded and only surfaced when the script fails. + Thread errReader = new Thread(() -> { + try (InputStream in = p.getErrorStream()) { + err.append(readBounded(in)); + } catch (IOException e) { + // Over the bound or broken pipe: stderr is diagnostics only. + } + }); + errReader.setName("Scheduler-license-script-err"); + errReader.setDaemon(true); + errReader.start(); + boolean exited = p.waitFor(timeoutSeconds, TimeUnit.SECONDS); + if (!exited) { + p.destroyForcibly(); + reader.join(TimeUnit.SECONDS.toMillis(2)); + errReader.join(TimeUnit.SECONDS.toMillis(2)); + throw new IOException( + "script timed out after " + timeoutSeconds + "s" + errExcerpt(err)); + } + reader.join(TimeUnit.SECONDS.toMillis(2)); + errReader.join(TimeUnit.SECONDS.toMillis(2)); + if (readError[0] != null) + throw readError[0]; + if (p.exitValue() != 0) + throw new IOException("script exited " + p.exitValue() + errExcerpt(err)); + return out.toString(); + } + + /** First 500 chars of the script's stderr, for failure messages only. */ + private static String errExcerpt(StringBuilder err) { + if (err.length() == 0) + return ""; + String s = err.substring(0, Math.min(err.length(), 500)); + return ", stderr: " + s.trim(); + } + + private static String readBounded(InputStream in) throws IOException { + StringBuilder sb = new StringBuilder(); + char[] buf = new char[8192]; + try (BufferedReader r = + new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + int n; + while ((n = r.read(buf)) > 0) { + sb.append(buf, 0, n); + if (sb.length() > MAX_RESPONSE_BYTES) + throw new IOException( + "provider response over " + MAX_RESPONSE_BYTES + " bytes"); + } + } + return sb.toString(); + } + + // ---- helpers --------------------------------------------------------- + + /** {@code "hengine, katana"} to {@code [hengine, katana]}, lowercased, blanks dropped. */ + static List splitNames(String csv) { + List out = new java.util.ArrayList<>(2); + if (csv == null) + return out; + for (String part : csv.split(",")) { + String s = part.trim().toLowerCase(); + if (!s.isEmpty() && !out.contains(s)) + out.add(s); + } + return out; + } + + private static String str(JsonObject o, String key, String dflt) { + JsonElement e = o.get(key); + return (e == null || e.isJsonNull()) ? dflt : e.getAsString(); + } + + private static int num(JsonObject o, String key, int dflt) { + JsonElement e = o.get(key); + return (e == null || e.isJsonNull()) ? dflt : e.getAsInt(); + } + + private static long lng(JsonObject o, String key, long dflt) { + JsonElement e = o.get(key); + return (e == null || e.isJsonNull()) ? dflt : e.getAsLong(); + } + + private static boolean bool(JsonObject o, String key, boolean dflt) { + JsonElement e = o.get(key); + return (e == null || e.isJsonNull()) ? dflt : e.getAsBoolean(); + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java index d443cb941d..f47a601fc7 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java @@ -49,6 +49,13 @@ public class LocalDispatcher extends AbstractDispatcher implements Dispatcher { private JobManager jobManager; private HostManager hostManager; + /** + * Gates dispatch on live application-license budgets (CUE_LICENSES). Local dispatch is gated + * like farm dispatch: an artist's local booking draws from the same license pools as the render + * nodes. Optional: when unset, dispatch behaves exactly as before. + */ + private LicenseBookingGate licenseBookingGate; + private static final int MAX_QUERY_FRAMES = 10; private static final int MAX_DISPATCHED_FRAMES = 10; @@ -99,8 +106,18 @@ private List dispatchHost(DispatchHost host, JobInterface job, logger.info("Frames found: " + frames.size() + " for host " + host.getName() + " " + host.idleCores + "/" + host.idleMemory + " on job " + job.getName()); + LicenseBookingGate.Session licenseSession = + licenseBookingGate == null ? null : licenseBookingGate.newSession(host.getName()); for (DispatchFrame frame : frames) { + /* + * Hold frames whose layer needs an application license with no free seat; other layers + * of the job may still book, so skip, not break. + */ + if (licenseSession != null && !licenseSession.canBook(frame.getLayerId())) { + continue; + } + /* * Check if we have enough memory/cores for this frame, if not move on. */ @@ -128,6 +145,9 @@ private List dispatchHost(DispatchHost host, JobInterface job, */ if (dispatchHost(frame, proc)) { + if (licenseSession != null) { + licenseSession.booked(frame.getLayerId()); + } procs.add(proc); long memReservedMin = @@ -184,8 +204,17 @@ private List dispatchHost(DispatchHost host, LayerInterface layer, logger.info("Frames found: " + frames.size() + " for host " + host.getName() + " " + host.idleCores + "/" + host.idleMemory + " on layer " + layer); + LicenseBookingGate.Session licenseSession = + licenseBookingGate == null ? null : licenseBookingGate.newSession(host.getName()); for (DispatchFrame frame : frames) { + /* + * Hold frames whose layer needs an application license with no free seat. + */ + if (licenseSession != null && !licenseSession.canBook(frame.getLayerId())) { + continue; + } + /* * Check if we have enough memory/cores for this frame, if not move on. */ @@ -213,6 +242,9 @@ private List dispatchHost(DispatchHost host, LayerInterface layer, */ if (dispatchHost(frame, proc)) { + if (licenseSession != null) { + licenseSession.booked(frame.getLayerId()); + } procs.add(proc); long memReservedMin = @@ -273,6 +305,14 @@ private List dispatchHost(DispatchHost host, FrameInterface frame, return procs; } + /* + * Hold the frame if its layer needs an application license with no free seat. + */ + if (licenseBookingGate != null + && !licenseBookingGate.newSession(host.getName()).canBook(dframe.getLayerId())) { + return procs; + } + VirtualProc proc = VirtualProc.build(host, dframe, lha); /* @@ -353,7 +393,12 @@ public void dispatchProcToJob(VirtualProc proc, JobInterface job) { logger.info("Frames found: " + frames.size() + " for host " + proc + " " + proc.coresReserved + "/" + proc.memoryReserved + " on job " + job.getName()); + LicenseBookingGate.Session licenseSession = + licenseBookingGate == null ? null : licenseBookingGate.newSession(proc.hostName); for (DispatchFrame frame : frames) { + if (licenseSession != null && !licenseSession.canBook(frame.getLayerId())) { + continue; + } if (dispatchProc(frame, proc)) { return; } @@ -409,4 +454,12 @@ public HostManager getHostManager() { public void setHostManager(HostManager hostManager) { this.hostManager = hostManager; } + + public LicenseBookingGate getLicenseBookingGate() { + return licenseBookingGate; + } + + public void setLicenseBookingGate(LicenseBookingGate licenseBookingGate) { + this.licenseBookingGate = licenseBookingGate; + } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLicensePack.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLicensePack.java new file mode 100644 index 0000000000..36c8597667 --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLicensePack.java @@ -0,0 +1,128 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher.commands; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.core.env.Environment; +import org.springframework.dao.EmptyResultDataAccessException; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.JobInterface; +import com.imageworks.spcue.ShowInterface; +import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.dispatcher.LicenseBookingGate; +import com.imageworks.spcue.grpc.report.RunningFrameInfo; +import com.imageworks.spcue.service.JobManager; + +/** + * License-packing variant of {@link DispatchBookHost}. + * + * Queued when a host report shows running frames that declare application licenses. All the real + * work: deciding which of those licenses are host-based with a live sample, finding pending jobs + * that need them, and dispatching happens here on a booking thread, so the report thread only pays + * a cache lookup to decide whether to queue this command at all. + * + * Jobs that need a host-based license this host already holds get the first shot at its idle + * resources (an extra frame on a seated machine is free; a fresh machine burns a seat). Whatever + * they leave falls through to the exact booking the host would have received without packing: its + * preferred show when it has one, then the normal all-shows order. + */ +public class DispatchBookHostLicensePack extends KeyRunnable { + + private static final Logger logger = LogManager.getLogger(DispatchBookHostLicensePack.class); + + private final DispatchHost host; + private final List runningFrames; + private final ShowInterface preferredShow; + private final LicenseBookingGate licenseBookingGate; + private final JobManager jobManager; + private final Dispatcher dispatcher; + private final Environment env; + + public DispatchBookHostLicensePack(DispatchHost host, List runningFrames, + ShowInterface preferredShow, LicenseBookingGate licenseBookingGate, + JobManager jobManager, Dispatcher dispatcher, Environment env) { + super(host.getId() + "_license_pack"); + this.host = host; + this.runningFrames = runningFrames; + this.preferredShow = preferredShow; + this.licenseBookingGate = licenseBookingGate; + this.jobManager = jobManager; + this.dispatcher = dispatcher; + this.env = env; + } + + public void run() { + new DispatchCommandTemplate() { + public void wrapDispatchCommand() { + long memReservedMin = + env.getRequiredProperty("dispatcher.memory.mem_reserved_min", Long.class); + long memGpuReservedMin = env + .getRequiredProperty("dispatcher.memory.mem_gpu_reserved_min", Long.class); + + for (JobInterface packJob : findPackJobs()) { + if (!host.hasAdditionalResources(Dispatcher.CORE_POINTS_RESERVED_MIN, + memReservedMin, Dispatcher.GPU_UNITS_RESERVED_MIN, memGpuReservedMin)) { + break; + } + dispatcher.dispatchHost(host, packJob); + } + + /* + * Fall through to the exact booking this host would have received without packing + * (DispatchBookHost ends with the generic remaining-resources booking). + */ + if (preferredShow != null) { + new DispatchBookHost(host, preferredShow, dispatcher, env).run(); + } else { + new DispatchBookHost(host, dispatcher, env).run(); + } + } + }.execute(); + } + + /** + * Jobs whose waiting layers need a host-based license this host's running frames already hold. + * Empty on any failure: packing is an optimization and must never block the fall-through + * booking. + */ + private List findPackJobs() { + List packJobs = new ArrayList(); + try { + Set packLicenses = licenseBookingGate.hostBasedLicensesRunning(runningFrames); + if (packLicenses.isEmpty()) { + return packJobs; + } + int packJobsMax = env.getProperty("scheduler.license.pack_jobs_max", Integer.class, 5); + for (String jobId : licenseBookingGate.findPackableJobs(packLicenses, host, + packJobsMax)) { + try { + packJobs.add(jobManager.getJob(jobId)); + } catch (EmptyResultDataAccessException e) { + // The job finished between the query and here; nothing to pack. + } + } + } catch (Exception e) { + logger.warn("Failed to find license pack jobs for " + host.getName() + ": " + e); + } + return packJobs; + } +} diff --git a/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLicense.java b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLicense.java new file mode 100644 index 0000000000..3c9c4a32ee --- /dev/null +++ b/cuebot/src/main/java/com/imageworks/spcue/servant/ManageLicense.java @@ -0,0 +1,98 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.servant; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; + +import com.imageworks.spcue.dispatcher.LicenseSource; +import com.imageworks.spcue.grpc.license.License; +import com.imageworks.spcue.grpc.license.LicenseFindRequest; +import com.imageworks.spcue.grpc.license.LicenseFindResponse; +import com.imageworks.spcue.grpc.license.LicenseGetAllRequest; +import com.imageworks.spcue.grpc.license.LicenseGetAllResponse; +import com.imageworks.spcue.grpc.license.LicenseInterfaceGrpc; +import com.imageworks.spcue.grpc.license.LicenseSourceStatus; + +/** + * Read-only view of the live application licenses {@link LicenseSource} polls from + * {@code scheduler.license.provider}. + * + * Unlike the other servants this one reads no database entity: license state deliberately lives + * in-process (a poll of the license server), so the wire view comes straight from the cached + * sample. There are no mutations here by design -- the numbers belong to the license server, and + * the tuning (provider, headroom, poll cadence) is an {@code opencue.properties} concern. + */ +public class ManageLicense extends LicenseInterfaceGrpc.LicenseInterfaceImplBase { + + private LicenseSource licenseSource; + + @Override + public void getAll(LicenseGetAllRequest request, + StreamObserver responseObserver) { + LicenseSource.SourceStatus status = licenseSource.describe(); + LicenseGetAllResponse.Builder response = + LicenseGetAllResponse.newBuilder().setSource(toSourceProto(status)); + for (LicenseSource.LicenseInfo info : status.licenses) { + response.addLicenses(toLicenseProto(info)); + } + responseObserver.onNext(response.build()); + responseObserver.onCompleted(); + } + + @Override + public void find(LicenseFindRequest request, + StreamObserver responseObserver) { + // Names are lowercased everywhere in the licensing path (provider parse, + // CUE_LICENSES split), so a mixed-case query should still find its pool. + String name = request.getName().trim().toLowerCase(); + for (LicenseSource.LicenseInfo info : licenseSource.describe().licenses) { + if (info.state.name.equals(name)) { + responseObserver.onNext( + LicenseFindResponse.newBuilder().setLicense(toLicenseProto(info)).build()); + responseObserver.onCompleted(); + return; + } + } + responseObserver.onError(Status.NOT_FOUND + .withDescription("No license '" + name + "' in the current provider sample.") + .asRuntimeException()); + } + + private static License toLicenseProto(LicenseSource.LicenseInfo info) { + return License.newBuilder().setName(info.state.name).setFeature(info.state.feature) + .setTotal(info.state.total).setAvailable(info.state.available) + .setHostBased(info.state.hostBased).setHeadroom(info.headroom) + .setRunningFrames(info.runningFrames).setRunningHosts(info.runningHosts) + .setProviderHostCount(info.state.hosts.size()).build(); + } + + private static LicenseSourceStatus toSourceProto(LicenseSource.SourceStatus status) { + return LicenseSourceStatus.newBuilder().setConfigured(status.configured) + .setProvider(status.provider).setEnvKey(status.envKey) + .setPollSeconds(status.pollSeconds).setStaleSeconds(status.staleSeconds) + .setHasSample(status.hasSample).setAgeSeconds(status.ageSeconds) + .setStale(status.stale).build(); + } + + public LicenseSource getLicenseSource() { + return licenseSource; + } + + public void setLicenseSource(LicenseSource licenseSource) { + this.licenseSource = licenseSource; + } +} diff --git a/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_env_key_index.sql b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_env_key_index.sql new file mode 100644 index 0000000000..119ea2c7ee --- /dev/null +++ b/cuebot/src/main/resources/conf/ddl/postgres/migrations/V46__Add_layer_env_key_index.sql @@ -0,0 +1,7 @@ +-- Live application licensing (CUE_LICENSES) reads layer_env by key on hot paths: +-- * LicenseSource.readInFlight scans running licensed frames per budget snapshot +-- * LicenseBookingGate.licensesForLayer resolves a layer's declaration (cached) +-- * LicenseBookingGate.findPackableJobs finds pending licensed work to pack +-- layer_env previously had no index on str_key, so those queries could only seq +-- scan a table holding every environment variable of every live layer. +CREATE INDEX i_layer_env_str_key ON layer_env (str_key, pk_layer); diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-grpc.xml b/cuebot/src/main/resources/conf/spring/applicationContext-grpc.xml index 612aeaff55..78511fda16 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-grpc.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-grpc.xml @@ -138,6 +138,10 @@ + + + + diff --git a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml index 18be8179f9..5dbe44cc1c 100644 --- a/cuebot/src/main/resources/conf/spring/applicationContext-service.xml +++ b/cuebot/src/main/resources/conf/spring/applicationContext-service.xml @@ -351,6 +351,7 @@ + @@ -359,6 +360,7 @@ + @@ -413,6 +415,7 @@ + diff --git a/cuebot/src/main/resources/opencue.properties b/cuebot/src/main/resources/opencue.properties index 1782177c5a..345b837277 100644 --- a/cuebot/src/main/resources/opencue.properties +++ b/cuebot/src/main/resources/opencue.properties @@ -204,6 +204,83 @@ dispatcher.memory.mem_gpu_reserved_min = 0 # Default = 100GB dispatcher.memory.mem_gpu_reserved_max = 104857600 +# ---- live application licenses ---------------------------------------------- +# Gate placement of licensed layers on how many seats the LICENSE SERVER says are +# free, not on a static cap. A Limit (limit_record) is a number an admin typed; +# it cannot see the seats artist workstations, CI and other farms are holding, so +# on a shared pool it is either wrong or set so low it wastes the farm. +# +# A layer declares what it needs in its environment: +# CUE_LICENSES=hengine,katana +# and needs a seat in EVERY pool it lists. That declaration IS the switch: there +# is deliberately no separate enable flag, because a licence is a requirement of +# the work rather than a site preference, and a flag that must be remembered +# eventually is not, leaving the farm booking straight through a licence pool. +# Layers that declare nothing are untouched and cost nothing. +# +# If a layer asks for a licence and no provider is configured below, that layer is +# HELD and a warning is logged. Running it blind is the one outcome worse than not +# running it. +# Where the live numbers come from. Either an endpoint or a site script wrapping +# a vendor CLI (sesictrl, rlmutil, lmstat); both return the same JSON: +# {"queried_at": , +# "licenses": [{"name": "hengine", "feature": "Houdini Engine", "total": 800, +# "available": 794, "host_based": false, +# "hosts": [{"host": "wolf1018", "count": 1}]}]} +# `available` is server truth and already nets out every consumer including us. +# `hosts` is optional: it enables seat counting for host_based licenses and +# reveals render nodes dual-used as workstations. +# scheduler.license.provider=http://houdini-lic:9101/licenses +# scheduler.license.provider=script:/site/bin/cue_licenses.sh +scheduler.license.provider= +# Poll cadence. Every Cuebot polls, not just the planning leader, so a standby +# promoted by failover already holds a fresh sample. +scheduler.license.poll_seconds=20 +# Hard deadline on one provider call; a hung vendor CLI is killed on it. +scheduler.license.timeout_seconds=10 +# Age at which a sample stops being actionable. Past it the booking gate FAILS +# CLOSED: licensed layers are held. Over-booking a pool we cannot see fails frames +# out on the farm at checkout; holding them costs throughput on licensed layers +# only. +scheduler.license.stale_seconds=300 +# Extra seconds added to the in-flight window (frames booked since the sample was +# taken, which the license server has not seen yet). A provider that stamps its +# response when collection FINISHES claims a sample fresher than it is, and the +# frames booked during that collection would then be counted neither in +# `available` nor as in-flight -- the one arithmetic that lets a tick over-book. +# Padding can only over-count in-flight, which books slightly less; erring the +# other way fails frames at checkout. Raise it if the provider is slow. +scheduler.license.inflight_pad_seconds=5 +# Seats kept back for interactive users, so the farm never takes the last one. +# Also absorbs the poll-window race. Per license: +# scheduler.license.headroom.hengine=5 +scheduler.license.headroom.default=0 +# Layer environment key that carries the license names. +scheduler.license.env_key=CUE_LICENSES +# Exit statuses meaning "the application could not get a license". Such a frame is +# requeued WAITING without spending a retry: it is a contended resource, not a +# broken frame, and burning retries on it would march a whole layer to DEAD when a +# pool is merely busy. The live gate plus headroom are what prevent this; the +# requeue catches the race they cannot -- an artist taking the last seat between +# the license sample and the real checkout. Vendor specific, so it is empty by +# default and frame completion behaves exactly as before until a site sets it. +# scheduler.license.denied_exit_statuses=11,203 +scheduler.license.denied_exit_statuses= +# How many times ONE frame may take the free requeue above. The booking gate +# holds licensed layers while their pool is full, so genuine denials are rare +# races; a frame denied over and over has a real problem (wrong feature name, +# broken license setup on the app side) and without a bound it would bounce +# between WAITING and the render node forever. Past the limit the vendor's exit +# status is stored again and ordinary retry accounting marches the frame to +# DEAD. 0 or negative = unbounded. +scheduler.license.denied_requeue_limit=10 +# License packing: when a host report shows running frames holding host-based +# licenses, up to this many pending jobs needing those licenses get the first +# shot at that host's idle resources (an extra frame on a seated machine is +# free; a fresh machine burns a seat). 0 disables the packing step entirely +# (no license work is done while handling host reports). +scheduler.license.pack_jobs_max=5 + # Whether to satisfy dependents (*_ON_FRAME and *_ON_LAYER) only on Frame success depend.satisfy_only_on_frame_success=true diff --git a/cuebot/src/test/java/com/imageworks/spcue/config/TestAppConfig.java b/cuebot/src/test/java/com/imageworks/spcue/config/TestAppConfig.java index f6031830d9..a94d59fa8f 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/config/TestAppConfig.java +++ b/cuebot/src/test/java/com/imageworks/spcue/config/TestAppConfig.java @@ -23,6 +23,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.context.annotation.ImportResource; import org.springframework.context.annotation.PropertySource; @@ -37,6 +38,7 @@ "classpath:conf/spring/applicationContext-monitoring.xml", "classpath:conf/spring/applicationContext-accounting.xml"}) @EnableConfigurationProperties +@Import(LicenseConfig.class) @PropertySource({"classpath:opencue.properties"}) public class TestAppConfig { diff --git a/cuebot/src/test/java/com/imageworks/spcue/dispatcher/FrameCompleteHandlerLicenseTests.java b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/FrameCompleteHandlerLicenseTests.java new file mode 100644 index 0000000000..624cbd0979 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/FrameCompleteHandlerLicenseTests.java @@ -0,0 +1,133 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.util.UUID; + +import org.junit.Test; +import org.springframework.mock.env.MockEnvironment; + +import com.imageworks.spcue.DispatchFrame; +import com.imageworks.spcue.DispatchJob; +import com.imageworks.spcue.LayerDetail; +import com.imageworks.spcue.grpc.job.FrameState; +import com.imageworks.spcue.grpc.report.FrameCompleteReport; +import com.imageworks.spcue.grpc.report.RenderHost; +import com.imageworks.spcue.grpc.report.RunningFrameInfo; + +import static org.junit.Assert.assertEquals; + +/** + * Unit tests for the license-denied exit-status handling in + * {@link FrameCompleteHandler#determineFrameState}: a configured denied status requeues the frame + * WAITING instead of marching it to DEAD, the per-frame requeue limit restores ordinary retry + * accounting, and an unconfigured site sees no behaviour change at all. + * + * No Spring context: the handler is constructed directly with a MockEnvironment, which (re)writes + * the static license-denied configuration each time, so every test sets up its own handler first. + */ +public class FrameCompleteHandlerLicenseTests { + + private static final int DENIED_EXIT = 11; + + private FrameCompleteHandler handler(String deniedStatuses, String requeueLimit) { + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.denied_exit_statuses", deniedStatuses); + env.setProperty("scheduler.license.denied_requeue_limit", requeueLimit); + return new FrameCompleteHandler(env); + } + + /** A running frame that has exhausted its retries: DEAD unless something intervenes. */ + private static DispatchFrame retriedOutFrame() { + DispatchFrame frame = new DispatchFrame(); + frame.id = UUID.randomUUID().toString(); + frame.state = FrameState.RUNNING; + frame.retries = 5; + return frame; + } + + private static DispatchJob job() { + DispatchJob job = new DispatchJob(); + job.maxRetries = 1; + job.autoEat = false; + return job; + } + + private static LayerDetail layer() { + LayerDetail layer = new LayerDetail(); + layer.timeout = 0; + layer.timeout_llu = 0; + return layer; + } + + private static FrameCompleteReport report(int exitStatus) { + return FrameCompleteReport.newBuilder().setExitStatus(exitStatus).setExitSignal(0) + .setRunTime(60).setFrame(RunningFrameInfo.newBuilder().setLluTime(0).build()) + .setHost(RenderHost.newBuilder().setNimbyLocked(false).build()).build(); + } + + @Test + public void deniedStatusRequeuesInsteadOfDead() { + handler(String.valueOf(DENIED_EXIT), "10"); + assertEquals("license-denied exit must requeue, not kill", FrameState.WAITING, + FrameCompleteHandler.determineFrameState(job(), layer(), retriedOutFrame(), + report(DENIED_EXIT))); + } + + @Test + public void unconfiguredSiteSeesNoBehaviourChange() { + handler("", "10"); + assertEquals("without configured statuses the same exit must go DEAD", FrameState.DEAD, + FrameCompleteHandler.determineFrameState(job(), layer(), retriedOutFrame(), + report(DENIED_EXIT))); + } + + @Test + public void otherExitStatusesUnaffected() { + handler(String.valueOf(DENIED_EXIT), "10"); + assertEquals(FrameState.DEAD, FrameCompleteHandler.determineFrameState(job(), layer(), + retriedOutFrame(), report(1))); + } + + @Test + public void requeueLimitRestoresNormalAccounting() { + handler(String.valueOf(DENIED_EXIT), "2"); + DispatchFrame frame = retriedOutFrame(); + + assertEquals(FrameState.WAITING, FrameCompleteHandler.determineFrameState(job(), layer(), + frame, report(DENIED_EXIT))); + FrameCompleteHandler.countLicenseDeniedRequeue(frame.getFrameId()); + + assertEquals(FrameState.WAITING, FrameCompleteHandler.determineFrameState(job(), layer(), + frame, report(DENIED_EXIT))); + FrameCompleteHandler.countLicenseDeniedRequeue(frame.getFrameId()); + + assertEquals("over the requeue limit the frame must fall back to ordinary accounting", + FrameState.DEAD, FrameCompleteHandler.determineFrameState(job(), layer(), frame, + report(DENIED_EXIT))); + } + + @Test + public void zeroLimitMeansUnbounded() { + handler(String.valueOf(DENIED_EXIT), "0"); + DispatchFrame frame = retriedOutFrame(); + for (int i = 0; i < 50; i++) { + FrameCompleteHandler.countLicenseDeniedRequeue(frame.getFrameId()); + } + assertEquals(FrameState.WAITING, FrameCompleteHandler.determineFrameState(job(), layer(), + frame, report(DENIED_EXIT))); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseBookingGateTests.java b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseBookingGateTests.java new file mode 100644 index 0000000000..7ecd9e8507 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseBookingGateTests.java @@ -0,0 +1,301 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.ResultSet; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.mock.env.MockEnvironment; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.grpc.report.RunningFrameInfo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link LicenseBookingGate}: the per-pass booking session (floating seats consumed + * per frame, host-based seats free on seated hosts and capped on fresh ones, fail closed on + * stale/unknown/no-provider) and the packing helpers. No Spring context and no database: the + * LicenseSource reads a temp JSON file through the script provider (as in + * {@link LicenseSourceTests}) and layer license declarations are stubbed. + */ +public class LicenseBookingGateTests { + + private Path dir; + private Path json; + + @Before + public void setUp() throws IOException { + dir = Files.createTempDirectory("licgate"); + json = dir.resolve("resp.json"); + } + + @After + public void tearDown() throws IOException { + if (dir != null) { + Files.walk(dir).sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + // best effort + } + }); + } + } + + private void writeJson(String body) throws IOException { + Files.write(json, body.getBytes(StandardCharsets.UTF_8)); + } + + private static long nowSec() { + return System.currentTimeMillis() / 1000L; + } + + /** A polled LicenseSource reading the temp JSON file through the script provider. */ + private LicenseSource polledSource() { + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:cat " + json); + env.setProperty("scheduler.license.timeout_seconds", "5"); + env.setProperty("scheduler.license.stale_seconds", "300"); + LicenseSource ls = new LicenseSource(env, Mockito.mock(JdbcTemplate.class)); + ls.poll(); + return ls; + } + + /** A LicenseSource with no provider configured at all. */ + private LicenseSource noProviderSource() { + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", ""); + return new LicenseSource(env, Mockito.mock(JdbcTemplate.class)); + } + + /** A gate whose layer license declarations come from a map instead of the database. */ + private static final class TestGate extends LicenseBookingGate { + private final Map> layers; + + TestGate(LicenseSource licenseSource, Map> layers) { + super(Mockito.mock(JdbcTemplate.class), licenseSource); + this.layers = layers; + } + + @Override + public List licensesForLayer(String layerId) { + return layers.getOrDefault(layerId, Collections.emptyList()); + } + } + + private static Map> layers(Object... pairs) { + Map> out = new HashMap<>(); + for (int i = 0; i + 1 < pairs.length; i += 2) { + @SuppressWarnings("unchecked") + List names = (List) pairs[i + 1]; + out.put((String) pairs[i], names); + } + return out; + } + + // ---- session: floating ------------------------------------------------ + + @Test + public void unlicensedLayerAlwaysBookable() { + TestGate gate = new TestGate(noProviderSource(), layers()); + assertTrue(gate.newSession("host1").canBook("layer-plain")); + } + + @Test + public void licensedLayerHeldWithoutProvider() { + TestGate gate = new TestGate(noProviderSource(), layers("l1", Arrays.asList("maya"))); + assertFalse(gate.newSession("host1").canBook("l1")); + } + + @Test + public void floatingSeatsConsumedPerBookedFrame() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"maya\", \"total\": 10, \"available\": 2}]}"); + TestGate gate = new TestGate(polledSource(), layers("l1", Arrays.asList("maya"))); + LicenseBookingGate.Session session = gate.newSession("host1"); + assertTrue(session.canBook("l1")); + session.booked("l1"); + assertTrue(session.canBook("l1")); + session.booked("l1"); + assertFalse("both free seats are spent within the pass", session.canBook("l1")); + } + + @Test + public void multiPoolLayerNeedsEveryPool() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"maya\", \"total\": 10, \"available\": 5}, " + + "{\"name\": \"katana\", \"total\": 10, \"available\": 0}]}"); + TestGate gate = new TestGate(polledSource(), layers("both", Arrays.asList("maya", "katana"), + "just-maya", Arrays.asList("maya"))); + LicenseBookingGate.Session session = gate.newSession("host1"); + assertFalse("one empty pool holds the layer", session.canBook("both")); + assertTrue(session.canBook("just-maya")); + } + + @Test + public void unknownLicenseHeld() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"maya\", \"total\": 10, \"available\": 5}]}"); + TestGate gate = new TestGate(polledSource(), layers("l1", Arrays.asList("katana"))); + assertFalse(gate.newSession("host1").canBook("l1")); + } + + @Test + public void staleSampleHolds() throws IOException { + writeJson("{\"queried_at\": " + (nowSec() - 400) + ", \"licenses\": [" + + "{\"name\": \"maya\", \"total\": 10, \"available\": 5}]}"); + TestGate gate = new TestGate(polledSource(), layers("l1", Arrays.asList("maya"))); + assertFalse(gate.newSession("host1").canBook("l1")); + } + + // ---- session: host-based ---------------------------------------------- + + @Test + public void hostBasedFreeOnSeatedHost() throws IOException { + // No seats available, but wolf1018 already holds one: frames there are free. + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"hengine\", \"total\": 8, \"available\": 0, " + + "\"host_based\": true, \"hosts\": [{\"host\": \"wolf1018\", \"count\": 1}]}]}"); + TestGate gate = new TestGate(polledSource(), layers("l1", Arrays.asList("hengine"))); + assertTrue(gate.newSession("wolf1018").canBook("l1")); + assertTrue("host name comparison must be case insensitive", + gate.newSession("WOLF1018").canBook("l1")); + assertFalse("a fresh host would need a seat that does not exist", + gate.newSession("wolf2000").canBook("l1")); + } + + @Test + public void hostBasedNewSeatAllowedWithinCap() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"hengine\", \"total\": 8, \"available\": 1, " + + "\"host_based\": true, \"hosts\": [{\"host\": \"h1\", \"count\": 1}]}]}"); + TestGate gate = new TestGate(polledSource(), layers("l1", Arrays.asList("hengine"))); + LicenseBookingGate.Session session = gate.newSession("h2"); + assertTrue("one seat is free, h2 may take it", session.canBook("l1")); + session.booked("l1"); + assertTrue("h2 is now seated, further frames are free", session.canBook("l1")); + } + + @Test + public void hostBasedSeatCapExhausted() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"hengine\", \"total\": 2, \"available\": 0, " + + "\"host_based\": true, \"hosts\": [" + + "{\"host\": \"h1\", \"count\": 1}, {\"host\": \"h2\", \"count\": 1}]}]}"); + TestGate gate = new TestGate(polledSource(), layers("l1", Arrays.asList("hengine"))); + assertFalse(gate.newSession("h3").canBook("l1")); + assertTrue(gate.newSession("h1").canBook("l1")); + } + + // ---- packing helpers ---------------------------------------------------- + + @Test + public void hostBasedLicensesRunningPicksOnlyHostBased() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"hengine\", \"total\": 8, \"available\": 2, \"host_based\": true}, " + + "{\"name\": \"maya\", \"total\": 10, \"available\": 5}]}"); + TestGate gate = new TestGate(polledSource(), + layers("l-hengine", Arrays.asList("hengine"), "l-maya", Arrays.asList("maya"))); + List running = + Arrays.asList(RunningFrameInfo.newBuilder().setLayerId("l-hengine").build(), + RunningFrameInfo.newBuilder().setLayerId("l-maya").build(), + RunningFrameInfo.newBuilder().setLayerId("l-plain").build()); + assertEquals(Collections.singleton("hengine"), gate.hostBasedLicensesRunning(running)); + } + + @Test + public void hostBasedLicensesRunningEmptyWithoutProvider() { + TestGate gate = new TestGate(noProviderSource(), layers("l1", Arrays.asList("hengine"))); + assertTrue(gate + .hostBasedLicensesRunning( + Arrays.asList(RunningFrameInfo.newBuilder().setLayerId("l1").build())) + .isEmpty()); + } + + @Test + public void hostBasedLicensesRunningEmptyForUnlicensedFrames() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"hengine\", \"total\": 8, \"available\": 2, \"host_based\": true}]}"); + TestGate gate = new TestGate(polledSource(), layers()); + assertTrue(gate + .hostBasedLicensesRunning( + Arrays.asList(RunningFrameInfo.newBuilder().setLayerId("l-plain").build())) + .isEmpty()); + } + + @Test + public void findPackableJobsFiltersOsAndLicenseAndDedupes() throws Exception { + // Rows the packing query would return, highest priority first. + Object[][] rows = { + // matches: wants hengine, no os restriction + {"job1", "", "hengine,katana"}, + // skipped: wrong os + {"job2", "Windows", "hengine"}, + // skipped: needs a license this host does not hold + {"job3", "linux", "maya"}, + // duplicate of job1 via a second licensed layer + {"job1", "", "hengine"}, + // matches: os fits, case-insensitive license match + {"job4", "linux", "HEngine"},}; + + JdbcTemplate jdbc = mock(JdbcTemplate.class); + doAnswer(invocation -> { + RowCallbackHandler handler = invocation.getArgument(1); + for (Object[] row : rows) { + ResultSet rs = mock(ResultSet.class); + when(rs.getString("pk_job")).thenReturn((String) row[0]); + when(rs.getString("str_os")).thenReturn((String) row[1]); + when(rs.getString("str_value")).thenReturn((String) row[2]); + handler.processRow(rs); + } + return null; + }).when(jdbc).query(anyString(), any(RowCallbackHandler.class), any(), any()); + + LicenseBookingGate gate = new LicenseBookingGate(jdbc, noProviderSource()); + DispatchHost host = new DispatchHost(); + host.facilityId = "facility1"; + host.setOs("linux"); + + Set licenses = new HashSet<>(Arrays.asList("hengine")); + assertEquals(Arrays.asList("job1", "job4"), gate.findPackableJobs(licenses, host, 5)); + assertEquals("the limit must cap the result", Arrays.asList("job1"), + gate.findPackableJobs(licenses, host, 1)); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceDescribeTests.java b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceDescribeTests.java new file mode 100644 index 0000000000..f1d7e19f21 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceDescribeTests.java @@ -0,0 +1,227 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.ResultSet; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.mock.env.MockEnvironment; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link LicenseSource#describe}, the read-only view behind the CueGUI Licenses + * widget: poller state, the full cached sample sorted by name, per-license headroom resolution, and + * this cluster's own running usage (all running licensed frames, not the in-flight window). + * + * Same harness as {@link LicenseSourceInFlightTests}: the provider sample comes from a temp JSON + * file behind a script provider, the database rows from a stubbed JdbcTemplate. + */ +public class LicenseSourceDescribeTests { + + private Path dir; + private Path json; + + @Before + public void setUp() throws IOException { + dir = Files.createTempDirectory("licdescribe"); + json = dir.resolve("resp.json"); + } + + @After + public void tearDown() throws IOException { + if (dir != null) { + Files.walk(dir).sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + // best effort + } + }); + } + } + + private void writeJson(String body) throws IOException { + Files.write(json, body.getBytes(StandardCharsets.UTF_8)); + } + + private static long nowSec() { + return System.currentTimeMillis() / 1000L; + } + + /** + * Rows for the running-use query: {lic csv, host}. Replayed through the RowCallbackHandler + * exactly as the database would. + */ + private static JdbcTemplate jdbcReturning(Object[][] rows) { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + doAnswer(invocation -> { + RowCallbackHandler handler = invocation.getArgument(1); + for (Object[] row : rows) { + ResultSet rs = mock(ResultSet.class); + when(rs.getString("lic")).thenReturn((String) row[0]); + when(rs.getString("host")).thenReturn((String) row[1]); + handler.processRow(rs); + } + return null; + }).when(jdbc).query(anyString(), any(RowCallbackHandler.class), any()); + return jdbc; + } + + @Test + public void unconfiguredSourceDescribesItselfHonestly() { + MockEnvironment env = new MockEnvironment(); + LicenseSource ls = new LicenseSource(env, mock(JdbcTemplate.class)); + LicenseSource.SourceStatus status = ls.describe(); + assertFalse(status.configured); + assertFalse(status.hasSample); + assertTrue("no provider means nothing actionable", status.stale); + assertEquals(0, status.ageSeconds); + assertTrue(status.licenses.isEmpty()); + assertEquals("CUE_LICENSES", status.envKey); + } + + @Test + public void configuredButUnpolledReportsNoSample() { + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:cat " + json); + env.setProperty("scheduler.license.poll_seconds", "45"); + env.setProperty("scheduler.license.stale_seconds", "120"); + LicenseSource ls = new LicenseSource(env, mock(JdbcTemplate.class)); + LicenseSource.SourceStatus status = ls.describe(); + assertTrue(status.configured); + assertEquals("script:cat " + json, status.provider); + assertEquals(45, status.pollSeconds); + assertEquals(120, status.staleSeconds); + assertFalse(status.hasSample); + assertTrue(status.stale); + assertTrue(status.licenses.isEmpty()); + } + + @Test + public void describeReportsSampleHeadroomAndRunningUsage() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"Maya\", \"feature\": \"Maya Batch\", \"total\": 10, " + + "\"available\": 5}," + "{\"name\": \"hengine\", \"total\": 8, \"available\": 2, " + + "\"host_based\": true, \"hosts\": [{\"host\": \"WS9\", \"count\": 1}]}]}"); + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:cat " + json); + env.setProperty("scheduler.license.timeout_seconds", "5"); + env.setProperty("scheduler.license.headroom.maya", "2"); + LicenseSource ls = new LicenseSource(env, jdbcReturning(new Object[][] { + // Three maya frames on two hosts; host case must normalize. + {"maya", "H1", null}, {"maya", "h1", null}, {"maya", "h2", null}, + // One hengine frame; csv declaration covers both pools. + {"hengine,maya", "h3", null}, + // A license outside the sample must not invent a row. + {"katana", "h4", null},})); + ls.poll(); + + LicenseSource.SourceStatus status = ls.describe(); + assertTrue(status.configured); + assertTrue(status.hasSample); + assertFalse(status.stale); + assertTrue("fresh sample age", status.ageSeconds < 60); + assertEquals("sorted by name", 2, status.licenses.size()); + + LicenseSource.LicenseInfo hengine = status.licenses.get(0); + assertEquals("hengine", hengine.state.name); + assertEquals("feature falls back to the name", "hengine", hengine.state.feature); + assertTrue(hengine.state.hostBased); + assertEquals(8, hengine.state.total); + assertEquals(2, hengine.state.available); + assertEquals("provider host list is counted", 1, hengine.state.hosts.size()); + assertEquals("default headroom", 0, hengine.headroom); + assertEquals(1, hengine.runningFrames); + assertEquals(1, hengine.runningHosts); + + LicenseSource.LicenseInfo maya = status.licenses.get(1); + assertEquals("maya", maya.state.name); + assertEquals("Maya Batch", maya.state.feature); + assertFalse(maya.state.hostBased); + assertEquals("per-license headroom read from properties", 2, maya.headroom); + assertEquals("three plain rows plus the csv row", 4, maya.runningFrames); + assertEquals("H1/h1 collapse to one host; h2 and h3 add two", 3, maya.runningHosts); + } + + @Test + public void describeRedactsHttpCredentials() { + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", + "http://svc:hunter2@lic-reporter:9101/licenses?token=abc&site=main"); + LicenseSource ls = new LicenseSource(env, mock(JdbcTemplate.class)); + assertEquals("userinfo and query values must not reach the wire", + "http://lic-reporter:9101/licenses?token=****&site=****", ls.describe().provider); + } + + @Test + public void describeLeavesPlainProvidersAlone() { + assertEquals("http://lic:9101/licenses", + LicenseSource.redactProvider("http://lic:9101/licenses")); + assertEquals("an @ past the path is not userinfo", "http://lic:9101/licenses/a@b", + LicenseSource.redactProvider("http://lic:9101/licenses/a@b")); + assertEquals("script command lines are shown as configured", + "script:/site/bin/cue_licenses.sh --token abc", + LicenseSource.redactProvider("script:/site/bin/cue_licenses.sh --token abc")); + } + + @Test + public void redactionSurvivesHostileUrlShapes() { + assertEquals("a raw @ in the password must not leak its tail", "http://host/licenses", + LicenseSource.redactProvider("http://user:p@ss@host/licenses")); + assertEquals("an @ inside the query of a path-less URL is not userinfo", + "http://host?email=****", + LicenseSource.redactProvider("http://host?email=a@b.com")); + } + + @Test + public void staleSampleStillListsLicenses() throws IOException { + // The whole point of the view is seeing what the poller knows, so a + // stale sample keeps its licenses visible and only the flag flips -- + // an operator debugging a dead provider needs the last numbers, not + // an empty table. + writeJson("{\"queried_at\": " + (nowSec() - 1000) + ", \"licenses\": [" + + "{\"name\": \"maya\", \"total\": 10, \"available\": 5}]}"); + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:cat " + json); + env.setProperty("scheduler.license.timeout_seconds", "5"); + env.setProperty("scheduler.license.stale_seconds", "300"); + LicenseSource ls = new LicenseSource(env, jdbcReturning(new Object[][] {})); + ls.poll(); + + LicenseSource.SourceStatus status = ls.describe(); + assertTrue(status.hasSample); + assertTrue("1000s old against a 300s threshold", status.stale); + assertTrue(status.ageSeconds >= 1000); + assertEquals(1, status.licenses.size()); + assertEquals("maya", status.licenses.get(0).state.name); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceInFlightTests.java b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceInFlightTests.java new file mode 100644 index 0000000000..a9655c34fb --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceInFlightTests.java @@ -0,0 +1,163 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.ResultSet; +import java.util.Collections; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.mock.env.MockEnvironment; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link LicenseSource}'s in-flight correction ({@code readInFlight}), exercised + * through {@link LicenseSource#snapshotBudgets}: recently started licensed frames reduce a floating + * budget, our running hosts join a host-based seat set, and recently seated hosts cancel out of the + * seat cap so the sample's {@code available} is not double counted. + * + * The provider sample comes from a temp JSON file (script provider, as in + * {@link LicenseSourceTests}); the database rows come from a stubbed JdbcTemplate driving the + * RowCallbackHandler. + */ +public class LicenseSourceInFlightTests { + + private Path dir; + private Path json; + + @Before + public void setUp() throws IOException { + dir = Files.createTempDirectory("licinflight"); + json = dir.resolve("resp.json"); + } + + @After + public void tearDown() throws IOException { + if (dir != null) { + Files.walk(dir).sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + // best effort + } + }); + } + } + + private void writeJson(String body) throws IOException { + Files.write(json, body.getBytes(StandardCharsets.UTF_8)); + } + + private static long nowSec() { + return System.currentTimeMillis() / 1000L; + } + + /** + * Rows for the in-flight query: {lic csv, host, recent}. The stub replays them through the + * RowCallbackHandler exactly as the database would. + */ + private static JdbcTemplate jdbcReturning(Object[][] rows) { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + doAnswer(invocation -> { + RowCallbackHandler handler = invocation.getArgument(1); + for (Object[] row : rows) { + ResultSet rs = mock(ResultSet.class); + when(rs.getString("lic")).thenReturn((String) row[0]); + when(rs.getString("host")).thenReturn((String) row[1]); + when(rs.getBoolean("recent")).thenReturn((Boolean) row[2]); + handler.processRow(rs); + } + return null; + }).when(jdbc).query(anyString(), any(RowCallbackHandler.class), any(), any()); + return jdbc; + } + + private LicenseSource polledSource(JdbcTemplate jdbc) { + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:cat " + json); + env.setProperty("scheduler.license.timeout_seconds", "5"); + env.setProperty("scheduler.license.stale_seconds", "300"); + LicenseSource ls = new LicenseSource(env, jdbc); + ls.poll(); + return ls; + } + + @Test + public void recentFramesReduceFloatingBudget() throws IOException { + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"maya\", \"total\": 10, \"available\": 5}]}"); + LicenseSource ls = polledSource(jdbcReturning( + new Object[][] {{"maya", "h1", Boolean.TRUE}, {"maya", "h2", Boolean.TRUE}, + // Not recent: the license server has already netted this one out. + {"maya", "h3", Boolean.FALSE}, + // Different license: must not touch maya's budget. + {"katana", "h4", Boolean.TRUE},})); + LicenseSource.LicenseBudget budget = + ls.snapshotBudgets(Collections.singleton("maya")).get("maya"); + assertFalse(budget.stale); + assertEquals("5 available minus 2 in-flight", 3, budget.usable); + } + + @Test + public void runningHostsJoinHostBasedSeats() throws IOException { + // Provider knows h1; we are running the license on h2 (not recent). + // Seats = {h1, h2}; no recent seats, so cap = 2 + available. + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"hengine\", \"total\": 8, \"available\": 1, " + + "\"host_based\": true, \"hosts\": [{\"host\": \"h1\", \"count\": 1}]}]}"); + LicenseSource ls = + polledSource(jdbcReturning(new Object[][] {{"hengine", "H2", Boolean.FALSE},})); + LicenseSource.LicenseBudget budget = + ls.snapshotBudgets(Collections.singleton("hengine")).get("hengine"); + assertTrue(budget.hostBased); + assertTrue("DB host must be lowercased into the seat set", budget.seats.contains("h2")); + assertTrue(budget.seats.contains("h1")); + assertEquals(2, budget.seats.size()); + assertEquals("cap = 2 seats + 1 still available", 3, budget.seatCap); + } + + @Test + public void recentlySeatedHostsCancelOutOfTheCap() throws IOException { + // h2 was seated INSIDE the sample window: it is in the seat set but the + // sample's `available` has not seen it, so it must be netted out of the + // cap -- otherwise every host we seat would raise the cap by one and + // immediately justify another. + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [" + + "{\"name\": \"hengine\", \"total\": 8, \"available\": 1, " + + "\"host_based\": true, \"hosts\": [{\"host\": \"h1\", \"count\": 1}]}]}"); + LicenseSource ls = + polledSource(jdbcReturning(new Object[][] {{"hengine", "h2", Boolean.TRUE},})); + LicenseSource.LicenseBudget budget = + ls.snapshotBudgets(Collections.singleton("hengine")).get("hengine"); + assertEquals(2, budget.seats.size()); + assertEquals("cap = 2 seats + max(0, 1 available - 1 recent seat)", 2, budget.seatCap); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceTests.java b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceTests.java new file mode 100644 index 0000000000..9524c3fe29 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/LicenseSourceTests.java @@ -0,0 +1,242 @@ +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.mock.env.MockEnvironment; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for {@link LicenseSource}'s provider contract: timestamp validation (a sample without + * a usable {@code queried_at} must be rejected, not treated as fresh), staleness, headroom, + * counts-only seat capping, and the script provider's stderr handling (a chatty script must not + * wedge on a full stderr pipe). No Spring context and no database: the JdbcTemplate is a mock whose + * queries return nothing, so in-flight counts are zero. The test lives in the dispatcher package to + * reach the package-private {@code poll()}. + */ +public class LicenseSourceTests { + + private Path dir; + private Path json; + private Path script; + + @Before + public void setUp() throws IOException { + dir = Files.createTempDirectory("licsrc"); + json = dir.resolve("resp.json"); + } + + @After + public void tearDown() throws IOException { + if (dir != null) { + Files.walk(dir).sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + // best effort + } + }); + } + } + + /** A LicenseSource reading the temp JSON file through the script provider. */ + private LicenseSource source(String... extraProps) { + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:cat " + json); + env.setProperty("scheduler.license.timeout_seconds", "5"); + env.setProperty("scheduler.license.stale_seconds", "300"); + for (int i = 0; i + 1 < extraProps.length; i += 2) { + env.setProperty(extraProps[i], extraProps[i + 1]); + } + JdbcTemplate jdbc = Mockito.mock(JdbcTemplate.class); + return new LicenseSource(env, jdbc); + } + + private void writeJson(String body) throws IOException { + Files.write(json, body.getBytes(StandardCharsets.UTF_8)); + } + + private static String mayaJson(long queriedAt, int total, int available) { + String ts = (queriedAt == Long.MIN_VALUE) ? "" : "\"queried_at\": " + queriedAt + ", "; + return "{" + ts + "\"licenses\": [{\"name\": \"maya\", \"total\": " + total + + ", \"available\": " + available + "}]}"; + } + + private static long nowSec() { + return System.currentTimeMillis() / 1000L; + } + + private static final Set MAYA = Collections.singleton("maya"); + + // ---- queried_at validation -------------------------------------------- + + @Test + public void validTimestampAccepted() throws IOException { + writeJson(mayaJson(nowSec(), 10, 5)); + LicenseSource ls = source(); + ls.poll(); + LicenseSource.LicenseBudget b = ls.snapshotBudgets(MAYA).get("maya"); + assertFalse("fresh sample must not be stale", b.stale); + assertEquals(5, b.usable); + } + + @Test + public void missingTimestampRejected() throws IOException { + writeJson(mayaJson(Long.MIN_VALUE, 10, 5)); + LicenseSource ls = source(); + ls.poll(); + assertTrue("sample without queried_at must be held", + ls.snapshotBudgets(MAYA).get("maya").stale); + } + + @Test + public void zeroTimestampRejected() throws IOException { + writeJson(mayaJson(0, 10, 5)); + LicenseSource ls = source(); + ls.poll(); + assertTrue(ls.snapshotBudgets(MAYA).get("maya").stale); + } + + @Test + public void negativeTimestampRejected() throws IOException { + writeJson(mayaJson(-42, 10, 5)); + LicenseSource ls = source(); + ls.poll(); + assertTrue(ls.snapshotBudgets(MAYA).get("maya").stale); + } + + @Test + public void futureTimestampClampedToFresh() throws IOException { + writeJson(mayaJson(nowSec() + 120, 10, 5)); + LicenseSource ls = source(); + ls.poll(); + assertFalse("a provider clock running ahead is clamped, not rejected", + ls.snapshotBudgets(MAYA).get("maya").stale); + } + + @Test + public void oldTimestampIsStaleOnArrival() throws IOException { + writeJson(mayaJson(nowSec() - 400, 10, 5)); + LicenseSource ls = source(); // stale_seconds=300 + ls.poll(); + assertTrue("lagging sample must arrive already stale", + ls.snapshotBudgets(MAYA).get("maya").stale); + } + + @Test + public void rejectedPollKeepsPreviousSample() throws IOException { + writeJson(mayaJson(nowSec(), 10, 5)); + LicenseSource ls = source(); + ls.poll(); + // Provider degrades: same numbers, no timestamp. The poll is rejected + // and the previous (still young) sample keeps serving. + writeJson(mayaJson(Long.MIN_VALUE, 10, 1)); + ls.poll(); + LicenseSource.LicenseBudget b = ls.snapshotBudgets(MAYA).get("maya"); + assertFalse(b.stale); + assertEquals("previous sample's numbers must survive the rejected poll", 5, b.usable); + } + + // ---- budgets ---------------------------------------------------------- + + @Test + public void headroomSubtracted() throws IOException { + writeJson(mayaJson(nowSec(), 10, 10)); + LicenseSource ls = source("scheduler.license.headroom.maya", "4"); + ls.poll(); + assertEquals(6, ls.snapshotBudgets(MAYA).get("maya").usable); + } + + @Test + public void unknownLicenseHeld() throws IOException { + writeJson(mayaJson(nowSec(), 10, 5)); + LicenseSource ls = source(); + ls.poll(); + Map out = + ls.snapshotBudgets(Collections.singleton("katana")); + assertTrue("a license the provider does not report must be held", out.get("katana").stale); + } + + @Test + public void noProviderHoldsEverything() { + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", " "); + LicenseSource ls = new LicenseSource(env, Mockito.mock(JdbcTemplate.class)); + assertFalse(ls.hasProvider()); + assertTrue(ls.snapshotBudgets(MAYA).get("maya").stale); + } + + @Test + public void countsOnlyHostBasedCapBoundedByAvailable() throws IOException { + // host_based without a hosts list (the sesictrl case): the seat cap + // must be bounded via `available`, never total - headroom. + writeJson("{\"queried_at\": " + nowSec() + ", \"licenses\": [{\"name\": \"hengine\", " + + "\"total\": 8, \"available\": 3, \"host_based\": true}]}"); + LicenseSource ls = source(); + ls.poll(); + LicenseSource.LicenseBudget b = + ls.snapshotBudgets(Collections.singleton("hengine")).get("hengine"); + assertFalse(b.stale); + assertTrue(b.hostBased); + assertEquals("no seats reported, no in-flight: cap = available", 3, b.seatCap); + } + + // ---- script provider robustness --------------------------------------- + + @Test + public void chattyStderrDoesNotWedgeTheScript() throws IOException { + // 200KB of stderr, far over the ~64KB pipe buffer: without a stderr + // drain the script blocks mid-write and times out. + writeJson(mayaJson(nowSec(), 10, 5)); + script = dir.resolve("chatty.sh"); + Files.write(script, ("#!/bin/sh\n" + "head -c 200000 /dev/zero | tr '\\0' 'e' 1>&2\n" + + "cat " + json + "\n").getBytes(StandardCharsets.UTF_8)); + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:sh " + script); + env.setProperty("scheduler.license.timeout_seconds", "5"); + LicenseSource ls = new LicenseSource(env, Mockito.mock(JdbcTemplate.class)); + ls.poll(); + assertFalse("valid stdout must win despite a flooded stderr", + ls.snapshotBudgets(MAYA).get("maya").stale); + } + + @Test + public void failingScriptHoldsLicenses() throws IOException { + script = dir.resolve("fail.sh"); + Files.write(script, ("#!/bin/sh\n" + "echo 'vendor said no' 1>&2\n" + "exit 3\n") + .getBytes(StandardCharsets.UTF_8)); + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:sh " + script); + env.setProperty("scheduler.license.timeout_seconds", "5"); + LicenseSource ls = new LicenseSource(env, Mockito.mock(JdbcTemplate.class)); + ls.poll(); + assertTrue(ls.snapshotBudgets(MAYA).get("maya").stale); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/dispatcher/ManageLicenseTests.java b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/ManageLicenseTests.java new file mode 100644 index 0000000000..029139c4b5 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/ManageLicenseTests.java @@ -0,0 +1,206 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.ResultSet; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; +import org.springframework.mock.env.MockEnvironment; + +import com.imageworks.spcue.servant.ManageLicense; + +import com.imageworks.spcue.grpc.license.License; +import com.imageworks.spcue.grpc.license.LicenseFindRequest; +import com.imageworks.spcue.grpc.license.LicenseFindResponse; +import com.imageworks.spcue.grpc.license.LicenseGetAllRequest; +import com.imageworks.spcue.grpc.license.LicenseGetAllResponse; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for the read-only {@link ManageLicense} servant: proto conversion of the + * {@link LicenseSource#describe} view, the unconfigured empty response, and Find's case + * normalization and NOT_FOUND path. No Spring and no database, matching the other licensing tests: + * the servant is driven directly with a recording observer. + */ +public class ManageLicenseTests { + + /** Captures the single response the servant emits. */ + private static final class RecordingObserver implements StreamObserver { + T value; + Throwable error; + boolean completed; + + @Override + public void onNext(T v) { + value = v; + } + + @Override + public void onError(Throwable t) { + error = t; + } + + @Override + public void onCompleted() { + completed = true; + } + } + + private Path dir; + private Path json; + + @Before + public void setUp() throws IOException { + dir = Files.createTempDirectory("licservant"); + json = dir.resolve("resp.json"); + } + + @After + public void tearDown() throws IOException { + if (dir != null) { + Files.walk(dir).sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + // best effort + } + }); + } + } + + private static JdbcTemplate jdbcReturning(Object[][] rows) { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + doAnswer(invocation -> { + RowCallbackHandler handler = invocation.getArgument(1); + for (Object[] row : rows) { + ResultSet rs = mock(ResultSet.class); + when(rs.getString("lic")).thenReturn((String) row[0]); + when(rs.getString("host")).thenReturn((String) row[1]); + handler.processRow(rs); + } + return null; + }).when(jdbc).query(anyString(), any(RowCallbackHandler.class), any()); + return jdbc; + } + + /** A servant over a polled source: hengine host-based, maya floating with headroom 2. */ + private ManageLicense polledServant() throws IOException { + Files.write(json, + ("{\"queried_at\": " + System.currentTimeMillis() / 1000L + ", \"licenses\": [" + + "{\"name\": \"hengine\", \"feature\": \"Houdini Engine\", \"total\": 8, " + + "\"available\": 2, \"host_based\": true, " + + "\"hosts\": [{\"host\": \"ws1\", \"count\": 1}]}," + + "{\"name\": \"maya\", \"total\": 10, \"available\": 5}]}") + .getBytes(StandardCharsets.UTF_8)); + MockEnvironment env = new MockEnvironment(); + env.setProperty("scheduler.license.provider", "script:cat " + json); + env.setProperty("scheduler.license.timeout_seconds", "5"); + env.setProperty("scheduler.license.headroom.maya", "2"); + LicenseSource source = new LicenseSource(env, + jdbcReturning(new Object[][] {{"maya", "h1", null}, {"maya", "h2", null},})); + source.poll(); + ManageLicense servant = new ManageLicense(); + servant.setLicenseSource(source); + return servant; + } + + @Test + public void getAllConvertsTheViewToProtos() throws IOException { + ManageLicense servant = polledServant(); + RecordingObserver observer = new RecordingObserver<>(); + servant.getAll(LicenseGetAllRequest.newBuilder().build(), observer); + + assertTrue(observer.completed); + assertNotNull(observer.value); + assertTrue(observer.value.getSource().getConfigured()); + assertTrue(observer.value.getSource().getHasSample()); + assertFalse(observer.value.getSource().getStale()); + assertEquals("CUE_LICENSES", observer.value.getSource().getEnvKey()); + assertEquals(2, observer.value.getLicensesCount()); + + License hengine = observer.value.getLicenses(0); + assertEquals("hengine", hengine.getName()); + assertEquals("Houdini Engine", hengine.getFeature()); + assertEquals(8, hengine.getTotal()); + assertEquals(2, hengine.getAvailable()); + assertTrue(hengine.getHostBased()); + assertEquals(1, hengine.getProviderHostCount()); + assertEquals(0, hengine.getRunningFrames()); + + License maya = observer.value.getLicenses(1); + assertEquals("maya", maya.getName()); + assertEquals(2, maya.getHeadroom()); + assertEquals(2, maya.getRunningFrames()); + assertEquals(2, maya.getRunningHosts()); + } + + @Test + public void getAllWithoutProviderReturnsUnconfiguredStatus() { + ManageLicense servant = new ManageLicense(); + servant.setLicenseSource( + new LicenseSource(new MockEnvironment(), mock(JdbcTemplate.class))); + RecordingObserver observer = new RecordingObserver<>(); + servant.getAll(LicenseGetAllRequest.newBuilder().build(), observer); + + assertTrue(observer.completed); + assertFalse(observer.value.getSource().getConfigured()); + assertTrue(observer.value.getSource().getStale()); + assertEquals(0, observer.value.getLicensesCount()); + } + + @Test + public void findNormalizesCaseAndFinds() throws IOException { + ManageLicense servant = polledServant(); + RecordingObserver observer = new RecordingObserver<>(); + servant.find(LicenseFindRequest.newBuilder().setName(" Maya ").build(), observer); + + assertTrue(observer.completed); + assertEquals("maya", observer.value.getLicense().getName()); + assertEquals(10, observer.value.getLicense().getTotal()); + } + + @Test + public void findUnknownLicenseIsNotFound() throws IOException { + ManageLicense servant = polledServant(); + RecordingObserver observer = new RecordingObserver<>(); + servant.find(LicenseFindRequest.newBuilder().setName("katana").build(), observer); + + assertFalse(observer.completed); + assertNotNull(observer.error); + assertEquals(Status.Code.NOT_FOUND, + ((StatusRuntimeException) observer.error).getStatus().getCode()); + } +} diff --git a/cuebot/src/test/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLicensePackTests.java b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLicensePackTests.java new file mode 100644 index 0000000000..71d9d1c629 --- /dev/null +++ b/cuebot/src/test/java/com/imageworks/spcue/dispatcher/commands/DispatchBookHostLicensePackTests.java @@ -0,0 +1,137 @@ + +/* + * Copyright Contributors to the OpenCue Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.imageworks.spcue.dispatcher.commands; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.InOrder; +import org.springframework.mock.env.MockEnvironment; + +import com.imageworks.spcue.DispatchHost; +import com.imageworks.spcue.JobEntity; +import com.imageworks.spcue.JobInterface; +import com.imageworks.spcue.ShowEntity; +import com.imageworks.spcue.dispatcher.Dispatcher; +import com.imageworks.spcue.dispatcher.LicenseBookingGate; +import com.imageworks.spcue.grpc.report.RunningFrameInfo; +import com.imageworks.spcue.service.JobManager; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link DispatchBookHostLicensePack}: pack jobs are dispatched before the normal + * booking order, the preferred show survives packing, and a pack pass that finds nothing still + * falls through to the exact booking the host would have received without packing. + */ +public class DispatchBookHostLicensePackTests { + + private static final List RUNNING = + Collections.singletonList(RunningFrameInfo.newBuilder().setLayerId("l1").build()); + + private DispatchHost host; + private LicenseBookingGate gate; + private JobManager jobManager; + private Dispatcher dispatcher; + private MockEnvironment env; + + @Before + public void setUp() { + host = new DispatchHost(); + host.id = "host1"; + host.name = "host1"; + host.idleCores = 800; + host.idleMemory = 8_000_000L; + host.idleGpus = 0; + host.idleGpuMemory = 0; + + gate = mock(LicenseBookingGate.class); + jobManager = mock(JobManager.class); + dispatcher = mock(Dispatcher.class); + env = new MockEnvironment(); + env.setProperty("dispatcher.memory.mem_reserved_min", "262144"); + env.setProperty("dispatcher.memory.mem_gpu_reserved_min", "0"); + } + + private DispatchBookHostLicensePack command(ShowEntity preferredShow) { + return new DispatchBookHostLicensePack(host, RUNNING, preferredShow, gate, jobManager, + dispatcher, env); + } + + @Test + public void packJobsDispatchedBeforeNormalBooking() { + JobInterface job = new JobEntity("job1"); + when(gate.hostBasedLicensesRunning(RUNNING)) + .thenReturn(new HashSet<>(Arrays.asList("hengine"))); + when(gate.findPackableJobs(anySet(), eq(host), anyInt())).thenReturn(Arrays.asList("job1")); + when(jobManager.getJob("job1")).thenReturn(job); + + command(null).run(); + + InOrder order = inOrder(dispatcher); + order.verify(dispatcher).dispatchHost(host, job); + order.verify(dispatcher).dispatchHost(host); + } + + @Test + public void preferredShowSurvivesPacking() { + ShowEntity show = new ShowEntity(); + show.id = "show1"; + show.name = "show1"; + JobInterface job = new JobEntity("job1"); + when(gate.hostBasedLicensesRunning(RUNNING)) + .thenReturn(new HashSet<>(Arrays.asList("hengine"))); + when(gate.findPackableJobs(anySet(), eq(host), anyInt())).thenReturn(Arrays.asList("job1")); + when(jobManager.getJob("job1")).thenReturn(job); + + command(show).run(); + + InOrder order = inOrder(dispatcher); + order.verify(dispatcher).dispatchHost(host, job); + order.verify(dispatcher).dispatchHost(host, show); + } + + @Test + public void emptyPackStillBooksNormally() { + when(gate.hostBasedLicensesRunning(RUNNING)).thenReturn(Collections.emptySet()); + + command(null).run(); + + verify(dispatcher, never()).dispatchHost(eq(host), any(JobInterface.class)); + verify(dispatcher).dispatchHost(host); + } + + @Test + public void gateFailureStillBooksNormally() { + when(gate.hostBasedLicensesRunning(RUNNING)).thenThrow(new RuntimeException("db down")); + + command(null).run(); + + verify(dispatcher).dispatchHost(host); + } +} diff --git a/cuegui/cuegui/Constants.py b/cuegui/cuegui/Constants.py index cfb7665e9f..ee2e5af6e2 100644 --- a/cuegui/cuegui/Constants.py +++ b/cuegui/cuegui/Constants.py @@ -250,6 +250,7 @@ def __get_version_from_cmd(command): TYPE_SUB = QtWidgets.QTreeWidgetItem.UserType + 14 TYPE_TASK = QtWidgets.QTreeWidgetItem.UserType + 15 TYPE_LIMIT = QtWidgets.QTreeWidgetItem.UserType + 16 +TYPE_LICENSE = QtWidgets.QTreeWidgetItem.UserType + 17 QVARIANT_NULL = None QT_MAX_INT = 2147483647 diff --git a/cuegui/cuegui/LicensesWidget.py b/cuegui/cuegui/LicensesWidget.py new file mode 100644 index 0000000000..23b7d1da51 --- /dev/null +++ b/cuegui/cuegui/LicensesWidget.py @@ -0,0 +1,216 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Widget for inspecting application licenses. + +Read-only by design: the numbers belong to the license server Cuebot polls +(``scheduler.license.provider``), and the tuning (provider, headroom, poll +cadence) lives in ``opencue.properties``. This view shows what the poller +currently knows and how much of each pool this OpenCue deploy is using. +""" + + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +from qtpy import QtCore +from qtpy import QtWidgets + +import opencue + +import cuegui.AbstractTreeWidget +import cuegui.AbstractWidgetItem +import cuegui.Constants +import cuegui.Logger +import cuegui.Utils + + +logger = cuegui.Logger.getLogger(__file__) + + +def _licenseType(lic): + """Display string for the license kind.""" + return "Host-based" if lic.hostBased() else "Floating" + + +class LicensesWidget(QtWidgets.QWidget): + """Widget for inspecting application licenses (read-only).""" + + def __init__(self, parent): + QtWidgets.QWidget.__init__(self, parent) + + self.__lblStatus = QtWidgets.QLabel("Loading license status...", self) + self.__lblStatus.setContentsMargins(4, 2, 4, 2) + self.__btnRefresh = QtWidgets.QPushButton("Refresh", self) + self.__btnRefresh.setFocusPolicy(QtCore.Qt.NoFocus) + + self.__monitorLicenses = LicensesTreeWidget(self) + + layout = QtWidgets.QGridLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(0) + layout.addWidget(self.__lblStatus, 0, 0, 1, 3) + layout.addWidget(self.__btnRefresh, 0, 3) + layout.addWidget(self.__monitorLicenses, 2, 0, 3, 4) + + # pylint: disable=no-member + self.__btnRefresh.clicked.connect(self.updateSoon) + # pylint: enable=no-member + self.__monitorLicenses.status_update.connect(self.__setStatus) + + def updateSoon(self): + """Requests a refresh of the license list.""" + # pylint: disable=protected-access + self.__monitorLicenses._update() + + def __setStatus(self, status): + """Renders the poller status line above the license table.""" + if status is None: + # The last refresh could not reach Cuebot: the table was emptied, + # so the status line must not keep promising a healthy provider. + self.__lblStatus.setText( + "License status unavailable — could not reach Cuebot.") + return + if not status.configured(): + text = ("License provider not configured (scheduler.license.provider); " + "layers declaring licenses are held.") + elif not status.hasSample(): + text = "Waiting for the first sample from %s" % status.provider() + else: + text = "Provider %s — sample %ds old (stale after %ds), poll every %ds" % ( + status.provider(), status.ageSeconds(), status.staleSeconds(), + status.pollSeconds()) + if status.stale(): + text = "STALE — licensed layers are held. " + text + self.__lblStatus.setText(text) + + def getColumnVisibility(self): + """Gets the table column visibility.""" + return self.__monitorLicenses.getColumnVisibility() + + def setColumnVisibility(self, settings): + """Sets the table column visibility.""" + self.__monitorLicenses.setColumnVisibility(settings) + + def getColumnOrder(self): + """Gets the table column order.""" + return self.__monitorLicenses.getColumnOrder() + + def setColumnOrder(self, settings): + """Sets the table column order.""" + self.__monitorLicenses.setColumnOrder(settings) + + +class LicensesTreeWidget(cuegui.AbstractTreeWidget.AbstractTreeWidget): + """Tree widget for displaying the licenses in the current provider sample.""" + + status_update = QtCore.Signal(object) + """Emitted with the :class:`opencue.wrappers.license.LicensingStatus` of the + last refresh, on the main thread, after the table has been updated.""" + + def __init__(self, parent): + self.startColumnsForType(cuegui.Constants.TYPE_LICENSE) + self.addColumn("License", 90, id=1, + data=lambda license: license.name()) + self.addColumn("Feature", 130, id=2, + data=lambda license: license.feature()) + self.addColumn("Type", 80, id=3, + data=_licenseType, + tip="Floating licenses consume one seat per frame.\n" + "Host-based licenses consume one seat per machine, shared\n" + "by every frame on it.") + self.addColumn("Total", 60, id=4, + data=lambda license: ("%d" % license.total()), + sort=lambda license: license.total(), + tip="Seats the license server owns.") + self.addColumn("Available", 75, id=5, + data=lambda license: ("%d" % license.available()), + sort=lambda license: license.available(), + tip="Seats the license server reports free, net of every\n" + "consumer: workstations, CI and other farms included.") + self.addColumn("In Use", 60, id=6, + data=lambda license: ("%d" % license.inUse()), + sort=lambda license: license.inUse(), + tip="Seats checked out anywhere, per the license server\n" + "(total minus available).") + self.addColumn("Headroom", 75, id=7, + data=lambda license: ("%d" % license.headroom()), + sort=lambda license: license.headroom(), + tip="Seats deliberately withheld from the farm for interactive\n" + "users (scheduler.license.headroom.).") + self.addColumn("Cue Frames", 80, id=8, + data=lambda license: ("%d" % license.runningFrames()), + sort=lambda license: license.runningFrames(), + tip="Frames currently running on this OpenCue deploy whose\n" + "layer declares this license.") + self.addColumn("Cue Hosts", 75, id=9, + data=lambda license: ("%d" % license.runningHosts()), + sort=lambda license: license.runningHosts(), + tip="Distinct hosts running those frames.") + self.addColumn("Provider Hosts", 95, id=10, + data=lambda license: ("%d" % license.providerHostCount()), + sort=lambda license: license.providerHostCount(), + tip="Hosts the license server reports holding a seat, any\n" + "consumer; 0 when the provider does not report hosts.") + + # Before the base __init__, which may run a synchronous first update + # through _processUpdate when no threadpool exists. + self.__lastStatus = None + + cuegui.AbstractTreeWidget.AbstractTreeWidget.__init__(self, parent) + + self.app.facility_changed.connect(self.__facilityChanged) + + self.setUpdateInterval(60) + + def __facilityChanged(self): + """Called when the facility is changed""" + self.removeAllItems() + self._update() + + def _createItem(self, rpcObject): + """Creates and returns the proper item""" + return LicenseWidgetItem(rpcObject, self) + + def _getUpdate(self): + """Returns the licenses in Cuebot's current provider sample.""" + try: + status = opencue.api.getLicensingStatus() + self.__lastStatus = status + return status.licenses() + except opencue.exception.CueException as e: + list(map(logger.warning, cuegui.Utils.exceptionOutput(e))) + # A stale status must not outlive the rows it described: the + # empty return clears the table, so clear the status with it. + self.__lastStatus = None + return [] + + def _processUpdate(self, work, rpcObjects): + """Updates the table, then publishes the poller status of the same + refresh; runs on the main thread.""" + super(LicensesTreeWidget, self)._processUpdate(work, rpcObjects) + self.status_update.emit(self.__lastStatus) + + def tick(self): + pass + + +class LicenseWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): + """Widget item for displaying a single license.""" + + def __init__(self, rpcObject, parent): + cuegui.AbstractWidgetItem.AbstractWidgetItem.__init__( + self, cuegui.Constants.TYPE_LICENSE, rpcObject, parent) diff --git a/cuegui/cuegui/plugins/LicensesPlugin.py b/cuegui/cuegui/plugins/LicensesPlugin.py new file mode 100644 index 0000000000..13bb530993 --- /dev/null +++ b/cuegui/cuegui/plugins/LicensesPlugin.py @@ -0,0 +1,49 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Plugin for inspecting application licenses (read-only).""" + + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import cuegui.AbstractDockWidget +import cuegui.LicensesWidget + + +PLUGIN_NAME = "Licenses" +PLUGIN_CATEGORY = "Cuecommander" +PLUGIN_DESCRIPTION = "A read-only view of the application licenses Cuebot polls" +PLUGIN_REQUIRES = "CueCommander" +PLUGIN_PROVIDES = "LicensesDockWidget" + + +class LicensesDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget): + """Plugin for inspecting application licenses (read-only).""" + + def __init__(self, parent): + super(LicensesDockWidget, self).__init__(parent, PLUGIN_NAME) + + self.__licensesWidget = cuegui.LicensesWidget.LicensesWidget(self) + + self.layout().addWidget(self.__licensesWidget) + + self.pluginRegisterSettings([("columnVisibility", + self.__licensesWidget.getColumnVisibility, + self.__licensesWidget.setColumnVisibility), + ("columnOrder", + self.__licensesWidget.getColumnOrder, + self.__licensesWidget.setColumnOrder)]) diff --git a/cuegui/tests/test_licenses_widget.py b/cuegui/tests/test_licenses_widget.py new file mode 100644 index 0000000000..e2a26590e7 --- /dev/null +++ b/cuegui/tests/test_licenses_widget.py @@ -0,0 +1,160 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Tests for cuegui.LicensesWidget.""" + + +import unittest + +import mock +import qtpy.QtCore +import qtpy.QtWidgets + +import opencue.wrappers.license +import opencue_proto.license_pb2 + +import cuegui.LicensesWidget +import cuegui.Style + +from . import test_utils + + +def _license(name='hengine', feature='Houdini Engine', host_based=True): + return opencue.wrappers.license.License( + opencue_proto.license_pb2.License( + name=name, + feature=feature, + total=800, + available=794, + host_based=host_based, + headroom=5, + running_frames=12, + running_hosts=3, + provider_host_count=2)) + + +def _status(configured=True, has_sample=True, stale=False, licenses=None): + return opencue.wrappers.license.LicensingStatus( + opencue_proto.license_pb2.LicenseGetAllResponse( + source=opencue_proto.license_pb2.LicenseSourceStatus( + configured=configured, + provider='http://lic-reporter:9101/licenses' if configured else '', + env_key='CUE_LICENSES', + poll_seconds=20, + stale_seconds=300, + has_sample=has_sample, + age_seconds=17 if has_sample else 0, + stale=stale), + licenses=licenses or [])) + + +@mock.patch('opencue.cuebot.Cuebot.getStub', new=mock.Mock()) +class LicensesWidgetTests(unittest.TestCase): + + def setUp(self): + app = test_utils.createApplication() + app.settings = qtpy.QtCore.QSettings() + cuegui.Style.init() + # Kept as instance attr so the parent isn't garbage-collected mid-test. + self.parentWidget = qtpy.QtWidgets.QWidget() + self.widget = cuegui.LicensesWidget.LicensesWidget(self.parentWidget) + self.tree = self.widget.findChild(cuegui.LicensesWidget.LicensesTreeWidget) + self.label = self.widget.findChild(qtpy.QtWidgets.QLabel) + + def test_itemDisplaysLicenseColumns(self): + item = cuegui.LicensesWidget.LicenseWidgetItem(_license(), self.tree) + + displayRole = qtpy.QtCore.Qt.DisplayRole + self.assertEqual('hengine', item.data(0, displayRole)) + self.assertEqual('Houdini Engine', item.data(1, displayRole)) + self.assertEqual('Host-based', item.data(2, displayRole)) + self.assertEqual('800', item.data(3, displayRole)) + self.assertEqual('794', item.data(4, displayRole)) + self.assertEqual('6', item.data(5, displayRole)) + self.assertEqual('5', item.data(6, displayRole)) + self.assertEqual('12', item.data(7, displayRole)) + self.assertEqual('3', item.data(8, displayRole)) + self.assertEqual('2', item.data(9, displayRole)) + + def test_floatingTypeString(self): + item = cuegui.LicensesWidget.LicenseWidgetItem( + _license(name='maya', host_based=False), self.tree) + + self.assertEqual('Floating', item.data(2, qtpy.QtCore.Qt.DisplayRole)) + + def test_updateFillsTableAndStatusLine(self): + status = _status(licenses=[ + opencue_proto.license_pb2.License(name='hengine', feature='Houdini Engine')]) + + with mock.patch.object(opencue.api, 'getLicensingStatus', return_value=status): + # pylint: disable=protected-access + rpcObjects = self.tree._getUpdate() + self.tree._processUpdate(None, rpcObjects) + + self.assertEqual(1, self.tree.topLevelItemCount()) + self.assertIn('http://lic-reporter:9101/licenses', self.label.text()) + self.assertNotIn('STALE', self.label.text()) + + def test_staleSampleIsCalledOutInStatusLine(self): + status = _status(stale=True) + + with mock.patch.object(opencue.api, 'getLicensingStatus', return_value=status): + # pylint: disable=protected-access + self.tree._processUpdate(None, self.tree._getUpdate()) + + self.assertIn('STALE', self.label.text()) + self.assertIn('held', self.label.text()) + + def test_unconfiguredProviderIsExplained(self): + status = _status(configured=False, has_sample=False) + + with mock.patch.object(opencue.api, 'getLicensingStatus', return_value=status): + # pylint: disable=protected-access + self.tree._processUpdate(None, self.tree._getUpdate()) + + self.assertIn('not configured', self.label.text()) + + def test_failedFetchKeepsGoing(self): + with mock.patch.object( + opencue.api, 'getLicensingStatus', + side_effect=opencue.exception.CueException('boom')): + # pylint: disable=protected-access + rpcObjects = self.tree._getUpdate() + + self.assertEqual([], rpcObjects) + + def test_failedRefreshDoesNotKeepPromisingAHealthyProvider(self): + # A good refresh, then Cuebot goes away: the table empties, so the + # status line must stop showing the stale healthy provider text. + good = _status(licenses=[ + opencue_proto.license_pb2.License(name='hengine', feature='Houdini Engine')]) + with mock.patch.object(opencue.api, 'getLicensingStatus', return_value=good): + # pylint: disable=protected-access + self.tree._processUpdate(None, self.tree._getUpdate()) + self.assertIn('http://lic-reporter:9101/licenses', self.label.text()) + + with mock.patch.object( + opencue.api, 'getLicensingStatus', + side_effect=opencue.exception.CueException('boom')): + # pylint: disable=protected-access + self.tree._processUpdate(None, self.tree._getUpdate()) + + self.assertEqual(0, self.tree.topLevelItemCount()) + self.assertIn('unavailable', self.label.text()) + self.assertNotIn('http://lic-reporter:9101/licenses', self.label.text()) + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/_docs/developer-guide/cuecommander-technical-reference.md b/docs/_docs/developer-guide/cuecommander-technical-reference.md index c9d63317a9..9098c45d38 100644 --- a/docs/_docs/developer-guide/cuecommander-technical-reference.md +++ b/docs/_docs/developer-guide/cuecommander-technical-reference.md @@ -27,6 +27,7 @@ CueCommander │ └── Settings Manager ├── Plugins │ ├── AllocationsPlugin +│ ├── LicensesPlugin │ ├── LimitsPlugin │ ├── MonitorCuePlugin │ ├── MonitorHostsPlugin @@ -83,6 +84,49 @@ class MonitorAllocations(AbstractTreeWidget): --- +### LicensesPlugin + +**Module**: `cuegui.plugins.LicensesPlugin` +**Widget**: `LicensesDockWidget` +**Data Source**: `opencue.api.getLicensingStatus()` + +#### Class Structure + +```python +class LicensesDockWidget(AbstractDockWidget): + - __licensesWidget: LicensesWidget + - pluginRegisterSettings() + +class LicensesTreeWidget(AbstractTreeWidget): + - Update interval: 60 seconds + - Read-only: no context menu, no mutations + - status_update signal: publishes the poller status after each refresh +``` + +#### Data Model + +| Field | Type | Description | +|-------|------|-------------| +| name | str | License pool name, e.g. `hengine` | +| feature | str | Human-readable feature name | +| total | int | Seats the license server owns | +| available | int | Seats free, net of every consumer | +| host_based | bool | One seat per machine vs per frame | +| headroom | int | Seats withheld for interactive users | +| running_frames | int | Frames running on this deploy declaring the license | +| running_hosts | int | Distinct hosts running those frames | +| provider_host_count | int | Hosts the provider reports holding a seat | + +#### Operations + +Read-only — the numbers come from the license server Cuebot polls +(`scheduler.license.provider`); tuning lives in `opencue.properties`. + +- Query all: `opencue.api.getLicenses()` / `opencue.api.getLicensingStatus()` +- Query one: `opencue.api.findLicense(name)` + +--- + ### LimitsPlugin **Module**: `cuegui.plugins.LimitsPlugin` diff --git a/docs/_docs/other-guides/configuring-application-licenses.md b/docs/_docs/other-guides/configuring-application-licenses.md new file mode 100644 index 0000000000..71b02c8439 --- /dev/null +++ b/docs/_docs/other-guides/configuring-application-licenses.md @@ -0,0 +1,183 @@ +--- +title: "Configuring application licenses" +layout: default +parent: Other Guides +nav_order: 58 +linkTitle: "Configuring application licenses" +date: 2026-08-04 +description: > + Gate booking of licensed layers on live seat counts from your license server +--- + +# Configuring application licenses + +### Gate booking of licensed layers on live seat counts from your license server + +--- + +This page describes how to configure live application licensing, which stops +Cuebot from booking frames that need an application license (Houdini Engine, +Katana, Maya, and so on) when the license server has no free seats. + +A [Limit](/docs/other-guides/configuring-limits/) holds a static number an +admin typed, which works for an internal throttle but not for a real license +pool: the pool is also drawn on from outside the render farm — artist +workstations, CI, other farms. A fixed cap of 100 means nothing when 60 seats +are already checked out to people. The only authority on how many seats are +free is the license server itself, so Cuebot polls it and gates booking on the +live number. + +## How it works + +1. A layer declares the licenses it needs in its environment: + + ```xml + + hengine,katana + + ``` + + That declaration is the whole switch — there is deliberately no separate + enable flag. A layer that declares nothing books exactly as before and pays + nothing. + +2. A background thread in every Cuebot polls + `scheduler.license.provider` and keeps the latest sample in memory. + +3. During booking, a frame whose layer declares licenses is only dispatched + while every pool it lists has a free seat. The budget for each pool is + `available - in-flight - headroom`, where *in-flight* counts the frames + OpenCue booked after the sample was taken (the license server has not seen + them yet) and *headroom* is seats deliberately left for interactive users. + +4. Licensing **fails closed**. A layer that asks for a license is held — not + run blind — when any of these are true: + + * no provider is configured, + * no sample has arrived yet, or the sample is older than + `scheduler.license.stale_seconds`, + * the provider does not report the requested license at all, + * the layer's declaration could not be read from the database. + + Holding licensed layers costs throughput on those layers only and recovers + by itself; over-booking a pool would fail frames at checkout on the farm. + +### Floating and host-based licenses + +* A **floating** license consumes one seat per running frame. +* A **host-based** license (`"host_based": true` in the provider response) + consumes one seat per machine, shared by every frame on that machine. Cuebot + packs work onto machines that already hold the license: when a host report + shows running frames holding host-based licenses, up to + `scheduler.license.pack_jobs_max` pending jobs needing those licenses get + the first shot at that host's idle resources, because an extra frame on a + seated machine shares its one checkout while a fresh machine burns a seat. + +## The license provider + +Cuebot does not speak any vendor's license protocol. The site provides one +endpoint that reports every license the farm cares about, either over HTTP or +as a script wrapping the vendor CLI (`sesictrl`, `rlmutil`, `lmstat`, ...): + +``` +scheduler.license.provider=http://lic-reporter:9101/licenses +scheduler.license.provider=script:/site/bin/cue_licenses.sh +``` + +Both must return the same JSON: + +```json +{"queried_at": 1690000000, + "licenses": [{"name": "hengine", "feature": "Houdini Engine", "total": 800, + "available": 794, "host_based": false, + "hosts": [{"host": "wolf1018", "count": 1}]}]} +``` + +* `queried_at` (epoch seconds, when the numbers were true) is **required**. A + response without it is rejected and the previous sample keeps aging toward + stale, because a provider re-serving a cached payload would otherwise look + fresh forever. +* `available` is server truth and must already net out every consumer, + including OpenCue itself. +* `hosts` is optional. When present it enables seat counting for host-based + licenses and reveals render nodes dual-used as workstations. + +The provider string is displayed in the CueGUI **Licenses** view. Credentials +embedded in an `http(s)` URL (userinfo or query values) are redacted before +display, but a `script:` command line is shown as configured — keep secrets +out of it (read them from a file or the environment inside the script +instead). + +## Configuration reference + +All settings live in `opencue.properties` under `scheduler.license.*`: + +| Property | Default | Description | +|----------|---------|-------------| +| `scheduler.license.provider` | (empty) | `http(s):` URL or `script:` reporting license JSON. Empty disables polling; layers declaring licenses are then held. | +| `scheduler.license.poll_seconds` | `20` | Poll cadence. Every Cuebot polls, not just a leader, so a standby promoted by failover already holds a fresh sample. | +| `scheduler.license.timeout_seconds` | `10` | Hard deadline on one provider call; a hung vendor CLI is killed on it. | +| `scheduler.license.stale_seconds` | `300` | Age at which a sample stops being actionable and licensed layers are held. | +| `scheduler.license.inflight_pad_seconds` | `5` | Extra seconds added to the in-flight window, absorbing providers that stamp `queried_at` when collection finishes rather than starts. | +| `scheduler.license.headroom.default` | `0` | Seats withheld from the farm for every license without its own headroom. | +| `scheduler.license.headroom.` | (default) | Per-license headroom, e.g. `scheduler.license.headroom.hengine=5`. | +| `scheduler.license.env_key` | `CUE_LICENSES` | Layer environment key that carries the license names. | +| `scheduler.license.denied_exit_statuses` | (empty) | Vendor exit codes meaning "could not get a license", e.g. `11,203`. | +| `scheduler.license.denied_requeue_limit` | `10` | How many times one frame may take the free license-denied requeue before ordinary retry accounting resumes. | +| `scheduler.license.pack_jobs_max` | `5` | Pending jobs offered a host already holding a host-based license, per host report. `0` disables packing. | + +### License-denied exit statuses + +Even with the gate, an artist can grab the last seat between the sample and +the real checkout on the render node. When the application then exits with a +status listed in `scheduler.license.denied_exit_statuses`, the frame is +requeued WAITING without spending a retry: the seat was a contended resource, +not a broken frame. `scheduler.license.denied_requeue_limit` bounds that free +requeue per frame, so a frame that is denied over and over (wrong feature +name, broken license setup on the application side) falls back to ordinary +retry accounting instead of bouncing forever. + +## Deployment notes + +* Migration `V46` adds the index `i_layer_env_str_key` on + `layer_env (str_key, pk_layer)`, which every licensing query drives from. A + plain `CREATE INDEX` briefly write-locks `layer_env`; on large installs + pre-create it with `CREATE INDEX CONCURRENTLY` before upgrading, or migrate + in a quiet window. +* No new services or leaders: every Cuebot polls the provider independently, + and the in-flight correction is derived from the shared database, so every + Cuebot computes the same budgets. + +## Monitoring + +* In CueGUI, load the read-only **Licenses** view from + **Views/Plugins->Cuecommander** to see every license in the current sample, + its seat counts, headroom and OpenCue's own usage, plus the poller status. + See [Monitoring application licenses](/docs/user-guides/monitoring-licenses/). +* From Python: + + ```python + import opencue + status = opencue.api.getLicensingStatus() + print(status.configured(), status.stale(), status.ageSeconds()) + for lic in status.licenses(): + print(lic.name(), lic.available(), "of", lic.total()) + ``` + +* Cuebot logs one throttled warning per minute while licensed layers are held + (`LicenseSource: ... holding licensed layers`), and a loud one when layers + declare licenses with no provider configured. + +## Troubleshooting + +| Symptom | Likely cause | +|---------|--------------| +| Licensed layers never book, everything else books | No provider configured, or the provider is down and the sample went stale — check the **Licenses** view status line. | +| A single layer never books | Its `CUE_LICENSES` names a license the provider does not report; the Licenses view shows exactly which pools exist. | +| Frames fail on the farm with a license error | Add the vendor's exit code to `scheduler.license.denied_exit_statuses` and consider raising that license's headroom. | +| Fewer frames book than seats appear free | Headroom plus the in-flight correction; see the Licenses view's Headroom and Cue Frames columns. | + +## What's next? + +* [Monitoring application licenses](/docs/user-guides/monitoring-licenses/) +* [Configuring limits](/docs/other-guides/configuring-limits/) diff --git a/docs/_docs/user-guides/cuecommander-administration-guide.md b/docs/_docs/user-guides/cuecommander-administration-guide.md index ecfa808522..216a08f421 100644 --- a/docs/_docs/user-guides/cuecommander-administration-guide.md +++ b/docs/_docs/user-guides/cuecommander-administration-guide.md @@ -168,6 +168,17 @@ Configures and monitors system-wide limits that restrict how many frames can run - **Database Connections**: Control simultaneous database queries - **Network Bandwidth**: Manage render output bandwidth usage +#### Related: live application licenses + +A limit caps concurrency at a number an admin typed, which cannot see the +seats consumed outside the farm (artist workstations, CI, other farms). For +real license pools, Cuebot can instead poll the license server and gate +booking on live seat counts. The read-only **Licenses** view +(**View/Plugins** > **Cuecommander** > **Licenses**) shows the polled pools, +their usage and headroom, and the poller's health. See +[Configuring application licenses](/docs/other-guides/configuring-application-licenses/) +and [Monitoring application licenses](/docs/user-guides/monitoring-licenses/). + --- ### 3. Monitor Cue diff --git a/docs/_docs/user-guides/monitoring-licenses.md b/docs/_docs/user-guides/monitoring-licenses.md new file mode 100644 index 0000000000..4d2e6810bf --- /dev/null +++ b/docs/_docs/user-guides/monitoring-licenses.md @@ -0,0 +1,79 @@ +--- +title: "Monitoring application licenses" +nav_order: 45 +parent: User Guides +layout: default +linkTitle: "Monitoring application licenses" +date: 2026-08-04 +description: > + Inspect live application license pools and OpenCue's usage of them +--- + +# Monitoring application licenses + +### Inspect live application license pools and OpenCue's usage of them + +--- + +This guide describes the **Licenses** view in CueGUI, a read-only window into +the application licenses Cuebot polls from its license provider. Use it to +answer "why is my licensed layer not booking?" and "how much of each pool is +the farm holding?" without shelling into Cuebot. + +The view is read-only by design: the seat counts belong to the license server, +and the tuning (provider endpoint, headroom, poll cadence) lives in Cuebot's +`opencue.properties`. See +[Configuring application licenses](/docs/other-guides/configuring-application-licenses/) +for the setup. + +## Opening the view + +1. Open CueGUI. + +1. Load the **Licenses** view from the **Views/Plugins->Cuecommander** menu. + +## The status line + +The line above the table reports the poller itself: + +* **Provider ... — sample Ns old (stale after Ms), poll every Ps** — healthy. + The sample age includes how old the provider said its numbers were when it + answered. +* **STALE — licensed layers are held. ...** — the sample aged past + `scheduler.license.stale_seconds` (provider down or unreachable). Booking of + licensed layers has stopped until a fresh sample lands. +* **Waiting for the first sample from ...** — Cuebot restarted recently or the + provider has not answered yet. +* **License provider not configured ...** — `scheduler.license.provider` is + unset on Cuebot; any layer declaring licenses is held. + +## Columns + +| Column | Meaning | +|--------|---------| +| License | Pool name as layers declare it in `CUE_LICENSES`, e.g. `hengine`. | +| Feature | Human-readable name from the provider, e.g. `Houdini Engine`. | +| Type | `Floating` (one seat per frame) or `Host-based` (one seat per machine, shared by all frames on it). | +| Total | Seats the license server owns. | +| Available | Seats the server reports free, net of every consumer — workstations, CI and other farms included. | +| In Use | Seats checked out anywhere (Total minus Available). | +| Headroom | Seats withheld from the farm for interactive users (`scheduler.license.headroom.`). | +| Cue Frames | Frames currently running on this OpenCue deploy whose layer declares the license. | +| Cue Hosts | Distinct hosts running those frames. | +| Provider Hosts | Hosts the license server reports holding a seat (any consumer); 0 when the provider does not report hosts. | + +The table refreshes every 60 seconds; **Refresh** forces it. + +## Reading the numbers + +* A licensed layer books only while `Available - Headroom` leaves room for it + (Cuebot also subtracts the frames it booked since the sample was taken). +* For a host-based license, an extra frame on a host that already holds the + license is free; only fresh machines consume seats. +* A license your layer declares that does **not** appear in the table is a + held layer: the provider does not report that pool, so Cuebot refuses to + book it blind. Fix the layer's `CUE_LICENSES` or the provider's report. + +## What's next? + +* [Configuring application licenses](/docs/other-guides/configuring-application-licenses/) diff --git a/proto/src/license.proto b/proto/src/license.proto new file mode 100644 index 0000000000..c77510a836 --- /dev/null +++ b/proto/src/license.proto @@ -0,0 +1,94 @@ + +syntax = "proto3"; +package license; + +option java_package = "com.imageworks.spcue.grpc.license"; +option java_multiple_files = true; + +option go_package = "opencue_gateway/gen/go"; + +// Read-only view of the live application licenses (Houdini Engine, Katana, ...) +// Cuebot polls from scheduler.license.provider. Unlike a Limit, none of this is +// editable over the wire: the numbers come from the license server and the +// tuning (provider, headroom, poll cadence) lives in opencue.properties. + +// -------- Services --------] + +service LicenseInterface { + // Get the license source status and every license in the current sample. + rpc GetAll(LicenseGetAllRequest) returns (LicenseGetAllResponse); + // Find one license in the current sample by (lowercased) name. + rpc Find(LicenseFindRequest) returns (LicenseFindResponse); +} + +// -------- Primary Message Types --------] + +// One application license as of the last provider sample, with this +// cluster's own usage alongside the license server's numbers. +message License { + // Pool name as layers declare it (lowercased), e.g. "hengine". + string name = 1; + // Human-readable feature name from the provider, e.g. "Houdini Engine". + string feature = 2; + // Total seats the license server owns. + int32 total = 3; + // Seats the license server reports free, net of every consumer + // (workstations, CI and other farms included). + int32 available = 4; + // True when one seat covers a whole machine (every frame on that host + // shares it); false when every running frame consumes a seat. + bool host_based = 5; + // Seats deliberately withheld from the farm for interactive users + // (scheduler.license.headroom.). + int32 headroom = 6; + // Frames currently RUNNING on this OpenCue deploy that declare this + // license, from the database at request time. + int32 running_frames = 7; + // Distinct hosts those running frames occupy. + int32 running_hosts = 8; + // Hosts the provider reports holding a seat (any consumer, not just + // OpenCue); 0 when the provider does not report hosts. + int32 provider_host_count = 9; +} + +// State of the poller feeding the sample, so a GUI can say how trustworthy +// the numbers are. +message LicenseSourceStatus { + // False when scheduler.license.provider is unset; licensed layers are + // held and the license list is empty. + bool configured = 1; + // The configured provider, e.g. "http://lic-reporter:9101/licenses" or + // "script:/site/bin/cue_licenses.sh". + string provider = 2; + // Layer environment key that binds a layer to licenses (CUE_LICENSES). + string env_key = 3; + // Poll cadence in seconds. + int32 poll_seconds = 4; + // Age past which the sample stops being actionable and booking fails + // closed. + int32 stale_seconds = 5; + // False until the first successful poll lands. + bool has_sample = 6; + // Age of the current sample in seconds, including how old the provider + // said its numbers were on arrival. 0 when there is no sample. + int64 age_seconds = 7; + // True when there is no sample or it aged past stale_seconds; licensed + // layers are being held. + bool stale = 8; +} + +// -------- Requests & Responses --------] + +// GetAll +message LicenseGetAllRequest {} // Empty +message LicenseGetAllResponse { + LicenseSourceStatus source = 1; + repeated License licenses = 2; +} +// Find +message LicenseFindRequest { + string name = 1; +} +message LicenseFindResponse { + License license = 1; +} diff --git a/pycue/opencue/api.py b/pycue/opencue/api.py index b0f769310c..909335d94f 100644 --- a/pycue/opencue/api.py +++ b/pycue/opencue/api.py @@ -27,6 +27,7 @@ from opencue_proto import filter_pb2 from opencue_proto import host_pb2 from opencue_proto import job_pb2 +from opencue_proto import license_pb2 from opencue_proto import limit_pb2 from opencue_proto import renderPartition_pb2 from opencue_proto import report_pb2 @@ -48,6 +49,7 @@ from .wrappers.host import Host, NestedHost from .wrappers.job import Job from .wrappers.layer import Layer +from .wrappers.license import License, LicensingStatus from .wrappers.limit import Limit from .wrappers.owner import Owner from .wrappers.proc import Proc @@ -830,3 +832,47 @@ def findLimit(name): :return: the matching Limit object""" return Limit(Cuebot.getStub('limit').Find( limit_pb2.LimitFindRequest(name=name), timeout=Cuebot.Timeout).limit) + +# +# Licenses +# +@util.grpcExceptionParser +def getLicenses(): + """Return all application licenses in Cuebot's current provider sample. + + Read-only: the numbers come from the license server Cuebot polls + (``scheduler.license.provider``). Empty when no provider is configured + or no sample has landed yet; use :func:`getLicensingStatus` to tell + those cases apart. + + :rtype: list[opencue.wrappers.license.License] + :return: a list of License objects, sorted by name""" + return [License(license) for license in Cuebot.getStub('license').GetAll( + license_pb2.LicenseGetAllRequest(), timeout=Cuebot.Timeout).licenses] + + +@util.grpcExceptionParser +def getLicensingStatus(): + """Return the license poller status plus the cached licenses. + + One round trip: the status says whether a provider is configured and + how fresh the sample is, and carries the same licenses + :func:`getLicenses` returns. + + :rtype: opencue.wrappers.license.LicensingStatus + :return: the licensing status""" + return LicensingStatus(Cuebot.getStub('license').GetAll( + license_pb2.LicenseGetAllRequest(), timeout=Cuebot.Timeout)) + + +@util.grpcExceptionParser +def findLicense(name): + """Returns the License in the current provider sample that matches the name. + + :type name: str + :param name: a license pool name, e.g. ``hengine``; matching is + case-insensitive + :rtype: opencue.wrappers.license.License + :return: the matching License object""" + return License(Cuebot.getStub('license').Find( + license_pb2.LicenseFindRequest(name=name), timeout=Cuebot.Timeout).license) diff --git a/pycue/opencue/cuebot.py b/pycue/opencue/cuebot.py index 4de529e6f3..321852d40e 100644 --- a/pycue/opencue/cuebot.py +++ b/pycue/opencue/cuebot.py @@ -46,6 +46,8 @@ from opencue_proto import host_pb2_grpc from opencue_proto import job_pb2 from opencue_proto import job_pb2_grpc +from opencue_proto import license_pb2 +from opencue_proto import license_pb2_grpc from opencue_proto import limit_pb2 from opencue_proto import limit_pb2_grpc from opencue_proto import renderPartition_pb2 @@ -107,6 +109,7 @@ class Cuebot(object): 'host': host_pb2, 'job': job_pb2, 'layer': job_pb2, + 'license': license_pb2, 'limit': limit_pb2, 'matcher': filter_pb2, 'monitoring': monitoring_pb2, @@ -133,6 +136,7 @@ class Cuebot(object): 'host': host_pb2_grpc.HostInterfaceStub, 'job': job_pb2_grpc.JobInterfaceStub, 'layer': job_pb2_grpc.LayerInterfaceStub, + 'license': license_pb2_grpc.LicenseInterfaceStub, 'limit': limit_pb2_grpc.LimitInterfaceStub, 'matcher': filter_pb2_grpc.MatcherInterfaceStub, 'monitoring': monitoring_pb2_grpc.MonitoringInterfaceStub, diff --git a/pycue/opencue/wrappers/license.py b/pycue/opencue/wrappers/license.py new file mode 100644 index 0000000000..e0319372ba --- /dev/null +++ b/pycue/opencue/wrappers/license.py @@ -0,0 +1,261 @@ +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for classes related to application licenses. + +Licenses are read-only over the wire: the numbers come from the license +server Cuebot polls (``scheduler.license.provider``), and the tuning lives +in ``opencue.properties`` on the Cuebot side. +""" + +from opencue_proto import license_pb2 +from opencue import Cuebot + + +class License(object): + """Read-only view of one application license in Cuebot's provider sample.""" + + def __init__(self, license_=None): + self.data = license_ + self.stub = Cuebot.getStub('license') + + def find(self, name): + """Finds a license in the current sample by its name. + + :type name: str + :param name: name of license to find + :rtype: opencue.wrappers.license.License + :return: the license found by name + """ + return License( + self.stub.Find( + license_pb2.LicenseFindRequest(name=name), timeout=Cuebot.Timeout).license) + + def id(self): + """Returns the license name, which is its identifier. + + Licenses have no database id; the pool name is unique within the + provider sample. + + :rtype: str + :return: the license name + """ + return self.name() + + def name(self): + """Returns the license pool name, e.g. ``hengine``. + + :rtype: str + :return: the license name + """ + if hasattr(self.data, 'name'): + return self.data.name + return "" + + def feature(self): + """Returns the human-readable feature name, e.g. ``Houdini Engine``. + + :rtype: str + :return: the feature name + """ + if hasattr(self.data, 'feature'): + return self.data.feature + return "" + + def total(self): + """Returns the total seats the license server owns. + + :rtype: int + :return: total seat count + """ + if hasattr(self.data, 'total'): + return self.data.total + return -1 + + def available(self): + """Returns the seats the license server reports free, net of every + consumer (workstations, CI and other farms included). + + :rtype: int + :return: available seat count + """ + if hasattr(self.data, 'available'): + return self.data.available + return -1 + + def inUse(self): + """Returns the seats currently checked out anywhere, per the server. + + :rtype: int + :return: total minus available + """ + if hasattr(self.data, 'total'): + return self.data.total - self.data.available + return -1 + + def hostBased(self): + """Returns whether one seat covers a whole machine rather than one frame. + + :rtype: bool + :return: True when the license is host-based + """ + if hasattr(self.data, 'host_based'): + return self.data.host_based + return False + + def headroom(self): + """Returns the seats deliberately withheld from the farm for + interactive users (``scheduler.license.headroom.``). + + :rtype: int + :return: headroom seat count + """ + if hasattr(self.data, 'headroom'): + return self.data.headroom + return -1 + + def runningFrames(self): + """Returns the frames currently running on this OpenCue deploy whose + layer declares this license. + + :rtype: int + :return: running licensed frame count + """ + if hasattr(self.data, 'running_frames'): + return self.data.running_frames + return -1 + + def runningHosts(self): + """Returns the distinct hosts running those frames. + + :rtype: int + :return: running licensed host count + """ + if hasattr(self.data, 'running_hosts'): + return self.data.running_hosts + return -1 + + def providerHostCount(self): + """Returns the hosts the provider reports holding a seat (any + consumer, not just OpenCue); 0 when the provider does not report hosts. + + :rtype: int + :return: provider-reported host count + """ + if hasattr(self.data, 'provider_host_count'): + return self.data.provider_host_count + return -1 + + +class LicensingStatus(object): + """Read-only state of the Cuebot license poller plus the cached licenses. + + Wraps one ``LicenseGetAllResponse``: the poller/source status and every + license in the current sample, fetched in a single round trip. + """ + + def __init__(self, response=None): + self.data = response.source if response is not None else None + self.__licenses = [ + License(lic) for lic in response.licenses] if response is not None else [] + + def licenses(self): + """Returns the licenses in the current sample. + + :rtype: list[opencue.wrappers.license.License] + :return: licenses, sorted by name + """ + return self.__licenses + + def configured(self): + """Returns whether ``scheduler.license.provider`` is set on Cuebot. + + :rtype: bool + :return: True when a provider is configured + """ + if hasattr(self.data, 'configured'): + return self.data.configured + return False + + def provider(self): + """Returns the configured provider endpoint or script. + + :rtype: str + :return: the provider string, empty when unconfigured + """ + if hasattr(self.data, 'provider'): + return self.data.provider + return "" + + def envKey(self): + """Returns the layer environment key that binds layers to licenses. + + :rtype: str + :return: the env key, e.g. ``CUE_LICENSES`` + """ + if hasattr(self.data, 'env_key'): + return self.data.env_key + return "" + + def pollSeconds(self): + """Returns the provider poll cadence in seconds. + + :rtype: int + :return: poll interval + """ + if hasattr(self.data, 'poll_seconds'): + return self.data.poll_seconds + return -1 + + def staleSeconds(self): + """Returns the age past which the sample stops being actionable. + + :rtype: int + :return: staleness threshold in seconds + """ + if hasattr(self.data, 'stale_seconds'): + return self.data.stale_seconds + return -1 + + def hasSample(self): + """Returns whether a first successful poll has landed. + + :rtype: bool + :return: True once the poller holds a sample + """ + if hasattr(self.data, 'has_sample'): + return self.data.has_sample + return False + + def ageSeconds(self): + """Returns the age of the current sample in seconds, including the + provider's own lag; 0 when there is no sample. + + :rtype: int + :return: sample age + """ + if hasattr(self.data, 'age_seconds'): + return self.data.age_seconds + return -1 + + def stale(self): + """Returns whether the sample is missing or too old to act on; + licensed layers are being held while this is True. + + :rtype: bool + :return: True when stale + """ + if hasattr(self.data, 'stale'): + return self.data.stale + return True diff --git a/pycue/tests/wrappers/test_license.py b/pycue/tests/wrappers/test_license.py new file mode 100644 index 0000000000..6d5c77371f --- /dev/null +++ b/pycue/tests/wrappers/test_license.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python + +# Copyright Contributors to the OpenCue Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for `opencue.wrappers.license`.""" + +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +import unittest + +import mock + +from opencue_proto import license_pb2 +import opencue.api +import opencue.wrappers.license + + +TEST_LICENSE_NAME = 'hengine' +TEST_LICENSE_FEATURE = 'Houdini Engine' + + +def _license(): + return license_pb2.License( + name=TEST_LICENSE_NAME, + feature=TEST_LICENSE_FEATURE, + total=800, + available=794, + host_based=True, + headroom=5, + running_frames=12, + running_hosts=3, + provider_host_count=2) + + +@mock.patch('opencue.cuebot.Cuebot.getStub') +class LicenseTests(unittest.TestCase): + + def testFind(self, getStubMock): + stubMock = mock.Mock() + stubMock.Find.return_value = license_pb2.LicenseFindResponse(license=_license()) + getStubMock.return_value = stubMock + + lic = opencue.wrappers.license.License().find(TEST_LICENSE_NAME) + + stubMock.Find.assert_called_with( + license_pb2.LicenseFindRequest(name=TEST_LICENSE_NAME), timeout=mock.ANY) + self.assertEqual(lic.name(), TEST_LICENSE_NAME) + + def testAccessors(self, getStubMock): + getStubMock.return_value = mock.Mock() + + lic = opencue.wrappers.license.License(_license()) + + self.assertEqual(lic.id(), TEST_LICENSE_NAME) + self.assertEqual(lic.name(), TEST_LICENSE_NAME) + self.assertEqual(lic.feature(), TEST_LICENSE_FEATURE) + self.assertEqual(lic.total(), 800) + self.assertEqual(lic.available(), 794) + self.assertEqual(lic.inUse(), 6) + self.assertTrue(lic.hostBased()) + self.assertEqual(lic.headroom(), 5) + self.assertEqual(lic.runningFrames(), 12) + self.assertEqual(lic.runningHosts(), 3) + self.assertEqual(lic.providerHostCount(), 2) + + +@mock.patch('opencue.cuebot.Cuebot.getStub') +class LicensingStatusTests(unittest.TestCase): + + def testStatusAndLicenses(self, getStubMock): + getStubMock.return_value = mock.Mock() + + response = license_pb2.LicenseGetAllResponse( + source=license_pb2.LicenseSourceStatus( + configured=True, + provider='http://lic-reporter:9101/licenses', + env_key='CUE_LICENSES', + poll_seconds=20, + stale_seconds=300, + has_sample=True, + age_seconds=17, + stale=False), + licenses=[_license()]) + status = opencue.wrappers.license.LicensingStatus(response) + + self.assertTrue(status.configured()) + self.assertEqual(status.provider(), 'http://lic-reporter:9101/licenses') + self.assertEqual(status.envKey(), 'CUE_LICENSES') + self.assertEqual(status.pollSeconds(), 20) + self.assertEqual(status.staleSeconds(), 300) + self.assertTrue(status.hasSample()) + self.assertEqual(status.ageSeconds(), 17) + self.assertFalse(status.stale()) + self.assertEqual(len(status.licenses()), 1) + self.assertEqual(status.licenses()[0].name(), TEST_LICENSE_NAME) + + def testEmptyStatusFailsClosed(self, getStubMock): + getStubMock.return_value = mock.Mock() + + status = opencue.wrappers.license.LicensingStatus() + + self.assertFalse(status.configured()) + self.assertFalse(status.hasSample()) + self.assertTrue(status.stale()) + self.assertEqual(status.licenses(), []) + + +@mock.patch('opencue.cuebot.Cuebot.getStub') +class ApiTests(unittest.TestCase): + + def testGetLicenses(self, getStubMock): + stubMock = mock.Mock() + stubMock.GetAll.return_value = license_pb2.LicenseGetAllResponse( + source=license_pb2.LicenseSourceStatus(configured=True), + licenses=[_license()]) + getStubMock.return_value = stubMock + + licenses = opencue.api.getLicenses() + + stubMock.GetAll.assert_called_with( + license_pb2.LicenseGetAllRequest(), timeout=mock.ANY) + self.assertEqual(len(licenses), 1) + self.assertEqual(licenses[0].name(), TEST_LICENSE_NAME) + + def testGetLicensingStatus(self, getStubMock): + stubMock = mock.Mock() + stubMock.GetAll.return_value = license_pb2.LicenseGetAllResponse( + source=license_pb2.LicenseSourceStatus(configured=False, stale=True)) + getStubMock.return_value = stubMock + + status = opencue.api.getLicensingStatus() + + self.assertFalse(status.configured()) + self.assertTrue(status.stale()) + self.assertEqual(status.licenses(), []) + + def testFindLicense(self, getStubMock): + stubMock = mock.Mock() + stubMock.Find.return_value = license_pb2.LicenseFindResponse(license=_license()) + getStubMock.return_value = stubMock + + lic = opencue.api.findLicense(TEST_LICENSE_NAME) + + stubMock.Find.assert_called_with( + license_pb2.LicenseFindRequest(name=TEST_LICENSE_NAME), timeout=mock.ANY) + self.assertEqual(lic.feature(), TEST_LICENSE_FEATURE) + + +if __name__ == '__main__': + unittest.main()