diff --git a/.github/runners/controller/Modules/FleetCore/FleetCore.psm1 b/.github/runners/controller/Modules/FleetCore/FleetCore.psm1 index 011e47d9310..a59553cf8ee 100644 --- a/.github/runners/controller/Modules/FleetCore/FleetCore.psm1 +++ b/.github/runners/controller/Modules/FleetCore/FleetCore.psm1 @@ -384,8 +384,14 @@ function Invoke-ArmList { Operation = $Operation } $page = Invoke-ArmJson @splatPage - if (-not $page) { - break + if ($null -eq $page -or $page.value -isnot [array]) { + # A list page without a value array is a garbled read, not an empty + # result -- ARM always ships value as an array, empty or not, so a + # missing value and a value of any other shape are the same garble. + # Emitting nothing here would flatten it into a clean empty inventory + # downstream, which is exactly the shape a destructive scale decision + # trusts. + throw (New-TransientFleetException -Message "$Operation returned a page without a value array; failing the pass rather than treating a garbled read as an empty list") } $page.value $next = [string]$page.nextLink @@ -527,6 +533,13 @@ function Get-FleetState { Operation = "list GitHub runners" } $runnerResponse = Invoke-GhJson @splatRunners + if ($runnerResponse.runners -isnot [array]) { + # The runners endpoint always ships a runners array, even when it is empty, + # so a response without one -- or with one of any other shape -- is a + # garbled read. Flattening it into an empty list would erase every runner + # from the fleet's view in a single pass. + throw (New-TransientFleetException -Message "list GitHub runners returned no runners array; failing the pass rather than treating a garbled read as an empty fleet") + } $runners = @($runnerResponse.runners | Where-Object { $PSItem.labels.name -contains $script:Fleet.RunnerLabel }) # One list call, not the CLI's --show-details fan-out: the projection below is # everything the fleet logic reads, and powerState was only ever projected, never @@ -1306,7 +1319,39 @@ function Invoke-FleetReconcile { } } + $vmssPath = "/subscriptions/$($script:Fleet.SubscriptionId)/resourceGroups/$($script:Fleet.ResourceGroup)/providers/Microsoft.Compute/virtualMachineScaleSets/$($script:Fleet.Vmss)" + $splatCapacity = @{ + Path = "$vmssPath`?api-version=2024-07-01" + Operation = "read VMSS capacity" + } + $capacityResponse = Invoke-ArmJson @splatCapacity + $capacity = 0 + if (-not [int]::TryParse([string]$capacityResponse.sku.capacity, [ref]$capacity)) { + # A missing sku block coerced through [int] reads as capacity 0, and a + # falsely-zero nominal turns the compensated scale-out into a down-PATCH + # from Azure's real figure -- the delete-live-instances mutation this + # controller exists to avoid. No number is safer than a wrong one. + # TryParse also refuses non-integral garble and digit strings past + # Int32.MaxValue: casting either through [int] would throw past the + # TransientFleetException catch and crash the invocation instead of + # skipping the pass. A parsed negative rides through to the policy, + # whose negative-telemetry guard skips the pass -- out-of-domain + # numbers are its call, unparseable ones are refused here. + throw (New-TransientFleetException -Message "read VMSS capacity returned no usable sku.capacity; skipping the pass rather than PATCHing from a guessed nominal") + } + $provisioningState = [string]$capacityResponse.properties.provisioningState + # The inventory list has to come after the provisioning-state read: a settled + # state proves any prior scale-out already finished, so a list taken now cannot + # be missing just-created members. Listed first, a stale-low count could send + # the normalization PATCH below the real membership and delete live instances. $state = Get-FleetState + if ($null -eq $state.Vms -or $null -eq $state.Runners) { + # A null inventory is a garbled read, not an empty fleet: @($null).Count + # is 1, which would masquerade as one live VM, and a null runner list + # would sail through the reclaim corroboration below as zero online. + # Neither figure may price a capacity step. + throw (New-TransientFleetException -Message "fleet inventory read returned no VM or runner list; skipping the pass rather than pricing a step from a guessed inventory") + } $transitionBusy = @($state.Vms | Where-Object { $runner = Get-RunnerForVm -State $state -VmName $PSItem.name $pool = Get-VmPool -Vm $PSItem @@ -1314,23 +1359,16 @@ function Invoke-FleetReconcile { $outsideDesiredPool -and $runner -and $runner.busy }).Count $target = [math]::Min($script:Fleet.MaxRunners, $desiredTotal + $transitionBusy) - $vmssPath = "/subscriptions/$($script:Fleet.SubscriptionId)/resourceGroups/$($script:Fleet.ResourceGroup)/providers/Microsoft.Compute/virtualMachineScaleSets/$($script:Fleet.Vmss)" - $splatCapacity = @{ - Path = "$vmssPath`?api-version=2024-07-01" - Operation = "read VMSS capacity" - } - $capacityResponse = Invoke-ArmJson @splatCapacity - $capacity = [int]$capacityResponse.sku.capacity - $provisioningState = [string]$capacityResponse.properties.provisioningState $actualCapacity = @($state.Vms).Count Write-Host "capacity=$capacity actual_capacity=$actualCapacity target=$target transition_busy=$transitionBusy provisioning_state=$provisioningState" # On Flexible orchestration, deleting spent VMs one at a time leaves sku.capacity # above the number of instances that really exist, and a PATCH computed from the # nominal figure alone creates target-minus-nominal VMs instead of # target-minus-actual; that gap held a ten-runner lane at six VMs (2026-08-08). - # Get-FleetCapacityStep walks capacity down to reality before raising it to the - # target, one settled pass at a time, so an in-flight scale-out is never - # mistaken for phantom capacity and dependent PATCHes never overlap. + # Get-FleetCapacityStep emits at most one mutation per settled pass -- a scale-out + # compensated for the drift, or the reclaim to zero once the fleet stands empty -- + # so an in-flight scale-out is never mistaken for phantom capacity, dependent + # PATCHes never overlap, and churn-minted drift can never starve creation. $splatCapacityStep = @{ ProvisioningState = $provisioningState NominalCapacity = $capacity @@ -1339,7 +1377,34 @@ function Invoke-FleetReconcile { } $newCapacity = Get-FleetCapacityStep @splatCapacityStep if ($null -ne $newCapacity) { - if (-not (Test-FleetDryRun -Decision "scale vmss=$($script:Fleet.Vmss) from=$capacity to=$newCapacity")) { + $reclaimBlocked = $null + if ($newCapacity -lt $capacity) { + # The only down-step the policy emits is the reclaim to zero, and it + # hangs entirely on an empty ARM list, which a single read can still + # fake: the shape guards in Get-FleetState make a garbled payload + # throw, but list endpoints are eventually consistent, so a + # well-shaped stale page can report empty while members exist. So + # emptiness needs two independent witnesses before capacity may + # cross below nominal -- + # GitHub first, because a runner cannot be online without a live VM + # behind it, then a second inventory read that must come back empty + # again. The extra ARM call is paid only on this rare empty-fleet + # path, never on the hot scale-out path. + $onlineRunners = @($state.Runners | Where-Object status -EQ "online").Count + if ($onlineRunners -gt 0) { + $reclaimBlocked = "$onlineRunners runner(s) are online" + } else { + $confirmState = Get-FleetState + $confirmVms = @($confirmState.Vms).Count + $confirmOnline = @($confirmState.Runners | Where-Object status -EQ "online").Count + if ($confirmVms -gt 0 -or $confirmOnline -gt 0) { + $reclaimBlocked = "a confirming re-read found $confirmVms VM(s) and $confirmOnline online runner(s)" + } + } + } + if ($reclaimBlocked) { + Write-Warning "Skipping capacity reclaim to ${newCapacity}: $reclaimBlocked, so the empty inventory is not trusted." + } elseif (-not (Test-FleetDryRun -Decision "scale vmss=$($script:Fleet.Vmss) from=$capacity to=$newCapacity")) { # Fire and forget, matching the CLI's --no-wait. Deliberately no in-line # readiness poll: the queue is serialized, so a pass that sleeps on # provisioning holds up every queued nudge behind it, and a burst of runs diff --git a/.github/runners/runner-policy.ps1 b/.github/runners/runner-policy.ps1 index bdd71f9dac4..e3fa7ecb733 100644 --- a/.github/runners/runner-policy.ps1 +++ b/.github/runners/runner-policy.ps1 @@ -247,34 +247,76 @@ function Get-FleetCapacityStep { [CmdletBinding()] param( [string]$ProvisioningState, - [ValidateRange(0, 35)] [int]$NominalCapacity, - [ValidateRange(0, 35)] [int]$ActualCapacity, [ValidateRange(0, 35)] [int]$TargetCapacity ) # The Function controller PATCHes capacity fire-and-forget, so unlike the CLI script - # it cannot await one mutation before issuing the next. Serialization comes from the - # pass structure instead. While the scale set is mid-mutation, a nominal-over-actual - # gap is Azure still working, not phantom capacity -- normalizing it away would - # cancel the instances being created -- so an unsettled pass emits nothing and a - # later pass converges. A settled pass takes only the first step of the plan, so a - # normalization and the scale-out it unblocks land on successive passes, each - # computed from a settled read. Failed still mutates: a capacity PATCH is how a - # stuck scale set recovers, and skipping it would freeze the fleet. - if ($ProvisioningState -in @("Creating", "Updating", "Deleting", "Migrating")) { + # it cannot run Get-VmssCapacityPlan's normalize-then-scale sequence: awaiting the + # normalization would reintroduce the in-line wait that put the queue 3.3 hours + # behind, and taking one plan step per pass starves scale-out entirely, because + # ephemeral deletes mint fresh phantom capacity between passes and normalization + # wins the slot every time -- the fleet drained from 8 VMs to 3 against a target of + # 20 that way (2026-08-08). So the scale-out step compensates for the drift instead + # of repairing it first: raising nominal by exactly the shortfall makes Azure create + # target-minus-actual instances no matter how stale the bookkeeping is, in a single + # mutation. Normalization above zero members is gone entirely: production proved + # (2026-08-08, six consecutive observations during the drain, then ten busy runners + # killed mid-job in one CI run) that a down-PATCH deletes nominal-minus-newValue + # LIVE instances, and that every capacity PATCH conserves the nominal-over-actual + # gap, so normalizing can never even catch the drift it chases. The gap grows on + # per-VM deletes and clears only at the zero-crossing, where a down-PATCH has no + # members left to take -- that reclaim to zero is the one down-step this function + # emits. A drifted nominal anywhere above it, runaway or pinned at the ceiling or + # demand-met surplus, stays untouched and costs at most gap-many slots of ceiling + # headroom until the fleet next empties on its own. While the scale set is + # mid-mutation a nominal-over-actual gap is Azure still working, not phantom + # capacity -- normalizing it away would cancel the instances being created -- so + # only the ARM terminal states may mutate: an unknown or missing state is + # indistinguishable from an operation in flight, and the cost of skipping a pass is + # one safety-tick delay while the cost of overlapping PATCHes is cancelled + # instances. Failed and Canceled still mutate: a capacity PATCH is how a stuck + # scale set recovers, and skipping them would freeze the fleet. + if ($ProvisioningState -notin @("Succeeded", "Failed", "Canceled")) { + return $null + } + if ($NominalCapacity -lt 0 -or $ActualCapacity -lt 0) { + # Negative telemetry is a garbled ARM read, not a real fleet state, and it must + # not leak into a PATCH body. Skipping the pass costs one safety tick and the + # next read starts clean; a ValidateRange would crash the pass instead, which + # is the exact failure mode the missing attributes above avoid. return $null } - $splatCapacity = @{ - NominalCapacity = $NominalCapacity - ActualCapacity = $ActualCapacity - TargetCapacity = $TargetCapacity + if ($ActualCapacity -lt $TargetCapacity) { + # 35 matches the ValidateRange every capacity function in this file shares. + # It is the MAX_RUNNERS hard ceiling, and in this function only Target still + # carries the gate, because the controller chooses it. Nominal and actual are + # ARM-read telemetry and deliberately carry no range gate: the janitor runbook + # treats capacity above the ceiling as a real state, and validating it here + # would crash every pass that observes it -- a ParameterBindingException is not + # TransientFleetException, so nothing catches it and the controller stops + # scaling entirely. The min bounds what this function emits; a nominal already + # at or past the ceiling gets no step here and unwinds through the + # zero-crossing reclaim below. + $unclipped = $NominalCapacity + ($TargetCapacity - $ActualCapacity) + $compensated = [math]::Min(35, $unclipped) + if ($ActualCapacity -eq 0 -and $NominalCapacity -gt 0 -and $compensated -lt $unclipped) { + # A clipped step on an empty fleet would create fewer than target instances + # and then pin there, below target, until the drift unwinds. With zero + # members the zero-crossing reclaim is free, so repay the whole drift now + # and let the next pass create the full target from a clean nominal. A + # clipped step over a nonzero fleet has no such option -- reclaiming would + # delete the live members -- so it still takes whatever headroom remains. + return 0 + } + if ($compensated -gt $NominalCapacity) { + return $compensated + } } - $plan = @(Get-VmssCapacityPlan @splatCapacity) - if ($plan.Count -gt 0) { - return $plan[0] + if ($ActualCapacity -eq 0 -and $NominalCapacity -gt 0) { + return 0 } return $null } diff --git a/.github/runners/tests/fleet-controller.Tests.ps1 b/.github/runners/tests/fleet-controller.Tests.ps1 index 48924fa4ff3..34b6edae67a 100644 --- a/.github/runners/tests/fleet-controller.Tests.ps1 +++ b/.github/runners/tests/fleet-controller.Tests.ps1 @@ -462,6 +462,271 @@ Describe "registration readiness" { } } +Describe "capacity step ordering" { + BeforeEach { + InModuleScope FleetCore { + $script:Fleet = [pscustomobject]@{ + DryRun = $false + SubscriptionId = "sub" + ResourceGroup = "rg" + Vmss = "dbatools-runners" + MaxRunners = 35 + } + } + # These flags parameterize the mock bodies below, which execute in this test + # file's scope even though the mocks intercept calls inside FleetCore -- only + # $script:Fleet above has to live in the module, because the real reconcile + # code reads it there. + $script:CapacityReadSeen = $false + $script:CapacityReadMalformed = $false + $script:CapacityReadNonNumeric = $false + $script:CapacityReadOverflow = $false + $script:FleetStateGarbled = $false + $script:FleetListsVms = $true + $script:FleetRepopulatesOnConfirm = $false + $script:PostReadStateCalls = 0 + $script:FleetOnlineRunners = @() + $script:CapacityStepValue = $null + Mock -ModuleName FleetCore Initialize-FleetContext { } + Mock -ModuleName FleetCore Get-RunnerDemand { + @{ + Dispatch = $null + Desired = @{} + } + } + Mock -ModuleName FleetCore Get-OrphanedNetworking { $null } + Mock -ModuleName FleetCore Get-FleetState { + if ($script:FleetStateGarbled) { + return [pscustomobject]@{ + Vms = $null + Runners = $null + } + } + $vms = @() + if ($script:CapacityReadSeen) { + $script:PostReadStateCalls++ + if ($script:FleetListsVms) { + $vms = @(foreach ($vmSuffix in "a", "b", "c") { + [pscustomobject]@{ + name = "dbatools-runners_$vmSuffix" + provisioning = "Succeeded" + tags = $null + } + }) + } elseif ($script:FleetRepopulatesOnConfirm -and $script:PostReadStateCalls -ge 2) { + $vms = @( + [pscustomobject]@{ + name = "dbatools-runners_late" + provisioning = "Succeeded" + tags = $null + } + ) + } + } + [pscustomobject]@{ + Vms = $vms + Runners = $script:FleetOnlineRunners + } + } + Mock -ModuleName FleetCore Invoke-ArmJson { + $script:CapacityReadSeen = $true + if ($script:CapacityReadMalformed) { + return [pscustomobject]@{ + properties = [pscustomobject]@{ provisioningState = "Succeeded" } + } + } + if ($script:CapacityReadNonNumeric) { + return [pscustomobject]@{ + sku = [pscustomobject]@{ capacity = "garbled" } + properties = [pscustomobject]@{ provisioningState = "Succeeded" } + } + } + if ($script:CapacityReadOverflow) { + return [pscustomobject]@{ + sku = [pscustomobject]@{ capacity = "4294967296" } + properties = [pscustomobject]@{ provisioningState = "Succeeded" } + } + } + [pscustomobject]@{ + sku = [pscustomobject]@{ capacity = 9 } + properties = [pscustomobject]@{ provisioningState = "Succeeded" } + } + } + Mock -ModuleName FleetCore Get-FleetCapacityStep { $script:CapacityStepValue } + Mock -ModuleName FleetCore Invoke-ArmWeb { } + Mock -ModuleName FleetCore Set-UnallocatedVmPool { } + Mock -ModuleName FleetCore Register-PoolVms { } + Mock -ModuleName FleetCore Set-VmOnlineObservedAt { } + Mock -ModuleName FleetCore Remove-OrphanedNetworking { } + Mock -ModuleName FleetCore Set-FleetHeartbeat { } + } + + It "prices the capacity step from an inventory listed after the settled-state read" { + # A list taken before the provisioning-state read can predate a just-completed + # scale-out, and a stale-low count would send a capacity PATCH below the real + # membership. The fleet here is empty until the capacity read happens and + # holds three VMs afterwards: only the read-then-list order sees all three. + Invoke-FleetReconcile + + $freshInventoryFilter = { + $ProvisioningState -eq "Succeeded" -and + $NominalCapacity -eq 9 -and + $ActualCapacity -eq 3 -and + $TargetCapacity -eq 0 + } + Should -Invoke Get-FleetCapacityStep -ModuleName FleetCore -Times 1 -Exactly -ParameterFilter $freshInventoryFilter + } + + It "bails out of the pass when the capacity read comes back without a sku" { + # [int]$null coerces to 0, and a falsely-zero nominal turns the compensated + # scale-out into a down-PATCH from Azure's real figure. The pass has to end + # before a step is priced from a guessed number. + $script:CapacityReadMalformed = $true + + Invoke-FleetReconcile 3>$null + + Should -Invoke Get-FleetCapacityStep -ModuleName FleetCore -Times 0 -Exactly + Should -Invoke Invoke-ArmWeb -ModuleName FleetCore -Times 0 -Exactly + Should -Invoke Set-FleetHeartbeat -ModuleName FleetCore -Times 0 -Exactly + } + + It "bails out of the pass when the capacity read carries a non-numeric sku" { + # A non-integral capacity would throw InvalidCastException at the [int] + # cast, which the TransientFleetException catch does not cover -- the + # invocation would crash instead of skipping the pass. The guard has to + # refuse it the same way it refuses a missing sku. + $script:CapacityReadNonNumeric = $true + + Invoke-FleetReconcile 3>$null + + Should -Invoke Get-FleetCapacityStep -ModuleName FleetCore -Times 0 -Exactly + Should -Invoke Invoke-ArmWeb -ModuleName FleetCore -Times 0 -Exactly + Should -Invoke Set-FleetHeartbeat -ModuleName FleetCore -Times 0 -Exactly + } + + It "bails out of the pass when the capacity read overflows Int32" { + # A digit-only string past Int32.MaxValue survives any digits-shaped + # validation and then overflows the [int] cast, which throws past the + # TransientFleetException catch. TryParse has to refuse it up front. + $script:CapacityReadOverflow = $true + + Invoke-FleetReconcile 3>$null + + Should -Invoke Get-FleetCapacityStep -ModuleName FleetCore -Times 0 -Exactly + Should -Invoke Invoke-ArmWeb -ModuleName FleetCore -Times 0 -Exactly + Should -Invoke Set-FleetHeartbeat -ModuleName FleetCore -Times 0 -Exactly + } + + It "bails out of the pass when the inventory read comes back null" { + # A null Vms list is a garbled read, not an empty fleet -- @($null).Count is 1, + # so unguarded it would impersonate a single live VM -- and a null runner list + # would slide through the reclaim corroboration as zero online. Neither may + # price a step. + $script:FleetStateGarbled = $true + + Invoke-FleetReconcile 3>$null + + Should -Invoke Get-FleetCapacityStep -ModuleName FleetCore -Times 0 -Exactly + Should -Invoke Invoke-ArmWeb -ModuleName FleetCore -Times 0 -Exactly + Should -Invoke Set-FleetHeartbeat -ModuleName FleetCore -Times 0 -Exactly + } + + It "refuses the zero reclaim while GitHub still shows an online runner" { + # The reclaim is computed from the ARM VM list, and GitHub is an independent + # witness against it: a runner cannot be online without a live VM behind it, + # so an online count contradicting an empty list means the list is stale and + # the PATCH would delete the instances the list failed to return. + $script:FleetListsVms = $false + $script:CapacityStepValue = 0 + $script:FleetOnlineRunners = @( + [pscustomobject]@{ + name = "dbatools-runners_a" + status = "online" + busy = $false + } + ) + + Invoke-FleetReconcile 3>$null + + Should -Invoke Invoke-ArmWeb -ModuleName FleetCore -Times 0 -Exactly + } + + It "refuses the zero reclaim when the confirming re-read finds the fleet repopulated" { + # The first read came back empty and priced the step at zero; the confirming + # read sees a VM that the first one missed. One witness recanting is enough + # to hold fire for a pass. + $script:FleetListsVms = $false + $script:CapacityStepValue = 0 + $script:FleetRepopulatesOnConfirm = $true + + Invoke-FleetReconcile 3>$null + + Should -Invoke Invoke-ArmWeb -ModuleName FleetCore -Times 0 -Exactly + } + + It "reclaims to zero once GitHub agrees the fleet is empty" { + $script:FleetListsVms = $false + $script:CapacityStepValue = 0 + + Invoke-FleetReconcile + + Should -Invoke Invoke-ArmWeb -ModuleName FleetCore -Times 1 -Exactly -ParameterFilter { $Body.sku.capacity -eq 0 } + } +} + +Describe "fleet state shape validation" { + BeforeEach { + InModuleScope FleetCore { + $script:Fleet = [pscustomobject]@{ + Repo = "dataplat/dbatools" + RunnerLabel = "dbatools-modern" + SubscriptionId = "sub" + ResourceGroup = "rg" + Vmss = "dbatools-runners" + } + } + $script:GhPayload = $null + $script:ArmPage = $null + Mock -ModuleName FleetCore Invoke-GhJson { $script:GhPayload } + Mock -ModuleName FleetCore Invoke-ArmJson { $script:ArmPage } + } + + It "throws on a GitHub payload without a runners array instead of erasing the fleet" { + # A garbled 200 flattened through @() reads as zero runners, which downstream + # is indistinguishable from a genuinely empty fleet. The real state function + # has to refuse the shape, not normalize it. + $script:GhPayload = [pscustomobject]@{ total_count = 3 } + + { InModuleScope FleetCore { Get-FleetState } } | Should -Throw "*runners array*" + } + + It "throws on an ARM list page without a value array instead of returning an empty inventory" { + $script:GhPayload = [pscustomobject]@{ runners = @() } + $script:ArmPage = [pscustomobject]@{ nextLink = $null } + + { InModuleScope FleetCore { Get-FleetState } } | Should -Throw "*value array*" + } + + It "throws on a runners property that is not an array" { + # A scalar or object where the array belongs is the same garble as a + # missing property: it survives a null check and flattens to an empty + # fleet through the label filter. + $script:GhPayload = [pscustomobject]@{ runners = "garbled" } + + { InModuleScope FleetCore { Get-FleetState } } | Should -Throw "*runners array*" + } + + It "throws on a value property that is not an array" { + $script:GhPayload = [pscustomobject]@{ runners = @() } + $script:ArmPage = [pscustomobject]@{ + value = [pscustomobject]@{ name = "dbatools-runners_a" } + nextLink = $null + } + + { InModuleScope FleetCore { Get-FleetState } } | Should -Throw "*value array*" + } +} + Describe "fail-closed settings" { BeforeAll { # Everything Initialize-FleetContext demands before it looks at DRY_RUN, so the diff --git a/.github/runners/tests/runner-policy.Tests.ps1 b/.github/runners/tests/runner-policy.Tests.ps1 index f997031dfd5..f509788c5d5 100644 --- a/.github/runners/tests/runner-policy.Tests.ps1 +++ b/.github/runners/tests/runner-policy.Tests.ps1 @@ -559,21 +559,14 @@ Describe "Get-VmssCapacityPlan" { } Describe "Get-FleetCapacityStep" { - It "normalizes phantom capacity on the first settled pass and scales out on the next" { - $splatFirstPass = @{ + It "scales out in one compensated step despite phantom capacity" { + $splatPhantom = @{ ProvisioningState = "Succeeded" NominalCapacity = 9 ActualCapacity = 6 TargetCapacity = 10 } - Get-FleetCapacityStep @splatFirstPass | Should -Be 6 - $splatSecondPass = @{ - ProvisioningState = "Succeeded" - NominalCapacity = 6 - ActualCapacity = 6 - TargetCapacity = 10 - } - Get-FleetCapacityStep @splatSecondPass | Should -Be 10 + Get-FleetCapacityStep @splatPhantom | Should -Be 13 } It "does not mistake an in-flight scale-out for phantom capacity" { @@ -586,14 +579,67 @@ Describe "Get-FleetCapacityStep" { Get-FleetCapacityStep @splatInFlight | Should -BeNullOrEmpty } - It "still normalizes a failed scale set so the fleet can recover" { + It "treats a missing provisioning state as an operation in flight" { + $splatMissing = @{ + ProvisioningState = "" + NominalCapacity = 9 + ActualCapacity = 6 + TargetCapacity = 10 + } + Get-FleetCapacityStep @splatMissing | Should -BeNullOrEmpty + } + + It "treats an unknown provisioning state as an operation in flight" { + $splatUnknown = @{ + ProvisioningState = "SomeFutureArmState" + NominalCapacity = 9 + ActualCapacity = 6 + TargetCapacity = 10 + } + Get-FleetCapacityStep @splatUnknown | Should -BeNullOrEmpty + } + + It "keeps scaling out while churn mints fresh phantom capacity" { + $splatChurn = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 7 + ActualCapacity = 5 + TargetCapacity = 20 + } + Get-FleetCapacityStep @splatChurn | Should -Be 22 + } + + It "leaves a ceiling-pinned shortfall alone rather than delete live runners" { + # A down-PATCH removes nominal-minus-newValue live members, so "reclaiming + # headroom" here would kill 15 running VMs. The drift unwinds only at the + # zero-crossing; until then the pinned lane just waits. + $splatPinned = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 35 + ActualCapacity = 20 + TargetCapacity = 25 + } + Get-FleetCapacityStep @splatPinned | Should -BeNullOrEmpty + } + + It "leaves phantom capacity alone once demand is satisfied" { + $splatQuiet = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 12 + ActualCapacity = 10 + TargetCapacity = 10 + } + Get-FleetCapacityStep @splatQuiet | Should -BeNullOrEmpty + } + + It "still recovers a failed scale set" { $splatFailed = @{ ProvisioningState = "Failed" NominalCapacity = 9 ActualCapacity = 6 TargetCapacity = 10 } - Get-FleetCapacityStep @splatFailed | Should -Be 6 + Get-FleetCapacityStep @splatFailed | Should -Be 13 } It "emits nothing when settled capacity already matches the target" { @@ -605,6 +651,130 @@ Describe "Get-FleetCapacityStep" { } Get-FleetCapacityStep @splatSettled | Should -BeNullOrEmpty } + + It "still recovers a canceled scale set" { + $splatCanceled = @{ + ProvisioningState = "Canceled" + NominalCapacity = 9 + ActualCapacity = 6 + TargetCapacity = 10 + } + Get-FleetCapacityStep @splatCanceled | Should -Be 13 + } + + It "compensates from nominal even when actual runs ahead of it" { + # 7 looks wrong next to 8 real instances, but creation is nominal-delta: Azure + # makes newValue-minus-nominal VMs, so 7 creates exactly the 2 the target needs. + $splatInverted = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 5 + ActualCapacity = 8 + TargetCapacity = 10 + } + Get-FleetCapacityStep @splatInverted | Should -Be 7 + } + + It "normalizes an emptied fleet down to zero" { + $splatDrained = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 3 + ActualCapacity = 0 + TargetCapacity = 0 + } + Get-FleetCapacityStep @splatDrained | Should -Be 0 + } + + It "emits nothing when actual exceeds nominal and demand is met" { + $splatSurplus = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 5 + ActualCapacity = 8 + TargetCapacity = 8 + } + Get-FleetCapacityStep @splatSurplus | Should -BeNullOrEmpty + } + + It "tolerates a nominal above the ceiling without crashing or reclaiming" { + $splatRunaway = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 40 + ActualCapacity = 16 + TargetCapacity = 20 + } + Get-FleetCapacityStep @splatRunaway | Should -BeNullOrEmpty + } + + It "waits out a single drifted slot when pinned one short of the target" { + # Reclaiming the slot would PATCH 35 down to 34 and delete a live runner to + # free it. One target slot lost to drift is cheaper than one killed VM. + $splatPinnedSlot = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 35 + ActualCapacity = 34 + TargetCapacity = 35 + } + Get-FleetCapacityStep @splatPinnedSlot | Should -BeNullOrEmpty + } + + It "skips the pass when nominal telemetry arrives negative" { + $splatNegativeNominal = @{ + ProvisioningState = "Succeeded" + NominalCapacity = -1 + ActualCapacity = 2 + TargetCapacity = 10 + } + Get-FleetCapacityStep @splatNegativeNominal | Should -BeNullOrEmpty + } + + It "skips the pass when actual telemetry arrives negative" { + $splatNegativeActual = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 5 + ActualCapacity = -3 + TargetCapacity = 10 + } + Get-FleetCapacityStep @splatNegativeActual | Should -BeNullOrEmpty + } + + It "emits nothing for an over-ceiling fleet with drift above it" { + # There is no safe move here: 42-to-40 would delete two live members, and + # clamping to the 35 ceiling would delete seven. The two-slot drift rides + # along until the fleet empties and crosses zero. + $splatOverCeilingActual = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 42 + ActualCapacity = 40 + TargetCapacity = 20 + } + Get-FleetCapacityStep @splatOverCeilingActual | Should -BeNullOrEmpty + } + + It "unwinds a ceiling-pinned nominal through the zero-crossing" { + # With zero members a down-PATCH deletes nothing, so this is the one place + # drift can be repaid: nominal drops to 0 and the next pass creates the + # whole target from a clean slate. + $splatPinnedEmpty = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 35 + ActualCapacity = 0 + TargetCapacity = 35 + } + Get-FleetCapacityStep @splatPinnedEmpty | Should -Be 0 + } + + It "reclaims an empty fleet to zero instead of taking a ceiling-clipped step" { + # Compensating here would PATCH to 35, which creates only five instances + # against a target of ten and then pins there. With zero members the + # reclaim is free, so the drift is repaid first and the next pass creates + # the full target from a clean nominal. + $splatClippedEmpty = @{ + ProvisioningState = "Succeeded" + NominalCapacity = 30 + ActualCapacity = 0 + TargetCapacity = 10 + } + Get-FleetCapacityStep @splatClippedEmpty | Should -Be 0 + } } Describe "Flexible VMSS capacity reconciliation" {