Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 18 additions & 19 deletions agent/internal/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,10 +138,6 @@ const (
containerResolveAttemptTimeout = 1 * time.Second
restoreContainerResolveInterval = 50 * time.Millisecond
restoreContainerResolveTimeout = 30 * time.Second
restoreFailedReason = "RestoreFailed"
restoreInProgressReason = "RestoreInProgress"
restorePartiallySucceededReason = "RestorePartiallySucceeded"
restoreSucceededReason = "RestoreSucceeded"
restoreAlreadyCompletedReason = "RestoreAlreadyCompleted"
restoreAlreadyPartialReason = "RestoreAlreadyPartiallySucceeded"
restoreAlreadyFailedReason = "RestoreAlreadyFailed"
Expand Down Expand Up @@ -696,7 +692,7 @@ func (w *NodeController) restorePodContainers(ctx context.Context, pod *corev1.P
// considering a CRIU replay.
recovering := restoreInProgress(pod)
message := fmt.Sprintf("Restoring %d destination container(s) from PodSnapshot %s", len(plan.mappings), plan.artifact.SnapshotName)
if err := w.applyRestoredCondition(ctx, pod, corev1.ConditionFalse, restoreInProgressReason, message); err != nil {
if err := w.applyRestoredCondition(ctx, pod, corev1.ConditionFalse, snapshotv1alpha1.RestoreReasonInProgress, message); err != nil {
emitPodEvent(ctx, w.clientset, w.log, pod, snapshotEventComponent, corev1.EventTypeWarning, restoreStatusUpdateFailedReason, err.Error())
return true
}
Expand Down Expand Up @@ -730,22 +726,22 @@ func (w *NodeController) recordRestoreResults(ctx context.Context, pod *corev1.P
"Restore from PodSnapshot %s remains in progress: %d succeeded, %d failed, %d pending (%s)",
artifact.SnapshotName, len(succeeded), len(failed), len(pending), strings.Join(pending, ", "),
)
if err := w.applyRestoredCondition(ctx, pod, corev1.ConditionFalse, restoreInProgressReason, message); err != nil {
if err := w.applyRestoredCondition(ctx, pod, corev1.ConditionFalse, snapshotv1alpha1.RestoreReasonInProgress, message); err != nil {
emitPodEvent(ctx, w.clientset, w.log, pod, snapshotEventComponent, corev1.EventTypeWarning, restoreStatusUpdateFailedReason, err.Error())
}
return true
}

if len(failed) == 0 {
message := fmt.Sprintf("Restored %d destination container(s) from PodSnapshot %s: %s", len(succeeded), artifact.SnapshotName, strings.Join(succeeded, ", "))
return w.finishRestore(ctx, pod, corev1.ConditionTrue, restoreSucceededReason, message) != nil
return w.finishRestore(ctx, pod, corev1.ConditionTrue, snapshotv1alpha1.RestoreReasonSucceeded, message) != nil
}
if len(succeeded) != 0 {
message := fmt.Sprintf("Restored %d of %d destination containers from PodSnapshot %s; failed: %s", len(succeeded), len(results), artifact.SnapshotName, strings.Join(failed, ", "))
return w.finishRestore(ctx, pod, corev1.ConditionFalse, restorePartiallySucceededReason, message) != nil
return w.finishRestore(ctx, pod, corev1.ConditionFalse, snapshotv1alpha1.RestoreReasonPartiallySucceeded, message) != nil
}
message := fmt.Sprintf("Restore failed for all %d destination container(s) from PodSnapshot %s: %s", len(failed), artifact.SnapshotName, strings.Join(failed, ", "))
return w.finishRestore(ctx, pod, corev1.ConditionFalse, restoreFailedReason, message) != nil
return w.finishRestore(ctx, pod, corev1.ConditionFalse, snapshotv1alpha1.RestoreReasonFailed, message) != nil
}

// restoreDestination resolves and restores one destination independently of
Expand Down Expand Up @@ -868,7 +864,7 @@ func (w *NodeController) runRestore(ctx context.Context, pod *corev1.Pod, artifa
// completion sentinel proves the operation already finished.
func (op *restoreOperation) recoverCompletedRestore(ctx context.Context) (bool, error) {
condition := findRestoredCondition(op.pod)
if condition == nil || condition.Status != corev1.ConditionFalse || condition.Reason != restoreInProgressReason {
if condition == nil || condition.Status != corev1.ConditionFalse || condition.Reason != snapshotv1alpha1.RestoreReasonInProgress {
return false, nil
}

Expand Down Expand Up @@ -1066,7 +1062,7 @@ func (w *NodeController) failRestorePod(ctx context.Context, pod *corev1.Pod, ca
ctx,
pod,
corev1.ConditionFalse,
restoreFailedReason,
snapshotv1alpha1.RestoreReasonFailed,
cause.Error(),
)
return err != nil
Expand Down Expand Up @@ -1224,24 +1220,27 @@ func findRestoredCondition(pod *corev1.Pod) *corev1.PodCondition {

func restoreInProgress(pod *corev1.Pod) bool {
condition := findRestoredCondition(pod)
return condition != nil && condition.Status == corev1.ConditionFalse && condition.Reason == restoreInProgressReason
return condition != nil && condition.Status == corev1.ConditionFalse && condition.Reason == snapshotv1alpha1.RestoreReasonInProgress
}

func isRestoreSucceeded(pod *corev1.Pod) bool {
condition := findRestoredCondition(pod)
return condition != nil && condition.Status == corev1.ConditionTrue
return snapshotv1alpha1.ClassifyRestoreOutcome(pod.Status.Conditions) == snapshotv1alpha1.RestoreOutcomeSucceeded
}

// isRestorePartiallySucceeded reports the terminal mixed worker outcome.
func isRestorePartiallySucceeded(pod *corev1.Pod) bool {
condition := findRestoredCondition(pod)
return condition != nil && condition.Status == corev1.ConditionFalse && condition.Reason == restorePartiallySucceededReason
return snapshotv1alpha1.ClassifyRestoreOutcome(pod.Status.Conditions) == snapshotv1alpha1.RestoreOutcomePartiallySucceeded
}

func isRestoreTerminal(pod *corev1.Pod) bool {
condition := findRestoredCondition(pod)
return isRestoreSucceeded(pod) || isRestorePartiallySucceeded(pod) ||
(condition != nil && condition.Status == corev1.ConditionFalse && condition.Reason == restoreFailedReason)
switch snapshotv1alpha1.ClassifyRestoreOutcome(pod.Status.Conditions) {
case snapshotv1alpha1.RestoreOutcomeSucceeded,
snapshotv1alpha1.RestoreOutcomeFailed,
snapshotv1alpha1.RestoreOutcomePartiallySucceeded:
return true
default:
return false
}
}

func isRestorePodActive(pod *corev1.Pod) bool {
Expand Down
49 changes: 39 additions & 10 deletions agent/internal/controller/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,35 @@ func TestReconcileRestorePodReportsAllDestinationsFailed(t *testing.T) {
assert.Contains(t, payload, "engine-1")
}

func TestIsRestoreTerminalRequiresKnownTerminalOutcome(t *testing.T) {
tests := []struct {
name string
status corev1.ConditionStatus
reason string
want bool
}{
{name: "succeeded", status: corev1.ConditionTrue, reason: snapshotv1alpha1.RestoreReasonSucceeded, want: true},
{name: "failed", status: corev1.ConditionFalse, reason: snapshotv1alpha1.RestoreReasonFailed, want: true},
{name: "partially succeeded", status: corev1.ConditionFalse, reason: snapshotv1alpha1.RestoreReasonPartiallySucceeded, want: true},
{name: "in progress", status: corev1.ConditionFalse, reason: snapshotv1alpha1.RestoreReasonInProgress},
{name: "unrecognized reason", status: corev1.ConditionFalse, reason: "RestoreIncompatible"},
{name: "unknown status", status: corev1.ConditionUnknown, reason: snapshotv1alpha1.RestoreReasonSucceeded},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pod := restorePod(nil)
pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition),
Status: test.status,
Reason: test.reason,
})

assert.Equal(t, test.want, isRestoreTerminal(pod))
})
}
}

func TestRestorePodContainersKeepsAggregateInProgressWhileDestinationIsPending(t *testing.T) {
pod := multiRestorePod()
pod.Status.ContainerStatuses = pod.Status.ContainerStatuses[:1]
Expand Down Expand Up @@ -624,7 +653,7 @@ func TestPreflightRestorePendingStates(t *testing.T) {
func TestPreflightRestoreFailsWhenInProgressSnapshotDisappears(t *testing.T) {
pod := restorePod(map[string]string{snapshotv1alpha1.RestoreFromAnnotation: "snapshot-a"})
pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition), Status: corev1.ConditionFalse, Reason: restoreInProgressReason,
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition), Status: corev1.ConditionFalse, Reason: snapshotv1alpha1.RestoreReasonInProgress,
})
w := makeTestController(t, pod)

Expand Down Expand Up @@ -720,15 +749,15 @@ func TestReconcileRestorePodReportsPendingPreflightConditionAndEvent(t *testing.
func TestPendingDependencyDoesNotOverwriteRestoreInProgress(t *testing.T) {
pod := restorePod(map[string]string{snapshotv1alpha1.RestoreFromAnnotation: "snapshot-a"})
pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition), Status: corev1.ConditionFalse, Reason: restoreInProgressReason,
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition), Status: corev1.ConditionFalse, Reason: snapshotv1alpha1.RestoreReasonInProgress,
})
w := makeTestController(t, pod)

requeue := w.handleRestorePreflightError(context.Background(), pod, newRestorePendingError("ArtifactPending", "waiting"))

assert.True(t, requeue)
assert.False(t, hasPodStatusApply(w))
assert.Equal(t, restoreInProgressReason, restoredPodCondition(pod).Reason)
assert.Equal(t, snapshotv1alpha1.RestoreReasonInProgress, restoredPodCondition(pod).Reason)
assert.True(t, sawEventReason(w.clientset.(*fake.Clientset), "ArtifactPending"))
}

Expand All @@ -742,7 +771,7 @@ func TestProcessRestoreQueueItemReportsNonRunningPhaseAsFailed(t *testing.T) {
payload := string(lastPodStatusApply(t, w).GetPatch())
assert.Contains(t, payload, `"reason":"RestoreFailed"`)
assert.Contains(t, payload, "phase Failed")
assert.True(t, sawEventReason(w.clientset.(*fake.Clientset), restoreFailedReason))
assert.True(t, sawEventReason(w.clientset.(*fake.Clientset), snapshotv1alpha1.RestoreReasonFailed))
}

func TestContainerPollingDoesNotSetInProgressBeforeExecution(t *testing.T) {
Expand Down Expand Up @@ -882,7 +911,7 @@ func TestProcessRestoreQueueItemIgnoresFailedRestoreDuringPreflight(t *testing.T
pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition),
Status: corev1.ConditionFalse,
Reason: restoreFailedReason,
Reason: snapshotv1alpha1.RestoreReasonFailed,
Message: "original restore failure",
})
w := makeTestController(t, pod)
Expand All @@ -904,7 +933,7 @@ func TestProcessRestoreQueueItemIgnoresFailedRestoreDuringPreflight(t *testing.T
assert.Zero(t, getCalls, "failed restore preflight must not read PodSnapshot or PodSnapshotContent")
condition := restoredPodCondition(pod)
require.NotNil(t, condition)
assert.Equal(t, restoreFailedReason, condition.Reason)
assert.Equal(t, snapshotv1alpha1.RestoreReasonFailed, condition.Reason)
assert.Equal(t, "original restore failure", condition.Message)
event := eventForReason(w.clientset.(*fake.Clientset), "RestoreAlreadyFailed")
require.NotNil(t, event)
Expand All @@ -921,7 +950,7 @@ func TestDeletingInProgressRestoreRemovesFinalizer(t *testing.T) {
now := metav1.Now()
pod.DeletionTimestamp = &now
pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition), Status: corev1.ConditionFalse, Reason: restoreInProgressReason,
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition), Status: corev1.ConditionFalse, Reason: snapshotv1alpha1.RestoreReasonInProgress,
})
w := makeTestController(t, pod)

Expand Down Expand Up @@ -1091,7 +1120,7 @@ func TestRestoreStatusRetryUsesCompletionSentinelWithoutReplayingRestore(t *test
assert.True(t, hasFinalizer(live, restorePodFinalizer))
condition := restoredPodCondition(live)
require.NotNil(t, condition)
assert.Equal(t, restoreInProgressReason, condition.Reason)
assert.Equal(t, snapshotv1alpha1.RestoreReasonInProgress, condition.Reason)

processQueuedRestorePod(t, w, live)
assert.Equal(t, 1, restoreCalls, "completion sentinel must prevent CRIU replay")
Expand All @@ -1103,7 +1132,7 @@ func TestRestoreStatusRetryUsesCompletionSentinelWithoutReplayingRestore(t *test
condition = restoredPodCondition(live)
require.NotNil(t, condition)
assert.Equal(t, corev1.ConditionTrue, condition.Status)
assert.Equal(t, restoreSucceededReason, condition.Reason)
assert.Equal(t, snapshotv1alpha1.RestoreReasonSucceeded, condition.Reason)
}

func TestRestoreFinalizerRemovalRetriesWithoutReplayingRestore(t *testing.T) {
Expand Down Expand Up @@ -1271,7 +1300,7 @@ func TestRunRestoreFinalizesExistingCompletionSentinelWithoutReplay(t *testing.T
pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{
Type: corev1.PodConditionType(snapshotv1alpha1.RestoredCondition),
Status: corev1.ConditionFalse,
Reason: restoreInProgressReason,
Reason: snapshotv1alpha1.RestoreReasonInProgress,
})
w := makeTestController(t, pod)
w.runtime = &fakeRuntime{resolveContainerPID: 4242}
Expand Down
66 changes: 66 additions & 0 deletions api/v1alpha1/restore_status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package v1alpha1

import corev1 "k8s.io/api/core/v1"

// RestoreOutcome is the stable, consumer-facing state of a restore request.
type RestoreOutcome string

const (
// RestoreOutcomeUnknown means the Restored condition exists but cannot be
// classified safely by this API version.
RestoreOutcomeUnknown RestoreOutcome = "Unknown"
// RestoreOutcomePending means restore has not reached a terminal outcome.
// It includes an absent condition and active restore execution.
RestoreOutcomePending RestoreOutcome = "Pending"
// RestoreOutcomeSucceeded means every requested destination was restored.
RestoreOutcomeSucceeded RestoreOutcome = "Succeeded"
// RestoreOutcomeFailed means no requested destination was restored.
RestoreOutcomeFailed RestoreOutcome = "Failed"
// RestoreOutcomePartiallySucceeded means some, but not all, requested
// destinations were restored.
RestoreOutcomePartiallySucceeded RestoreOutcome = "PartiallySucceeded"
)

// Stable reasons used on the nvidia.com/Restored Pod condition. Dependency-wait
// reasons remain agent-internal; consumers should use ClassifyRestoreOutcome.
const (
// RestoreReasonInProgress marks active restore execution.
RestoreReasonInProgress = "RestoreInProgress"
// RestoreReasonSucceeded marks a terminal all-destinations success.
RestoreReasonSucceeded = "RestoreSucceeded"
// RestoreReasonFailed marks a terminal all-destinations failure.
RestoreReasonFailed = "RestoreFailed"
// RestoreReasonPartiallySucceeded marks a terminal mixed destination outcome.
RestoreReasonPartiallySucceeded = "RestorePartiallySucceeded"
)

// ClassifyRestoreOutcome returns the public restore outcome represented by Pod
// conditions. A missing Restored condition is Pending. An unrecognized status
// or reason is Unknown so callers can apply their own version-skew policy.
func ClassifyRestoreOutcome(conditions []corev1.PodCondition) RestoreOutcome {
for _, condition := range conditions {
if condition.Type != corev1.PodConditionType(RestoredCondition) {
continue
}
if condition.Status == corev1.ConditionTrue {
return RestoreOutcomeSucceeded
}
if condition.Status != corev1.ConditionFalse {
return RestoreOutcomeUnknown
}
switch condition.Reason {
case RestoreReasonFailed:
return RestoreOutcomeFailed
case RestoreReasonPartiallySucceeded:
return RestoreOutcomePartiallySucceeded
case RestoreReasonInProgress:
return RestoreOutcomePending
default:
return RestoreOutcomeUnknown
}
}
return RestoreOutcomePending
}
81 changes: 81 additions & 0 deletions api/v1alpha1/restore_status_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package v1alpha1

import (
"testing"

corev1 "k8s.io/api/core/v1"
)

func TestClassifyRestoreOutcome(t *testing.T) {
restored := func(status corev1.ConditionStatus, reason string) corev1.PodCondition {
return corev1.PodCondition{
Type: corev1.PodConditionType(RestoredCondition),
Status: status,
Reason: reason,
}
}
tests := []struct {
name string
conditions []corev1.PodCondition
want RestoreOutcome
}{
{name: "condition absent", want: RestoreOutcomePending},
{
name: "unrelated condition",
conditions: []corev1.PodCondition{{
Type: corev1.PodReady, Status: corev1.ConditionTrue,
}},
want: RestoreOutcomePending,
},
{
name: "unrecognized dependency reason",
conditions: []corev1.PodCondition{restored(corev1.ConditionFalse, "SnapshotPending")},
want: RestoreOutcomeUnknown,
},
{
name: "unrecognized terminal reason",
conditions: []corev1.PodCondition{restored(corev1.ConditionFalse, "RestoreIncompatible")},
want: RestoreOutcomeUnknown,
},
{
name: "restore in progress",
conditions: []corev1.PodCondition{restored(corev1.ConditionFalse, RestoreReasonInProgress)},
want: RestoreOutcomePending,
},
{
name: "unknown status",
conditions: []corev1.PodCondition{restored(corev1.ConditionUnknown, RestoreReasonSucceeded)},
want: RestoreOutcomeUnknown,
},
{
name: "succeeded",
conditions: []corev1.PodCondition{restored(corev1.ConditionTrue, RestoreReasonSucceeded)},
want: RestoreOutcomeSucceeded,
},
{
name: "succeeded with omitted reason",
conditions: []corev1.PodCondition{restored(corev1.ConditionTrue, "")},
want: RestoreOutcomeSucceeded,
},
{
name: "failed",
conditions: []corev1.PodCondition{restored(corev1.ConditionFalse, RestoreReasonFailed)},
want: RestoreOutcomeFailed,
},
{
name: "partially succeeded",
conditions: []corev1.PodCondition{restored(corev1.ConditionFalse, RestoreReasonPartiallySucceeded)},
want: RestoreOutcomePartiallySucceeded,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := ClassifyRestoreOutcome(test.conditions); got != test.want {
t.Fatalf("ClassifyRestoreOutcome() = %q, want %q", got, test.want)
}
})
}
}