@@ -2,21 +2,25 @@ package elasticsearch
22
33import (
44 "fmt"
5+ "time"
56
67 "github.com/spf13/cobra"
78 "github.com/stackvista/stackstate-backup-cli/cmd/cmdutils"
89 "github.com/stackvista/stackstate-backup-cli/internal/app"
910 es "github.com/stackvista/stackstate-backup-cli/internal/clients/elasticsearch"
1011 "github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
12+ "github.com/stackvista/stackstate-backup-cli/internal/foundation/logger"
1113 "github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward"
1214 "github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore"
1315 "github.com/stackvista/stackstate-backup-cli/internal/orchestration/scale"
1416)
1517
1618// Check-and-finalize command flags
1719var (
18- checkOperationID string
19- checkWait bool
20+ checkOperationID string
21+ checkWait bool
22+ checkNoProgressIn time.Duration
23+ checkFinalizeOnly bool
2024)
2125
2226func checkAndFinalizeCmd (globalFlags * config.CLIGlobalFlags ) * cobra.Command {
@@ -25,19 +29,38 @@ func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
2529 Short : "Check restore status and finalize if complete" ,
2630 Long : `Check the status of a restore operation and perform finalization (scale up deployments) if complete.
2731If the restore is still running and --wait is specified, wait for completion before finalizing.` ,
32+ PreRunE : func (_ * cobra.Command , _ []string ) error {
33+ if ! checkFinalizeOnly && checkOperationID == "" {
34+ return fmt .Errorf ("--operation-id is required unless --finalize-only is set" )
35+ }
36+
37+ return nil
38+ },
2839 Run : func (_ * cobra.Command , _ []string ) {
2940 cmdutils .Run (globalFlags , runCheckAndFinalize , cmdutils .StorageIsRequired )
3041 },
3142 }
3243
33- cmd .Flags ().StringVar (& checkOperationID , "operation-id" , "" , "Operation ID of the restore operation (required)" )
44+ cmd .Flags ().StringVar (& checkOperationID , "operation-id" , "" ,
45+ "Snapshot name of the restore operation (required unless --finalize-only)" )
3446 cmd .Flags ().BoolVar (& checkWait , "wait" , false , "Wait for restore to complete if still running" )
35- _ = cmd .MarkFlagRequired ("operation-id" )
47+ cmd .Flags ().DurationVar (& checkNoProgressIn , "no-progress-timeout" , defaultNoProgressTimeout , noProgressTimeoutUsage )
48+ cmd .Flags ().BoolVar (& checkFinalizeOnly , "finalize-only" , false ,
49+ "Scale the deployments back up and release the restore lock without checking restore status. " +
50+ "Use when the snapshot can no longer be read and the restore is known to be finished" )
51+ cmd .MarkFlagsMutuallyExclusive ("finalize-only" , "wait" )
3652
3753 return cmd
3854}
3955
4056func runCheckAndFinalize (appCtx * app.Context ) error {
57+ // Finalizing needs Kubernetes only, so it stays available when Elasticsearch or the snapshot
58+ // cannot be reached at all - otherwise nothing in the CLI can release the restore lock.
59+ if checkFinalizeOnly {
60+ appCtx .Logger .Warningf ("Skipping the restore status check on request; finalizing" )
61+ return finalizeRestore (appCtx )
62+ }
63+
4164 // Setup port-forward to Elasticsearch
4265 serviceName := appCtx .Config .Elasticsearch .Service .Name
4366 remotePort := appCtx .Config .Elasticsearch .Service .Port
@@ -56,13 +79,223 @@ func runCheckAndFinalize(appCtx *app.Context) error {
5679
5780 repository := appCtx .Config .Elasticsearch .Restore .Repository
5881
59- return checkAndFinalize (esClient , appCtx , repository , checkOperationID , checkWait )
82+ return checkAndFinalize (esClient , appCtx , repository , checkOperationID , checkWait , checkNoProgressIn )
83+ }
84+
85+ const (
86+ // defaultNoProgressTimeout bounds the wait so a stuck restore fails instead of polling until
87+ // the caller's own timeout with the workloads scaled down and the restore lock held. Generous
88+ // on purpose: a spurious failure aborts a working restore, while the only cost of waiting too
89+ // long is a later failure. Progress is counted per primary shard, so tripping this means no
90+ // shard at all completed in the window.
91+ defaultNoProgressTimeout = 2 * time .Hour
92+
93+ noProgressTimeoutUsage = "Fail if no primary shard finishes restoring within this duration. " +
94+ "Bounds inactivity, not total restore time; 0 waits indefinitely"
95+
96+ // restoreStatusMaxErrors tolerates a port-forward dropping mid-restore. Polling now spans the
97+ // whole restore, so a pod restart must not end a restore that is still running server-side.
98+ restoreStatusMaxErrors = 5
99+ )
100+
101+ // reconnectingHealthClient rebuilds the port-forward and Elasticsearch client after a failed
102+ // health check. It starts from the caller's client so the happy path opens no extra port-forward.
103+ type reconnectingHealthClient struct {
104+ appCtx * app.Context
105+ client indicesHealthGetter
106+ pf * portforward.Conn
107+ }
108+
109+ func (r * reconnectingHealthClient ) GetIndicesHealth () (map [string ]es.IndexHealth , error ) {
110+ if r .client == nil {
111+ if err := r .connect (); err != nil {
112+ return nil , err
113+ }
114+ }
115+
116+ health , err := r .client .GetIndicesHealth ()
117+ if err != nil {
118+ // The tunnel is bound to a fixed local port, so a broken one never recovers. Drop it and
119+ // let the next poll dial a fresh pod.
120+ r .disconnect ()
121+ return nil , err
122+ }
123+
124+ return health , nil
125+ }
126+
127+ func (r * reconnectingHealthClient ) connect () error {
128+ pf , err := portforward .SetupPortForward (
129+ r .appCtx .K8sClient ,
130+ r .appCtx .Namespace ,
131+ r .appCtx .Config .Elasticsearch .Service .Name ,
132+ r .appCtx .Config .Elasticsearch .Service .Port ,
133+ r .appCtx .Logger ,
134+ )
135+ if err != nil {
136+ return err
137+ }
138+
139+ client , err := r .appCtx .NewESClient (pf .LocalPort )
140+ if err != nil {
141+ close (pf .StopChan )
142+ return fmt .Errorf ("failed to create Elasticsearch client: %w" , err )
143+ }
144+
145+ r .pf , r .client = pf , client
146+
147+ return nil
148+ }
149+
150+ // disconnect drops the client and closes only a port-forward this type opened itself; the one the
151+ // caller passed in is closed by the caller.
152+ func (r * reconnectingHealthClient ) disconnect () {
153+ if r .pf != nil {
154+ close (r .pf .StopChan )
155+ r .pf = nil
156+ }
157+
158+ r .client = nil
159+ }
160+
161+ // expectedRestoredIndices returns the snapshot's indices that this restore recreates. The snapshot
162+ // is re-read rather than passed in so that check-and-finalize works from just a snapshot name;
163+ // snapshots are immutable, so the list cannot drift.
164+ func expectedRestoredIndices (esClient es.Interface , appCtx * app.Context , repository , snapshotName string ) ([]string , error ) {
165+ snapshot , err := esClient .GetSnapshot (repository , snapshotName )
166+ if err != nil {
167+ return nil , fmt .Errorf ("failed to get snapshot details: %w" , err )
168+ }
169+
170+ expected := filterSTSIndices (
171+ snapshot .Indices ,
172+ appCtx .Config .Elasticsearch .Restore .IndexPrefix ,
173+ appCtx .Config .Elasticsearch .Restore .DatastreamIndexPrefix ,
174+ )
175+ if len (expected ) == 0 {
176+ return nil , fmt .Errorf ("snapshot %s contains no indices matching the configured STS prefixes" , snapshotName )
177+ }
178+
179+ return expected , nil
180+ }
181+
182+ // restoreProgress summarises how far a restore has got. Replicas are deliberately ignored: a
183+ // cluster with fewer nodes than replicas keeps them unassigned forever, so requiring green would
184+ // never complete there.
185+ type restoreProgress struct {
186+ // indicesRestored counts expected indices with every primary shard active, and decides completion.
187+ indicesRestored int
188+ // primariesActive counts active primary shards, and drives stall detection. Whole indices are
189+ // too coarse for that: a large multi-shard index can restore for a long time without finishing.
190+ primariesActive int
60191}
61192
62- func checkAndFinalize (esClient es.Interface , appCtx * app.Context , repository , snapshotName string , waitForComplete bool ) error {
193+ func measureRestore (expected []string , health map [string ]es.IndexHealth ) restoreProgress {
194+ var progress restoreProgress
195+
196+ for _ , index := range expected {
197+ indexHealth , exists := health [index ]
198+ if ! exists {
199+ continue
200+ }
201+
202+ progress .primariesActive += indexHealth .ActivePrimaryShards
203+ if indexHealth .NumberOfShards > 0 && indexHealth .ActivePrimaryShards == indexHealth .NumberOfShards {
204+ progress .indicesRestored ++
205+ }
206+ }
207+
208+ return progress
209+ }
210+
211+ type indicesHealthGetter interface {
212+ GetIndicesHealth () (map [string ]es.IndexHealth , error )
213+ }
214+
215+ // newRestoreStatusFn builds the status callback used for both the single check and the wait loop.
216+ // Completion is derived from the restored indices themselves: a restore that has been accepted but
217+ // not yet applied is indistinguishable from a finished one when judged by recovery activity alone.
218+ func newRestoreStatusFn (
219+ esClient indicesHealthGetter ,
220+ log * logger.Logger ,
221+ expected []string ,
222+ noProgressTimeout time.Duration ,
223+ maxErrors int ,
224+ ) func () (string , bool , error ) {
225+ lastPrimariesActive := - 1
226+ lastProgressAt := time .Now ()
227+ errCount := 0
228+
229+ return func () (string , bool , error ) {
230+ health , err := esClient .GetIndicesHealth ()
231+ if err != nil {
232+ errCount ++
233+ if errCount >= maxErrors {
234+ return "" , false , err
235+ }
236+
237+ log .Warningf ("Restore status check failed (%d/%d), retrying: %v" , errCount , maxErrors , err )
238+
239+ return es .StatusInProgress , false , nil
240+ }
241+ errCount = 0
242+
243+ progress := measureRestore (expected , health )
244+ if progress .indicesRestored == len (expected ) {
245+ return es .StatusSuccess , true , nil
246+ }
247+
248+ if progress .primariesActive != lastPrimariesActive {
249+ lastPrimariesActive = progress .primariesActive
250+ lastProgressAt = time .Now ()
251+ } else if noProgressTimeout > 0 && time .Since (lastProgressAt ) > noProgressTimeout {
252+ // Deliberately not reported as a failed restore: all this establishes is that no progress
253+ // was observed from here. Restarting a restore deletes every STS index first, so a caller
254+ // that reads this as "failed" and retries would destroy a restore that is merely slow.
255+ return "" , false , fmt .Errorf (
256+ "elasticsearch restore stalled: no primary shard finished restoring in %s " +
257+ "(%d of %d indices complete, %d primaries active); " +
258+ "it may still be running server-side, so check before restarting it" ,
259+ noProgressTimeout , progress .indicesRestored , len (expected ), progress .primariesActive ,
260+ )
261+ }
262+
263+ log .Debugf (
264+ "Restored %d of %d indices (%d primaries active)" ,
265+ progress .indicesRestored , len (expected ), progress .primariesActive ,
266+ )
267+
268+ return es .StatusInProgress , false , nil
269+ }
270+ }
271+
272+ func checkAndFinalize (
273+ esClient es.Interface ,
274+ appCtx * app.Context ,
275+ repository , snapshotName string ,
276+ waitForComplete bool ,
277+ noProgressTimeout time.Duration ,
278+ ) error {
279+ expected , err := expectedRestoredIndices (esClient , appCtx , repository , snapshotName )
280+ if err != nil {
281+ return err
282+ }
283+
284+ healthClient := & reconnectingHealthClient {appCtx : appCtx , client : esClient }
285+ defer healthClient .disconnect ()
286+
287+ // Retrying only pays off while polling. A one-shot check that swallowed the error would report
288+ // a dead tunnel as a running restore and exit 0.
289+ maxErrors := restoreStatusMaxErrors
290+ if ! waitForComplete {
291+ maxErrors = 1
292+ }
293+
294+ statusFn := newRestoreStatusFn (healthClient , appCtx .Logger , expected , noProgressTimeout , maxErrors )
295+
63296 // Get restore status
64- appCtx .Logger .Infof ("Checking restore status for snapshot: %s" , snapshotName )
65- status , isComplete , err := esClient . GetRestoreStatus ( repository , snapshotName )
297+ appCtx .Logger .Infof ("Checking restore status for snapshot: %s (%d indices) " , snapshotName , len ( expected ) )
298+ status , isComplete , err := statusFn ( )
66299 if err != nil {
67300 return fmt .Errorf ("failed to get restore status: %w" , err )
68301 }
@@ -72,17 +305,9 @@ func checkAndFinalize(esClient es.Interface, appCtx *app.Context, repository, sn
72305 // Handle different scenarios
73306 if isComplete {
74307 switch status {
75- case "SUCCESS" :
308+ case es . StatusSuccess :
76309 appCtx .Logger .Successf ("Restore completed successfully" )
77310 return finalizeRestore (appCtx )
78- case "NOT_FOUND" :
79- appCtx .Logger .Infof ("No restore operation found for snapshot: %s" , snapshotName )
80- appCtx .Logger .Infof ("The restore may have already been finalized" )
81- appCtx .Logger .Println ()
82- appCtx .Logger .Infof ("Checking if deployments need to be scaled up..." )
83- return attemptScaleUp (appCtx )
84- case "FAILED" :
85- return fmt .Errorf ("restore failed with status: %s" , status )
86311 default :
87312 return fmt .Errorf ("restore completed with unexpected status: %s" , status )
88313 }
@@ -93,7 +318,7 @@ func checkAndFinalize(esClient es.Interface, appCtx *app.Context, repository, sn
93318
94319 if waitForComplete {
95320 appCtx .Logger .Println ()
96- return waitAndFinalize (esClient , appCtx , repository , snapshotName )
321+ return waitAndFinalize (statusFn , appCtx , snapshotName )
97322 }
98323
99324 // Not waiting - print status and exit
@@ -103,15 +328,10 @@ func checkAndFinalize(esClient es.Interface, appCtx *app.Context, repository, sn
103328}
104329
105330// waitAndFinalize waits for restore to complete and finalizes (scale up)
106- func waitAndFinalize (esClient es. Interface , appCtx * app.Context , repository , snapshotName string ) error {
331+ func waitAndFinalize (statusFn func () ( string , bool , error ), appCtx * app.Context , snapshotName string ) error {
107332 restore .PrintAPIWaitingMessage ("elasticsearch" , snapshotName , appCtx .Namespace , appCtx .Logger )
108333
109- // Wait for restore to complete
110- checkStatusFn := func () (string , bool , error ) {
111- return esClient .GetRestoreStatus (repository , snapshotName )
112- }
113-
114- if err := restore .WaitForAPIRestore (checkStatusFn , 0 , appCtx .Logger ); err != nil {
334+ if err := restore .WaitForAPIRestore (statusFn , 0 , appCtx .Logger ); err != nil {
115335 return err
116336 }
117337
@@ -129,20 +349,3 @@ func finalizeRestore(appCtx *app.Context) error {
129349
130350 return restore .FinalizeRestore (scaleUpFn , appCtx .Logger )
131351}
132-
133- // attemptScaleUp tries to scale up deployments and release lock (used when restore is not found/already complete)
134- func attemptScaleUp (appCtx * app.Context ) error {
135- labelSelector := appCtx .Config .Elasticsearch .Restore .ScaleDownLabelSelector
136- scaleUpFn := func () error {
137- return scale .ScaleUpAndReleaseLock (appCtx .K8sClient , appCtx .Namespace , labelSelector , appCtx .Logger )
138- }
139-
140- if err := scaleUpFn (); err != nil {
141- // Don't fail if no deployments found to scale up
142- appCtx .Logger .Infof ("No deployments found to scale up (this is normal if already finalized)" )
143- return nil
144- }
145-
146- appCtx .Logger .Successf ("Finalization completed successfully" )
147- return nil
148- }
0 commit comments