Skip to content

Commit 0b157e1

Browse files
committed
STAC-25639: elasticsearch restore wait until all shards and indices are healthy
1 parent b692f9e commit 0b157e1

8 files changed

Lines changed: 686 additions & 87 deletions

File tree

‎cmd/elasticsearch/check_and_finalize.go‎

Lines changed: 245 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,25 @@ package elasticsearch
22

33
import (
44
"fmt"
5+
"time"
56

67
"github.com/spf13/cobra"
78
"github.com/stackvista/stackstate-backup-cli/cmd/cmdutils"
89
"github.com/stackvista/stackstate-backup-cli/internal/app"
910
es "github.com/stackvista/stackstate-backup-cli/internal/clients/elasticsearch"
1011
"github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
12+
"github.com/stackvista/stackstate-backup-cli/internal/foundation/logger"
1113
"github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward"
1214
"github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore"
1315
"github.com/stackvista/stackstate-backup-cli/internal/orchestration/scale"
1416
)
1517

1618
// Check-and-finalize command flags
1719
var (
18-
checkOperationID string
19-
checkWait bool
20+
checkOperationID string
21+
checkWait bool
22+
checkNoProgressIn time.Duration
23+
checkFinalizeOnly bool
2024
)
2125

2226
func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
@@ -25,19 +29,38 @@ func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
2529
Short: "Check restore status and finalize if complete",
2630
Long: `Check the status of a restore operation and perform finalization (scale up deployments) if complete.
2731
If the restore is still running and --wait is specified, wait for completion before finalizing.`,
32+
PreRunE: func(_ *cobra.Command, _ []string) error {
33+
if !checkFinalizeOnly && checkOperationID == "" {
34+
return fmt.Errorf("--operation-id is required unless --finalize-only is set")
35+
}
36+
37+
return nil
38+
},
2839
Run: func(_ *cobra.Command, _ []string) {
2940
cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.StorageIsRequired)
3041
},
3142
}
3243

33-
cmd.Flags().StringVar(&checkOperationID, "operation-id", "", "Operation ID of the restore operation (required)")
44+
cmd.Flags().StringVar(&checkOperationID, "operation-id", "",
45+
"Snapshot name of the restore operation (required unless --finalize-only)")
3446
cmd.Flags().BoolVar(&checkWait, "wait", false, "Wait for restore to complete if still running")
35-
_ = cmd.MarkFlagRequired("operation-id")
47+
cmd.Flags().DurationVar(&checkNoProgressIn, "no-progress-timeout", defaultNoProgressTimeout, noProgressTimeoutUsage)
48+
cmd.Flags().BoolVar(&checkFinalizeOnly, "finalize-only", false,
49+
"Scale the deployments back up and release the restore lock without checking restore status. "+
50+
"Use when the snapshot can no longer be read and the restore is known to be finished")
51+
cmd.MarkFlagsMutuallyExclusive("finalize-only", "wait")
3652

3753
return cmd
3854
}
3955

4056
func runCheckAndFinalize(appCtx *app.Context) error {
57+
// Finalizing needs Kubernetes only, so it stays available when Elasticsearch or the snapshot
58+
// cannot be reached at all - otherwise nothing in the CLI can release the restore lock.
59+
if checkFinalizeOnly {
60+
appCtx.Logger.Warningf("Skipping the restore status check on request; finalizing")
61+
return finalizeRestore(appCtx)
62+
}
63+
4164
// Setup port-forward to Elasticsearch
4265
serviceName := appCtx.Config.Elasticsearch.Service.Name
4366
remotePort := appCtx.Config.Elasticsearch.Service.Port
@@ -56,13 +79,223 @@ func runCheckAndFinalize(appCtx *app.Context) error {
5679

5780
repository := appCtx.Config.Elasticsearch.Restore.Repository
5881

59-
return checkAndFinalize(esClient, appCtx, repository, checkOperationID, checkWait)
82+
return checkAndFinalize(esClient, appCtx, repository, checkOperationID, checkWait, checkNoProgressIn)
83+
}
84+
85+
const (
86+
// defaultNoProgressTimeout bounds the wait so a stuck restore fails instead of polling until
87+
// the caller's own timeout with the workloads scaled down and the restore lock held. Generous
88+
// on purpose: a spurious failure aborts a working restore, while the only cost of waiting too
89+
// long is a later failure. Progress is counted per primary shard, so tripping this means no
90+
// shard at all completed in the window.
91+
defaultNoProgressTimeout = 2 * time.Hour
92+
93+
noProgressTimeoutUsage = "Fail if no primary shard finishes restoring within this duration. " +
94+
"Bounds inactivity, not total restore time; 0 waits indefinitely"
95+
96+
// restoreStatusMaxErrors tolerates a port-forward dropping mid-restore. Polling now spans the
97+
// whole restore, so a pod restart must not end a restore that is still running server-side.
98+
restoreStatusMaxErrors = 5
99+
)
100+
101+
// reconnectingHealthClient rebuilds the port-forward and Elasticsearch client after a failed
102+
// health check. It starts from the caller's client so the happy path opens no extra port-forward.
103+
type reconnectingHealthClient struct {
104+
appCtx *app.Context
105+
client indicesHealthGetter
106+
pf *portforward.Conn
107+
}
108+
109+
func (r *reconnectingHealthClient) GetIndicesHealth() (map[string]es.IndexHealth, error) {
110+
if r.client == nil {
111+
if err := r.connect(); err != nil {
112+
return nil, err
113+
}
114+
}
115+
116+
health, err := r.client.GetIndicesHealth()
117+
if err != nil {
118+
// The tunnel is bound to a fixed local port, so a broken one never recovers. Drop it and
119+
// let the next poll dial a fresh pod.
120+
r.disconnect()
121+
return nil, err
122+
}
123+
124+
return health, nil
125+
}
126+
127+
func (r *reconnectingHealthClient) connect() error {
128+
pf, err := portforward.SetupPortForward(
129+
r.appCtx.K8sClient,
130+
r.appCtx.Namespace,
131+
r.appCtx.Config.Elasticsearch.Service.Name,
132+
r.appCtx.Config.Elasticsearch.Service.Port,
133+
r.appCtx.Logger,
134+
)
135+
if err != nil {
136+
return err
137+
}
138+
139+
client, err := r.appCtx.NewESClient(pf.LocalPort)
140+
if err != nil {
141+
close(pf.StopChan)
142+
return fmt.Errorf("failed to create Elasticsearch client: %w", err)
143+
}
144+
145+
r.pf, r.client = pf, client
146+
147+
return nil
148+
}
149+
150+
// disconnect drops the client and closes only a port-forward this type opened itself; the one the
151+
// caller passed in is closed by the caller.
152+
func (r *reconnectingHealthClient) disconnect() {
153+
if r.pf != nil {
154+
close(r.pf.StopChan)
155+
r.pf = nil
156+
}
157+
158+
r.client = nil
159+
}
160+
161+
// expectedRestoredIndices returns the snapshot's indices that this restore recreates. The snapshot
162+
// is re-read rather than passed in so that check-and-finalize works from just a snapshot name;
163+
// snapshots are immutable, so the list cannot drift.
164+
func expectedRestoredIndices(esClient es.Interface, appCtx *app.Context, repository, snapshotName string) ([]string, error) {
165+
snapshot, err := esClient.GetSnapshot(repository, snapshotName)
166+
if err != nil {
167+
return nil, fmt.Errorf("failed to get snapshot details: %w", err)
168+
}
169+
170+
expected := filterSTSIndices(
171+
snapshot.Indices,
172+
appCtx.Config.Elasticsearch.Restore.IndexPrefix,
173+
appCtx.Config.Elasticsearch.Restore.DatastreamIndexPrefix,
174+
)
175+
if len(expected) == 0 {
176+
return nil, fmt.Errorf("snapshot %s contains no indices matching the configured STS prefixes", snapshotName)
177+
}
178+
179+
return expected, nil
180+
}
181+
182+
// restoreProgress summarises how far a restore has got. Replicas are deliberately ignored: a
183+
// cluster with fewer nodes than replicas keeps them unassigned forever, so requiring green would
184+
// never complete there.
185+
type restoreProgress struct {
186+
// indicesRestored counts expected indices with every primary shard active, and decides completion.
187+
indicesRestored int
188+
// primariesActive counts active primary shards, and drives stall detection. Whole indices are
189+
// too coarse for that: a large multi-shard index can restore for a long time without finishing.
190+
primariesActive int
60191
}
61192

62-
func checkAndFinalize(esClient es.Interface, appCtx *app.Context, repository, snapshotName string, waitForComplete bool) error {
193+
func measureRestore(expected []string, health map[string]es.IndexHealth) restoreProgress {
194+
var progress restoreProgress
195+
196+
for _, index := range expected {
197+
indexHealth, exists := health[index]
198+
if !exists {
199+
continue
200+
}
201+
202+
progress.primariesActive += indexHealth.ActivePrimaryShards
203+
if indexHealth.NumberOfShards > 0 && indexHealth.ActivePrimaryShards == indexHealth.NumberOfShards {
204+
progress.indicesRestored++
205+
}
206+
}
207+
208+
return progress
209+
}
210+
211+
type indicesHealthGetter interface {
212+
GetIndicesHealth() (map[string]es.IndexHealth, error)
213+
}
214+
215+
// newRestoreStatusFn builds the status callback used for both the single check and the wait loop.
216+
// Completion is derived from the restored indices themselves: a restore that has been accepted but
217+
// not yet applied is indistinguishable from a finished one when judged by recovery activity alone.
218+
func newRestoreStatusFn(
219+
esClient indicesHealthGetter,
220+
log *logger.Logger,
221+
expected []string,
222+
noProgressTimeout time.Duration,
223+
maxErrors int,
224+
) func() (string, bool, error) {
225+
lastPrimariesActive := -1
226+
lastProgressAt := time.Now()
227+
errCount := 0
228+
229+
return func() (string, bool, error) {
230+
health, err := esClient.GetIndicesHealth()
231+
if err != nil {
232+
errCount++
233+
if errCount >= maxErrors {
234+
return "", false, err
235+
}
236+
237+
log.Warningf("Restore status check failed (%d/%d), retrying: %v", errCount, maxErrors, err)
238+
239+
return es.StatusInProgress, false, nil
240+
}
241+
errCount = 0
242+
243+
progress := measureRestore(expected, health)
244+
if progress.indicesRestored == len(expected) {
245+
return es.StatusSuccess, true, nil
246+
}
247+
248+
if progress.primariesActive != lastPrimariesActive {
249+
lastPrimariesActive = progress.primariesActive
250+
lastProgressAt = time.Now()
251+
} else if noProgressTimeout > 0 && time.Since(lastProgressAt) > noProgressTimeout {
252+
// Deliberately not reported as a failed restore: all this establishes is that no progress
253+
// was observed from here. Restarting a restore deletes every STS index first, so a caller
254+
// that reads this as "failed" and retries would destroy a restore that is merely slow.
255+
return "", false, fmt.Errorf(
256+
"elasticsearch restore stalled: no primary shard finished restoring in %s "+
257+
"(%d of %d indices complete, %d primaries active); "+
258+
"it may still be running server-side, so check before restarting it",
259+
noProgressTimeout, progress.indicesRestored, len(expected), progress.primariesActive,
260+
)
261+
}
262+
263+
log.Debugf(
264+
"Restored %d of %d indices (%d primaries active)",
265+
progress.indicesRestored, len(expected), progress.primariesActive,
266+
)
267+
268+
return es.StatusInProgress, false, nil
269+
}
270+
}
271+
272+
func checkAndFinalize(
273+
esClient es.Interface,
274+
appCtx *app.Context,
275+
repository, snapshotName string,
276+
waitForComplete bool,
277+
noProgressTimeout time.Duration,
278+
) error {
279+
expected, err := expectedRestoredIndices(esClient, appCtx, repository, snapshotName)
280+
if err != nil {
281+
return err
282+
}
283+
284+
healthClient := &reconnectingHealthClient{appCtx: appCtx, client: esClient}
285+
defer healthClient.disconnect()
286+
287+
// Retrying only pays off while polling. A one-shot check that swallowed the error would report
288+
// a dead tunnel as a running restore and exit 0.
289+
maxErrors := restoreStatusMaxErrors
290+
if !waitForComplete {
291+
maxErrors = 1
292+
}
293+
294+
statusFn := newRestoreStatusFn(healthClient, appCtx.Logger, expected, noProgressTimeout, maxErrors)
295+
63296
// Get restore status
64-
appCtx.Logger.Infof("Checking restore status for snapshot: %s", snapshotName)
65-
status, isComplete, err := esClient.GetRestoreStatus(repository, snapshotName)
297+
appCtx.Logger.Infof("Checking restore status for snapshot: %s (%d indices)", snapshotName, len(expected))
298+
status, isComplete, err := statusFn()
66299
if err != nil {
67300
return fmt.Errorf("failed to get restore status: %w", err)
68301
}
@@ -72,17 +305,9 @@ func checkAndFinalize(esClient es.Interface, appCtx *app.Context, repository, sn
72305
// Handle different scenarios
73306
if isComplete {
74307
switch status {
75-
case "SUCCESS":
308+
case es.StatusSuccess:
76309
appCtx.Logger.Successf("Restore completed successfully")
77310
return finalizeRestore(appCtx)
78-
case "NOT_FOUND":
79-
appCtx.Logger.Infof("No restore operation found for snapshot: %s", snapshotName)
80-
appCtx.Logger.Infof("The restore may have already been finalized")
81-
appCtx.Logger.Println()
82-
appCtx.Logger.Infof("Checking if deployments need to be scaled up...")
83-
return attemptScaleUp(appCtx)
84-
case "FAILED":
85-
return fmt.Errorf("restore failed with status: %s", status)
86311
default:
87312
return fmt.Errorf("restore completed with unexpected status: %s", status)
88313
}
@@ -93,7 +318,7 @@ func checkAndFinalize(esClient es.Interface, appCtx *app.Context, repository, sn
93318

94319
if waitForComplete {
95320
appCtx.Logger.Println()
96-
return waitAndFinalize(esClient, appCtx, repository, snapshotName)
321+
return waitAndFinalize(statusFn, appCtx, snapshotName)
97322
}
98323

99324
// Not waiting - print status and exit
@@ -103,15 +328,10 @@ func checkAndFinalize(esClient es.Interface, appCtx *app.Context, repository, sn
103328
}
104329

105330
// waitAndFinalize waits for restore to complete and finalizes (scale up)
106-
func waitAndFinalize(esClient es.Interface, appCtx *app.Context, repository, snapshotName string) error {
331+
func waitAndFinalize(statusFn func() (string, bool, error), appCtx *app.Context, snapshotName string) error {
107332
restore.PrintAPIWaitingMessage("elasticsearch", snapshotName, appCtx.Namespace, appCtx.Logger)
108333

109-
// Wait for restore to complete
110-
checkStatusFn := func() (string, bool, error) {
111-
return esClient.GetRestoreStatus(repository, snapshotName)
112-
}
113-
114-
if err := restore.WaitForAPIRestore(checkStatusFn, 0, appCtx.Logger); err != nil {
334+
if err := restore.WaitForAPIRestore(statusFn, 0, appCtx.Logger); err != nil {
115335
return err
116336
}
117337

@@ -129,20 +349,3 @@ func finalizeRestore(appCtx *app.Context) error {
129349

130350
return restore.FinalizeRestore(scaleUpFn, appCtx.Logger)
131351
}
132-
133-
// attemptScaleUp tries to scale up deployments and release lock (used when restore is not found/already complete)
134-
func attemptScaleUp(appCtx *app.Context) error {
135-
labelSelector := appCtx.Config.Elasticsearch.Restore.ScaleDownLabelSelector
136-
scaleUpFn := func() error {
137-
return scale.ScaleUpAndReleaseLock(appCtx.K8sClient, appCtx.Namespace, labelSelector, appCtx.Logger)
138-
}
139-
140-
if err := scaleUpFn(); err != nil {
141-
// Don't fail if no deployments found to scale up
142-
appCtx.Logger.Infof("No deployments found to scale up (this is normal if already finalized)")
143-
return nil
144-
}
145-
146-
appCtx.Logger.Successf("Finalization completed successfully")
147-
return nil
148-
}

0 commit comments

Comments
 (0)