STAC-24630: Add stackgraph-v2 backup/restore

cmd/root.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stackvista/stackstate-backup-cli/cmd/elasticsearch"
 	"github.com/stackvista/stackstate-backup-cli/cmd/settings"
 	"github.com/stackvista/stackstate-backup-cli/cmd/stackgraph"
+	"github.com/stackvista/stackstate-backup-cli/cmd/stackgraphv2"
 	"github.com/stackvista/stackstate-backup-cli/cmd/version"
 	"github.com/stackvista/stackstate-backup-cli/cmd/victoriametrics"
 	"github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
@@ -42,6 +43,10 @@ func init() {
 	addBackupConfigFlags(stackgraphCmd)
 	rootCmd.AddCommand(stackgraphCmd)
 
+	stackgraphv2Cmd := stackgraphv2.Cmd(flags)
+	addBackupConfigFlags(stackgraphv2Cmd)
+	rootCmd.AddCommand(stackgraphv2Cmd)
+
 	settingsCmd := settings.Cmd(flags)
 	addBackupConfigFlags(settingsCmd)
 	rootCmd.AddCommand(settingsCmd)

cmd/settings/restore.go

@@ -197,7 +197,7 @@ func buildEnvVar(extraEnvVar []corev1.EnvVar, config *config.Config) []corev1.En
 	commonVar := []corev1.EnvVar{
 		{Name: "BACKUP_CONFIGURATION_BUCKET_NAME", Value: config.Settings.Bucket},
 		{Name: "BACKUP_CONFIGURATION_S3_PREFIX", Value: config.Settings.S3Prefix},
-		{Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)},
+		{Name: "S3_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)},
 		{Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()},
 		{Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()},
 		{Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()},

cmd/stackgraph/restore.go

@@ -276,7 +276,7 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV
 		{Name: "FORCE_DELETE", Value: purgeStackgraphDataFlag},
 		{Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket},
 		{Name: "BACKUP_STACKGRAPH_S3_PREFIX", Value: config.Stackgraph.S3Prefix},
-		{Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)},
+		{Name: "S3_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)},
 		{Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()},
 		{Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()},
 		{Name: "PLATFORM_VERSION", Value: config.GetPlatformVersion()},

cmd/stackgraphv2/abort.go

@@ -0,0 +1,74 @@
+package stackgraphv2
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/stackvista/stackstate-backup-cli/cmd/cmdutils"
+	"github.com/stackvista/stackstate-backup-cli/internal/app"
+	"github.com/stackvista/stackstate-backup-cli/internal/clients/k8s"
+	"github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
+	"github.com/stackvista/stackstate-backup-cli/internal/foundation/logger"
+	"github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore"
+)
+
+const (
+	abortNameTemplate = "stackgraph-v2-abort"
+	abortScript       = "/backup-restore-scripts/restore-stackgraph-backup-v2-abort.sh"
+)
+
+func abortCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "abort",
+		Short: "Abort a partially backfilled restore.",
+		Long: "Abort a partially backfilled restore. This will not load (backfill) any data but keep the data as is." +
+			"Be aware this leaves the restore incomplete, historical data will not be recovered, but leave the instance in a usable state." +
+			"This should be used when backfilling is failing and the database restore is accepted as-is.",
+		Run: func(_ *cobra.Command, _ []string) {
+			cmdutils.Run(globalFlags, runAbort, cmdutils.StorageIsRequired)
+		},
+	}
+
+	return cmd
+}
+
+func runAbort(appCtx *app.Context) error {
+	// Setup Kubernetes resources for restore job
+	appCtx.Logger.Println()
+	if err := restore.EnsureResources(appCtx.K8sClient, appCtx.Namespace, appCtx.Config, appCtx.Logger); err != nil {
+		return err
+	}
+
+	// Create restore job
+	appCtx.Logger.Println()
+	appCtx.Logger.Infof("Creating abort job.")
+
+	jobName := fmt.Sprintf("%s-%s", abortNameTemplate, time.Now().Format("20060102t150405"))
+
+	if err := createAbortJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Config); err != nil {
+		return fmt.Errorf("failed to create abort job: %w", err)
+	}
+
+	appCtx.Logger.Successf("Abort job created: %s", jobName)
+
+	err := waitAndCleanupAbortJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger)
+
+	if err != nil {
+		logAfterJobResult(appCtx.Logger, checkJobName, false)
+		return err
+	}
+
+	return nil
+}
+
+// waitAndCleanupAbortJob waits for job completion and cleans up resources
+func waitAndCleanupAbortJob(k8sClient *k8s.Client, namespace, jobName string, log *logger.Logger) error {
+	restore.PrintWaitingMessage(log, "stackgraph-v2", jobName, namespace)
+	return restore.WaitAndCleanup(k8sClient, namespace, jobName, log, false)
+}
+
+// createAbortJob creates a Kubernetes Job for aborting a partially backfilled restore
+func createAbortJob(k8sClient *k8s.Client, namespace, jobName string, config *config.Config) error {
+	return createSimpleJob(k8sClient, namespace, jobName, "abort", abortScript, config)
+}

cmd/stackgraphv2/backfill.go

@@ -0,0 +1,75 @@
+package stackgraphv2
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/stackvista/stackstate-backup-cli/cmd/cmdutils"
+	"github.com/stackvista/stackstate-backup-cli/internal/app"
+	"github.com/stackvista/stackstate-backup-cli/internal/clients/k8s"
+	"github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
+	"github.com/stackvista/stackstate-backup-cli/internal/foundation/logger"
+	"github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore"
+)
+
+const (
+	backfillNameTemplate = "stackgraph-v2-backfill"
+	backfillScript       = "/backup-restore-scripts/restore-stackgraph-backup-v2-backfill.sh"
+)
+
+func backfillCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "backfill",
+		Short: "Complete an interrupted restore by backfilling old data",
+		Long: "Complete an interrupted restore by backfilling old data. This process can run while the system is already up and running." +
+			"After the backfill is done, the restore process is complete. " +
+			"During a normal restore the restore command will take care of backfilling the data, but if that gets interrupted (Ctrl-C) " +
+			"or somehow crashes due to cluster instability, this command allows retrying the backfill portion of the restore.",
+		Run: func(_ *cobra.Command, _ []string) {
+			cmdutils.Run(globalFlags, runBackfill, cmdutils.StorageIsRequired)
+		},
+	}
+
+	return cmd
+}
+
+func runBackfill(appCtx *app.Context) error {
+	// Setup Kubernetes resources for restore job
+	appCtx.Logger.Println()
+	if err := restore.EnsureResources(appCtx.K8sClient, appCtx.Namespace, appCtx.Config, appCtx.Logger); err != nil {
+		return err
+	}
+
+	// Create restore job
+	appCtx.Logger.Println()
+	appCtx.Logger.Infof("Creating backfill job.")
+
+	jobName := fmt.Sprintf("%s-%s", backfillNameTemplate, time.Now().Format("20060102t150405"))
+
+	if err := createBackfillJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Config); err != nil {
+		return fmt.Errorf("failed to create backfill job: %w", err)
+	}
+
+	appCtx.Logger.Successf("Backfill job created: %s", jobName)
+
+	err := waitAndCleanupBackfillJob(appCtx.K8sClient, appCtx.Namespace, jobName, appCtx.Logger)
+
+	if err != nil {
+		logAfterJobResult(appCtx.Logger, checkJobName, false)
+		return err
+	}
+
+	return nil
+}
+
+// waitAndCleanupBackfillJob waits for job completion and cleans up resources
+func waitAndCleanupBackfillJob(k8sClient *k8s.Client, namespace, jobName string, log *logger.Logger) error {
+	restore.PrintWaitingMessage(log, "stackgraph-v2", jobName, namespace)
+	return restore.WaitAndCleanup(k8sClient, namespace, jobName, log, false)
+}
+
+// createBackfillJob creates a Kubernetes Job for backfilling old data during a restore
+func createBackfillJob(k8sClient *k8s.Client, namespace, jobName string, config *config.Config) error {
+	return createSimpleJob(k8sClient, namespace, jobName, "backfill", backfillScript, config)
+}

cmd/stackgraphv2/check_and_finalize.go

@@ -0,0 +1,60 @@
+package stackgraphv2
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/stackvista/stackstate-backup-cli/cmd/cmdutils"
+	"github.com/stackvista/stackstate-backup-cli/internal/app"
+	"github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
+	"github.com/stackvista/stackstate-backup-cli/internal/orchestration/restore"
+	"github.com/stackvista/stackstate-backup-cli/internal/orchestration/scale"
+)
+
+// Check and finalize command flags
+var (
+	checkJobName string
+	waitForJob   bool
+)
+
+func checkAndFinalizeCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "check-and-finalize",
+		Short: "Check and finalize a Stackgraph restore (v2) job",
+		Long: `Check the status of a background Stackgraph restore job and clean up resources.
+
+This command is useful when a restore/backfill/abort job was interrupted (Ctrl+C).
+It will check the job status, print logs if it failed, and clean up the job and PVC resources.
+
+Examples:
+  # Check job status without waiting
+  sts-backup stackgraph-v2 check-and-finalize --job stackgraph-v2-restore-20250128t143000 -n my-namespace
+
+  # Wait for job completion and cleanup
+  sts-backup stackgraph-v2 check-and-finalize --job stackgraph-v2-restore-20250128t143000 --wait -n my-namespace`,
+		Run: func(_ *cobra.Command, _ []string) {
+			cmdutils.Run(globalFlags, runCheckAndFinalize, cmdutils.StorageIsRequired)
+		},
+	}
+
+	cmd.Flags().StringVarP(&checkJobName, "job", "j", "", "Stackgraph restore job name (required)")
+	cmd.Flags().BoolVarP(&waitForJob, "wait", "w", false, "Wait for job to complete before cleanup")
+	_ = cmd.MarkFlagRequired("job")
+
+	return cmd
+}
+
+func runCheckAndFinalize(appCtx *app.Context) error {
+	err := restore.CheckAndFinalize(restore.CheckAndFinalizeParams{
+		K8sClient:     appCtx.K8sClient,
+		Namespace:     appCtx.Namespace,
+		JobName:       checkJobName,
+		ServiceName:   "stackgraph-v2",
+		ScaleUpFn:     scale.ScaleUpAndReleaseLock,
+		ScaleDownFn:   scale.ScaleDown,
+		ScaleSelector: appCtx.Config.Stackgraph.Restore.ScaleDownLabelSelector,
+		CleanupPVC:    true,
+		WaitForJob:    waitForJob,
+		Log:           appCtx.Logger,
+	})
+	logAfterJobResult(appCtx.Logger, checkJobName, err == nil)
+	return err
+}

cmd/stackgraphv2/list.go

@@ -0,0 +1,102 @@
+package stackgraphv2
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/spf13/cobra"
+	"github.com/stackvista/stackstate-backup-cli/cmd/cmdutils"
+	"github.com/stackvista/stackstate-backup-cli/internal/app"
+	s3client "github.com/stackvista/stackstate-backup-cli/internal/clients/s3"
+	"github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
+	"github.com/stackvista/stackstate-backup-cli/internal/foundation/output"
+	"github.com/stackvista/stackstate-backup-cli/internal/orchestration/portforward"
+)
+
+const (
+	backupFileNameRegex = `^sts-backup-.*\.graph.v2$`
+)
+
+func listCmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
+	return &cobra.Command{
+		Use:   "list",
+		Short: "List available Stackgraph backups (v2) from S3",
+		Run: func(_ *cobra.Command, _ []string) {
+			cmdutils.Run(globalFlags, runList, cmdutils.StorageIsRequired)
+		},
+	}
+}
+
+func runList(appCtx *app.Context) error {
+	// Setup port-forward to S3-compatible storage
+	storageService := appCtx.Config.GetStorageService()
+	serviceName := storageService.Name
+	remotePort := storageService.Port
+
+	pf, err := portforward.SetupPortForward(appCtx.K8sClient, appCtx.Namespace, serviceName, remotePort, appCtx.Logger)
+	if err != nil {
+		return err
+	}
+	defer close(pf.StopChan)
+
+	// Create S3 client with actual port
+	s3Client, err := appCtx.NewS3Client(pf.LocalPort)
+	if err != nil {
+		return fmt.Errorf("failed to create S3 client: %w", err)
+	}
+
+	// List objects in bucket
+	bucket := appCtx.Config.Stackgraph.Bucket
+	prefix := appCtx.Config.Stackgraph.S3Prefix + "v2/"
+
+	appCtx.Logger.Infof("Listing Stackgraph backups in bucket '%s' with prefix '%s'...", bucket, prefix)
+
+	input := &s3.ListObjectsV2Input{
+		Bucket:    aws.String(bucket),
+		Prefix:    aws.String(prefix),
+		Delimiter: aws.String("/"),
+	}
+
+	result, err := s3Client.ListObjectsV2(context.Background(), input)
+	if err != nil {
+		return fmt.Errorf("failed to list S3 objects: %w", err)
+	}
+
+	filteredObjects := s3client.ConvertBackupObjects(result.Contents)
+
+	// Filter to only include direct children of the prefix that match the backup filename pattern,
+	// and strip the prefix from the key
+	filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, prefix, backupFileNameRegex)
+	if err != nil {
+		return fmt.Errorf("failed to filter objects: %w", err)
+	}
+
+	// Sort by LastModified time (most recent first)
+	sort.Slice(filteredObjects, func(i, j int) bool {
+		return filteredObjects[i].LastModified.After(filteredObjects[j].LastModified)
+	})
+
+	if len(filteredObjects) == 0 {
+		appCtx.Formatter.PrintMessage("No backups found")
+		return nil
+	}
+
+	table := output.Table{
+		Headers: []string{"NAME", "LAST MODIFIED"},
+		Rows:    make([][]string, 0, len(filteredObjects)),
+	}
+
+	for _, obj := range filteredObjects {
+		row := []string{
+			strings.TrimPrefix(obj.Key, prefix),
+			obj.LastModified.Format("2006-01-02 15:04:05 MST"),
+		}
+		table.Rows = append(table.Rows, row)
+	}
+
+	return appCtx.Formatter.PrintTable(table)
+}

cmd/stackgraphv2/logging.go

@@ -0,0 +1,42 @@
+package stackgraphv2
+
+import (
+	"strings"
+
+	"github.com/stackvista/stackstate-backup-cli/internal/foundation/logger"
+)
+
+func logAfterJobResult(log *logger.Logger, jobName string, success bool) {
+	switch {
+	case strings.HasPrefix(jobName, restoreNameTemplate):
+		if success {
+			log.Println()
+			log.Infof("Job '%s' has successfully restored the live data. The system is running again.", jobName)
+			log.Infof("Now run `sts-backup stackgraph-v2 backfill` to load the historic data.")
+		} else {
+			log.Println()
+			log.Infof("Job '%s' has failed to restore the live data. The system was brought back up but the data was not restored.", jobName)
+			log.Infof("Rerun your `sts-backup stackgraph-v2 restore ...` command toretry restoring data.")
+		}
+	case strings.HasPrefix(jobName, backfillNameTemplate):
+		if success {
+			log.Println()
+			log.Infof("Job '%s' has successfully backfilled the historic data. The restore is complete.", jobName)
+		} else {
+			log.Println()
+			log.Infof("Job '%s' has failed to backfill all historic data.", jobName)
+			log.Infof("Run `sts-backup stackgraph-v2 backfill` to retry restoring old data, or")
+			log.Infof("run `sts-backup stackgraph-v2 abort` to leave the restored data as-is and continue with")
+			log.Infof("the data currently in the system.")
+		}
+	case strings.HasPrefix(jobName, abortNameTemplate):
+		if success {
+			log.Println()
+			log.Infof("Job '%s' has successfully aborted. Historic data is incomplete but the system is functional.", jobName)
+		} else {
+			log.Println()
+			log.Infof("Job '%s' has failed to abort the restore.", jobName)
+			log.Infof("Rerun `sts-backup stackgraph-v2 abort` again to try again.")
+		}
+	}
+}