fix(cli): retry transient mounted-drive reads (ENG-4049)

cli/apply.go

@@ -27,6 +27,7 @@ type ResourceOperationResult struct {
 	ErrorMsg       string
 	CallbackSecret string
 	MetadataURL    string
+	cause          error
 }
 
 type ApplyResult struct {
@@ -35,6 +36,20 @@ type ApplyResult struct {
 	Result ResourceOperationResult
 }
 
+func summarizeApplyFailures(results []ApplyResult) (hasFailures, allFailuresExpected bool) {
+	allFailuresExpected = true
+	for _, result := range results {
+		if result.Result.Status != "failed" {
+			continue
+		}
+		hasFailures = true
+		if !core.IsExpectedCLIError(result.Result.cause) {
+			allFailuresExpected = false
+		}
+	}
+	return hasFailures, allFailuresExpected
+}
+
 // ApplyOption defines a function type for apply options
 type ApplyOption func(*applyOptions)
 
@@ -155,22 +170,21 @@ via -e flag for .env files or -s flag for command-line secrets.`,
 				core.ExitWithError(err)
 			}
 
-			// Check if any resources failed
-			hasFailures := false
-			for _, result := range applyResults {
-				if result.Result.Status == "failed" {
-					hasFailures = true
-					break
-				}
-			}
+			// Check if any resources failed without allowing one unexpected
+			// failure to be hidden by expected failures in the same manifest.
+			hasFailures, allFailuresExpected := summarizeApplyFailures(applyResults)
 
 			outputFmt := core.GetOutputFormat()
 			if outputFmt == "json" || outputFmt == "yaml" {
 				printApplyStructuredOutput(applyResults, outputFmt, !hasFailures)
 			}
 
 			if hasFailures {
-				core.ExitWithError(fmt.Errorf("one or more resources failed to apply"))
+				err := fmt.Errorf("one or more resources failed to apply")
+				if allFailuresExpected {
+					err = core.MarkExpectedError(err, core.CLIErrorOperational)
+				}
+				core.ExitWithError(err)
 			}
 		},
 	}
@@ -212,6 +226,10 @@ func ApplyResources(results []core.Result) ([]ApplyResult, error) {
 							Result: ResourceOperationResult{
 								Status:   "failed",
 								ErrorMsg: fmt.Sprintf("metadata.%s is required", resource.ParentField),
+								cause: core.MarkExpectedError(
+									fmt.Errorf("metadata.%s is required", resource.ParentField),
+									core.CLIErrorValidation,
+								),
 							},
 						})
 						continue
@@ -697,12 +715,14 @@ func PostThenPutFn(resource *core.Resource, resourceName string, name string, re
 		return &ResourceOperationResult{
 			Status:   "failed",
 			ErrorMsg: errorMsg,
+			cause:    err,
 		}
 	}
 	if opResult == nil {
 		return &ResourceOperationResult{
 			Status:   "failed",
 			ErrorMsg: "operation returned no result",
+			cause:    fmt.Errorf("operation returned no result"),
 		}
 	}
 
@@ -756,10 +776,15 @@ func PutFn(resource *core.Resource, resourceName string, name string, resourceOb
 		return &ResourceOperationResult{
 			Status:   "failed",
 			ErrorMsg: errorMsg,
+			cause:    err,
 		}
 	}
 	if opResult == nil {
-		return nil
+		return &ResourceOperationResult{
+			Status:   "failed",
+			ErrorMsg: "operation returned no result",
+			cause:    fmt.Errorf("operation returned no result"),
+		}
 	}
 
 	result := ResourceOperationResult{
@@ -773,15 +798,16 @@ func PutFn(resource *core.Resource, resourceName string, name string, resourceOb
 		result.CallbackSecret = extractCallbackSecret(opResult.Response)
 	}
 
-	if resourceName == "Preview" {
+	switch resourceName {
+	case "Preview":
 		printPreviewURL(opResult.Response, resourceName, name, "configured")
-	} else if resourceName == "PreviewToken" {
+	case "PreviewToken":
 		if tokenURL := buildPreviewTokenURL(opResult.Response, parentName, metadata); tokenURL != "" {
 			core.Print(fmt.Sprintf("Resource %s:%s configured url=%s\n", resourceName, name, tokenURL))
 		} else {
 			core.Print(fmt.Sprintf("Resource %s:%s configured\n", resourceName, name))
 		}
-	} else {
+	default:
 		core.Print(fmt.Sprintf("Resource %s:%s configured\n", resourceName, name))
 	}
 
@@ -797,12 +823,14 @@ func PostFn(resource *core.Resource, resourceName string, name string, resourceO
 		return &ResourceOperationResult{
 			Status:   "failed",
 			ErrorMsg: errorMsg,
+			cause:    err,
 		}
 	}
 	if opResult == nil {
 		return &ResourceOperationResult{
 			Status:   "failed",
 			ErrorMsg: "operation returned no result",
+			cause:    fmt.Errorf("operation returned no result"),
 		}
 	}
 
@@ -817,15 +845,16 @@ func PostFn(resource *core.Resource, resourceName string, name string, resourceO
 		result.CallbackSecret = extractCallbackSecret(opResult.Response)
 	}
 
-	if resourceName == "Preview" {
+	switch resourceName {
+	case "Preview":
 		printPreviewURL(opResult.Response, resourceName, name, "created")
-	} else if resourceName == "PreviewToken" {
+	case "PreviewToken":
 		if tokenURL := buildPreviewTokenURL(opResult.Response, parentName, metadata); tokenURL != "" {
 			core.Print(fmt.Sprintf("Resource %s:%s created url=%s\n", resourceName, name, tokenURL))
 		} else {
 			core.Print(fmt.Sprintf("Resource %s:%s created\n", resourceName, name))
 		}
-	} else {
+	default:
 		core.Print(fmt.Sprintf("Resource %s:%s created\n", resourceName, name))
 	}
 

cli/auth/client_credentials.go

@@ -15,7 +15,7 @@ func LoginClientCredentials(workspace string, clientCredentials string) {
 
 	err := validateWorkspace(workspace, creds)
 	if err != nil {
-		err = fmt.Errorf("failed to access workspace '%s': %s", workspace, err)
+		err = fmt.Errorf("failed to access workspace '%s': %w", workspace, err)
 		core.PrintError("Login", err)
 		core.ExitWithError(err)
 	}

cli/auth/device.go

@@ -133,7 +133,10 @@ func deviceModeLoginFinalize(deviceCode string, workspace string, retries int) {
 			deviceModeLoginFinalize(deviceCode, workspace, retries-1)
 			return
 		} else {
-			err := fmt.Errorf("login timed out waiting for confirmation")
+			err := core.MarkExpectedError(
+				fmt.Errorf("login timed out waiting for confirmation"),
+				core.CLIErrorOperational,
+			)
 			core.PrintError("Login", err)
 			core.ExitWithError(err)
 		}
@@ -149,7 +152,10 @@ func deviceModeLoginFinalize(deviceCode string, workspace string, retries int) {
 					deviceModeLoginFinalize(deviceCode, workspace, retries-1)
 					return
 				} else {
-					err := fmt.Errorf("login timed out waiting for confirmation")
+					err := core.MarkExpectedError(
+						fmt.Errorf("login timed out waiting for confirmation"),
+						core.CLIErrorOperational,
+					)
 					core.PrintError("Login", err)
 					core.ExitWithError(err)
 				}
@@ -164,7 +170,10 @@ func deviceModeLoginFinalize(deviceCode string, workspace string, retries int) {
 
 	if res.StatusCode != http.StatusOK {
 		// This is a real error, not just pending
-		err := fmt.Errorf("authentication failed with status %d: %s", res.StatusCode, string(body))
+		err := core.MarkExpectedError(
+			fmt.Errorf("authentication failed with status %d: %s", res.StatusCode, string(body)),
+			core.CLIErrorAuthentication,
+		)
 		core.PrintError("Login", err)
 		core.ExitWithError(err)
 	}
@@ -186,7 +195,10 @@ func deviceModeLoginFinalize(deviceCode string, workspace string, retries int) {
 			return
 		}
 		if len(workspaces) == 0 {
-			err := fmt.Errorf("no workspaces are available for your account.\nVisit %s to create one", blaxel.GetAppURL())
+			err := core.MarkExpectedError(
+				fmt.Errorf("no workspaces are available for your account.\nVisit %s to create one", blaxel.GetAppURL()),
+				core.CLIErrorOperational,
+			)
 			core.PrintError("Login", err)
 			core.ExitWithError(err)
 			return

cli/auth/utils.go

@@ -8,6 +8,17 @@ import (
 	"github.com/blaxel-ai/sdk-go/option"
 )
 
+type workspaceValidationError struct {
+	workspace string
+	cause     error
+}
+
+func (e *workspaceValidationError) Error() string {
+	return fmt.Sprintf("permission denied for workspace %q", e.workspace)
+}
+
+func (e *workspaceValidationError) Unwrap() error { return e.cause }
+
 // WorkspaceClient interface for workspace lookups (allows mocking)
 type WorkspaceClient interface {
 	Get(ctx context.Context, workspaceName string, opts ...option.RequestOption) (*blaxel.Workspace, error)
@@ -72,8 +83,9 @@ func validateWorkspaceWithFactory(workspace string, credentials blaxel.Credentia
 	// before the workspace is persisted as the current context.
 	if workspace != "" {
 		if _, err := client.Get(context.Background(), workspace); err != nil {
-			// Use one message for every explicit workspace validation failure.
-			return fmt.Errorf("permission denied for workspace %q", workspace)
+			// Keep the stable, non-sensitive user message while preserving the
+			// concrete cause for typed telemetry classification.
+			return &workspaceValidationError{workspace: workspace, cause: err}
 		}
 		return nil
 	}

cli/auth/utils_integration_test.go

@@ -7,6 +7,7 @@ import (
 
 	blaxel "github.com/blaxel-ai/sdk-go"
 	"github.com/blaxel-ai/sdk-go/option"
+	"github.com/blaxel-ai/toolkit/cli/core"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -134,6 +135,16 @@ func TestValidateWorkspaceError(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, "permission denied for workspace \"test-workspace\"", err.Error())
 	assert.NotContains(t, err.Error(), "API error")
+	assert.False(t, core.IsExpectedCLIError(err), "an untyped client failure must remain reportable")
+}
+
+func TestValidateWorkspaceTypedAuthenticationErrorIsExpected(t *testing.T) {
+	factory := mockClientFactory(nil, &blaxel.Error{StatusCode: 403})
+
+	err := validateWorkspaceWithFactory("test-workspace", blaxel.Credentials{APIKey: "key"}, factory)
+	require.Error(t, err)
+	assert.Equal(t, "permission denied for workspace \"test-workspace\"", err.Error())
+	assert.True(t, core.IsExpectedCLIError(err))
 }
 
 // TestValidateWorkspaceMissingWorkspace tests explicit workspace validation failure wording.

cli/cli_test.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -13,6 +14,35 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestSummarizeApplyFailuresDoesNotHideUnexpectedFailure(t *testing.T) {
+	expected := core.MarkExpectedError(errors.New("invalid manifest"), core.CLIErrorValidation)
+	unexpected := errors.New("internal apply invariant failed")
+
+	tests := []struct {
+		name                string
+		results             []ApplyResult
+		hasFailures         bool
+		allFailuresExpected bool
+	}{
+		{name: "success", results: []ApplyResult{{Result: ResourceOperationResult{Status: "created"}}}, allFailuresExpected: true},
+		{name: "expected only", results: []ApplyResult{{Result: ResourceOperationResult{Status: "failed", cause: expected}}}, hasFailures: true, allFailuresExpected: true},
+		{name: "unexpected only", results: []ApplyResult{{Result: ResourceOperationResult{Status: "failed", cause: unexpected}}}, hasFailures: true, allFailuresExpected: false},
+		{name: "mixed", results: []ApplyResult{
+			{Result: ResourceOperationResult{Status: "failed", cause: expected}},
+			{Result: ResourceOperationResult{Status: "failed", cause: unexpected}},
+		}, hasFailures: true, allFailuresExpected: false},
+		{name: "missing cause", results: []ApplyResult{{Result: ResourceOperationResult{Status: "failed"}}}, hasFailures: true, allFailuresExpected: false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			hasFailures, allFailuresExpected := summarizeApplyFailures(test.results)
+			assert.Equal(t, test.hasFailures, hasFailures)
+			assert.Equal(t, test.allFailuresExpected, allFailuresExpected)
+		})
+	}
+}
+
 func TestGetCmd(t *testing.T) {
 	cmd := GetCmd()
 

cli/connect.go

@@ -58,7 +58,10 @@ Examples:
 
 			// Check if stdin is a terminal
 			if !term.IsTerminal(int(os.Stdin.Fd())) {
-				err := fmt.Errorf("this command requires an interactive terminal")
+				err := core.MarkExpectedError(
+					fmt.Errorf("this command requires an interactive terminal"),
+					core.CLIErrorUsage,
+				)
 				core.PrintError("Connect", err)
 				core.ExitWithError(err)
 			}
@@ -67,15 +70,21 @@ Examples:
 			currentContext, _ := blaxel.CurrentContext()
 			workspace := currentContext.Workspace
 			if workspace == "" {
-				err := fmt.Errorf("no workspace found in current context. Please run 'bl login' first")
+				err := core.MarkExpectedError(
+					fmt.Errorf("no workspace found in current context. Please run 'bl login' first"),
+					core.CLIErrorAuthentication,
+				)
 				core.PrintError("Connect", err)
 				core.ExitWithError(err)
 			}
 
 			// Load credentials
 			credentials, _ := blaxel.LoadCredentials(workspace)
 			if !credentials.IsValid() {
-				err := fmt.Errorf("no valid credentials found. Please run 'bl login' first")
+				err := core.MarkExpectedError(
+					fmt.Errorf("no valid credentials found. Please run 'bl login' first"),
+					core.CLIErrorAuthentication,
+				)
 				core.PrintError("Connect", err)
 				core.ExitWithError(err)
 			}
@@ -86,7 +95,10 @@ Examples:
 				token = credentials.APIKey
 			}
 			if token == "" {
-				err := fmt.Errorf("no access token or Blaxel API key found. Please run 'bl login' first")
+				err := core.MarkExpectedError(
+					fmt.Errorf("no access token or Blaxel API key found. Please run 'bl login' first"),
+					core.CLIErrorAuthentication,
+				)
 				core.PrintError("Connect", err)
 				core.ExitWithError(err)
 			}
@@ -97,7 +109,10 @@ Examples:
 			if err != nil {
 				var apiErr *blaxel.Error
 				if isBlaxelError(err, &apiErr) && apiErr.StatusCode == 404 {
-					err := fmt.Errorf("sandbox '%s' not found", sandboxName)
+					err := core.MarkExpectedError(
+						fmt.Errorf("sandbox '%s' not found", sandboxName),
+						core.CLIErrorNotFound,
+					)
 					core.PrintError("Connect", err)
 
 					// List available sandboxes

cli/core/auth_source.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"strings"
@@ -73,7 +74,8 @@ func IsAuthError(err error) bool {
 		return false
 	}
 	// Try the SDK concrete type first.
-	if e, ok := err.(*blaxel.Error); ok {
+	var e *blaxel.Error
+	if errors.As(err, &e) {
 		return e.StatusCode == 401 || e.StatusCode == 403
 	}
 	msg := strings.ToLower(err.Error())