diff --git a/README.md b/README.md index 424440b1..e441738b 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ The JS and Python versions are kept in lock-step by `release-please` and guarded ## Extensions Catalog -This repository contains **2 marketplace(s)** with **64 extensions** (54 skills, 10 plugins). +This repository contains **2 marketplace(s)** with **65 extensions** (55 skills, 10 plugins). ### large-codebase @@ -108,7 +108,7 @@ OpenHands skills for interacting, improving, and refactoring large codebases Official skills and plugins for OpenHands — the open-source AI software engineer. -**60 extensions** (52 skills, 8 plugins) +**61 extensions** (53 skills, 8 plugins) | Name | Type | Description | Commands | |------|------|-------------|----------| @@ -152,6 +152,7 @@ Official skills and plugins for OpenHands — the open-source AI software engine | openhands | plugin | Unified OpenHands plugin — bundles Cloud CLI, REST API (openhands-api), and Automations (openhands-automation) into a... | `/openhands-cloud` | | openhands-api | skill | Use the OpenHands Cloud REST API (V1) and agent-server APIs to create and manage Cloud or local backend conversations... | — | | openhands-automation | skill | Create and manage OpenHands automations - scheduled tasks that run in sandboxes. Use the prompt preset to create auto... | `/automation:create` | +| openhands-enterprise-troubleshooting | skill | Diagnose and resolve common issues on OpenHands Enterprise self-hosted installations. Covers sandbox startup failures... | — | | openhands-sdk | skill | Reference skill for the OpenHands Software Agent SDK - build AI agents with custom tools, LLM configuration, conversa... | `/sdk` | | pdflatex | skill | Install and use pdflatex to compile LaTeX documents into PDFs on Linux. Use when generating academic papers, research... | — | | plain-english-content | skill | Write and edit clear, accessible prose in a plain English content style: active voice, front-loaded content, sentence... | — | diff --git a/marketplaces/openhands-extensions.json b/marketplaces/openhands-extensions.json index dfed7887..34ef7f4d 100644 --- a/marketplaces/openhands-extensions.json +++ b/marketplaces/openhands-extensions.json @@ -141,7 +141,7 @@ { "name": "bitbucket-cloud", "source": "./skills/bitbucket-cloud", - "description": "Bitbucket Cloud (bitbucket.org) specifics — authenticate with BITBUCKET_TOKEN, use the REST API v2, workspace/repo_slug repositories, and the create_bitbucket_pr tool. Loaded on demand by the bitbucket skill once a Cloud environment is detected.", + "description": "Bitbucket Cloud (bitbucket.org) specifics \u2014 authenticate with BITBUCKET_TOKEN, use the REST API v2, workspace/repo_slug repositories, and the create_bitbucket_pr tool. Loaded on demand by the bitbucket skill once a Cloud environment is detected.", "category": "code-hosting", "keywords": [ "bitbucket", @@ -153,7 +153,7 @@ { "name": "bitbucket-data-center", "source": "./skills/bitbucket-data-center", - "description": "Bitbucket Data Center (self-hosted Bitbucket Server) specifics — authenticate with BITBUCKET_DATA_CENTER_TOKEN, use the REST API 1.0, PROJECT/repo_slug repositories, scm/ git remotes, and the create_bitbucket_data_center_pr tool. Loaded on demand by the bitbucket skill once a Data Center environment is detected.", + "description": "Bitbucket Data Center (self-hosted Bitbucket Server) specifics \u2014 authenticate with BITBUCKET_DATA_CENTER_TOKEN, use the REST API 1.0, PROJECT/repo_slug repositories, scm/ git remotes, and the create_bitbucket_data_center_pr tool. Loaded on demand by the bitbucket skill once a Data Center environment is detected.", "category": "code-hosting", "keywords": [ "bitbucket", @@ -770,6 +770,21 @@ "jira", "atlassian" ] + }, + { + "name": "openhands-enterprise-troubleshooting", + "source": "./skills/openhands-enterprise-troubleshooting", + "description": "Diagnose and resolve common issues on OpenHands Enterprise self-hosted installations. Covers sandbox startup failures, auth issues, certificate errors, LLM connectivity, Keycloak login, Replicated Admin Console access, upgrade failures, and resource exhaustion. Includes support bundle generation and escalation handoffs.", + "category": "integrations", + "keywords": [ + "openhands", + "enterprise", + "troubleshooting", + "debugging", + "replicated", + "support", + "diagnostics" + ] } ] -} +} \ No newline at end of file diff --git a/skills/openhands-enterprise-troubleshooting/README.md b/skills/openhands-enterprise-troubleshooting/README.md new file mode 100644 index 00000000..2cb3d71c --- /dev/null +++ b/skills/openhands-enterprise-troubleshooting/README.md @@ -0,0 +1,61 @@ +# OpenHands Enterprise Troubleshooting + +An agent-runnable skill for diagnosing and resolving common issues on **OpenHands Enterprise (OHE)** - self-hosted installations using Replicated on VM-based infrastructure. + +## What This Skill Does + +### 1. Triage and Diagnosis +- Detects failure modes from symptoms or log output +- Checks common problem areas: sandbox startup, auth, certificates, LLM connectivity, Keycloak, Replicated Admin Console, upgrades, resource exhaustion +- Runs targeted diagnostic commands against the live environment + +### 2. Guided Recovery +- Walks through resolution steps for identified issues +- Validates each step before proceeding +- Covers the most common failures seen across real OHE installations + +### 3. Support Bundle Generation +- Guides customers through generating and sending support bundles +- Parses and summarizes bundle output to highlight likely root cause +- Reduces back-and-forth with the platform team + +### 4. Escalation Handoff +- Produces a clear summary when issues cannot be resolved +- Documents what was tried, what logs show, and likely root cause +- Ready to paste into a support ticket + +## Common Issues Covered + +- Sandbox fails to start / 120s timeout +- Git provider auth broken (GitHub App, GitLab token) +- Certificate errors (self-signed, expired, chain issues) +- LLM connectivity failures (endpoint unreachable, bad credentials) +- Keycloak login issues +- Replicated Admin Console unreachable +- Upgrade stuck or failed +- OOM / resource exhaustion on the VM + +## Usage + +This skill is automatically triggered when users describe OHE issues such as: +- "OpenHands is not working" +- "Sandbox failed to start" +- "Can't access admin console" +- "Certificate error" +- "LLM connection failed" +- "Upgrade failed" + +## Files + +- `SKILL.md` - Main skill with diagnostic workflow and quick reference +- `references/diagnostics.md` - Detailed diagnostic commands and log interpretation for each failure mode + +## For Contributors + +When new failure modes are discovered in the field, update `references/diagnostics.md` with: +1. New symptoms and error patterns +2. Diagnostic commands to run +3. Resolution steps that worked +4. Log excerpts showing the error + +This skill should grow with each support issue resolved. diff --git a/skills/openhands-enterprise-troubleshooting/SKILL.md b/skills/openhands-enterprise-troubleshooting/SKILL.md new file mode 100644 index 00000000..d2d3053b --- /dev/null +++ b/skills/openhands-enterprise-troubleshooting/SKILL.md @@ -0,0 +1,217 @@ +--- +name: openhands-enterprise-troubleshooting +description: This skill should be used when a user reports an issue with OpenHands Enterprise (OHE) on a self-hosted (Replicated VM-based) installation. Use for diagnosing sandbox startup failures, auth issues, certificate errors, LLM connectivity problems, Keycloak login issues, Replicated Admin Console access, upgrade failures, or resource exhaustion. Helps triage symptoms, run diagnostic commands, guide through recovery steps, generate support bundles, and produce escalation handoffs. +triggers: +- openhands enterprise +- OHE troubleshooting +- openhands not working +- sandbox failed +- replicated admin console +- keycloak login +- certificate error +- LLM connectivity +- upgrade failed +- support bundle +- openhands install +--- + +# OpenHands Enterprise Troubleshooting + +This skill helps diagnose and resolve common issues on OpenHands Enterprise (OHE) self-hosted installations using Replicated. It covers triage, guided recovery, support bundle generation, and escalation handoffs. + +## Diagnostic Workflow + +When a user reports an OHE issue: + +1. **Collect symptoms** - Ask user to describe what they see, error messages, when it started +2. **Identify failure mode** - Match symptoms to one of the common issues below +3. **Run targeted diagnostics** - Use commands in `references/diagnostics.md` +4. **Guide recovery** - Follow resolution steps for the identified issue +5. **Verify fix** - Confirm the issue is resolved +6. **Generate handoff** - If unresolved, produce a clear summary for the platform team + +## Common Failure Modes + +### 1. Sandbox Fails to Start / 120s Timeout + +**Symptoms:** +- Conversation hangs then times out +- "Sandbox failed to start" error +- 120-second timeout in logs + +**Diagnosis:** Check sandbox service status, podman/docker runtime, resource availability + +**Reference:** See `references/diagnostics.md` - Section "Sandbox Startup" + +### 2. Git Provider Auth Broken + +**Symptoms:** +- "Authentication failed" for GitHub/GitLab +- Can't clone or push repos +- GitHub App shows as disconnected + +**Diagnosis:** Check gitProvider secrets in kubernetes, GitHub App installation status + +**Reference:** See `references/diagnostics.md` - Section "Git Provider Auth" + +### 3. Certificate Errors + +**Symptoms:** +- "certificate expired" or "self-signed certificate" errors +- TLS handshake failures +- Browser shows insecure connection warning + +**Diagnosis:** Check cert expiry, certificate chain, ingress configuration + +**Reference:** See `references/diagnostics.md` - Section "Certificate Issues" + +### 4. LLM Connectivity Failures + +**Symptoms:** +- "LLM endpoint unreachable" +- "Authentication failed" for LLM API +- Conversations fail to start + +**Diagnosis:** Check LLM endpoint URL, API key secrets, network policies + +**Reference:** See `references/diagnostics.md` - Section "LLM Connectivity" + +### 5. Keycloak Login Issues + +**Symptoms:** +- Can't access admin console +- Login loop or "invalid credentials" +- Keycloak pod showing errors + +**Diagnosis:** Check Keycloak pod status, database connectivity, realm configuration + +**Reference:** See `references/diagnostics.md` - Section "Keycloak" + +### 6. Replicated Admin Console Unreachable + +**Symptoms:** +- Can't access admin console URL +- Connection refused or timeout +- Browser shows "site cannot be reached" + +**Diagnosis:** Check Replicated operator pod, ingress, service endpoints + +**Reference:** See `references/diagnostics.md` - Section "Replicated Admin Console" + +### 7. Upgrade Stuck or Failed + +**Symptoms:** +- Replicated shows upgrade as "failed" +- Pods in crash loop after upgrade +- Migration jobs failing + +**Diagnosis:** Check failed job logs, resource availability, pre-flight failures + +**Reference:** See `references/diagnostics.md` - Section "Upgrade Issues" + +### 8. OOM / Resource Exhaustion + +**Symptoms:** +- Pods being OOMKilled +- "Too many open files" errors +- Services becoming unresponsive + +**Diagnosis:** Check node resources (memory, disk, file descriptors) + +**Reference:** See `references/diagnostics.md` - Section "Resource Exhaustion" + +## Diagnostic Commands Quick Reference + +Access the VM and run these common commands: + +```bash +# Check overall pod status +kubectl get pods -n openhands + +# View pod logs (replace POD_NAME) +kubectl logs -n openhands POD_NAME +kubectl logs -n openhands POD_NAME --previous + +# Describe a pod for events +kubectl describe pod -n openhands POD_NAME + +# Check resource usage +kubectl top nodes +kubectl top pods -n openhands + +# Check certificate expiry +echo | openssl s_client -connect HOST:443 2>/dev/null | openssl x509 -noout -dates + +# Check Replicated operator +kubectl get pods -n replicated +kubectl logs -n replicated -l app=replicated-operator +``` + +## Support Bundle Generation + +When the issue requires deeper investigation, guide the user to generate a support bundle. + +### Generating the Support Bundle + +1. Access the VM via SSH +2. Run the Replicated support bundle command: + +```bash +replicated admin support-bundle --kubecontext=KUBE_CONTEXT --namespace=openhands +``` + +3. The bundle will be saved locally, then upload/share with the platform team + +### Parsing the Support Bundle + +After obtaining a support bundle: + +1. Extract the archive +2. Focus on these key files: + - `pod-status.json` - Current pod states + - `pod-logs/*.log` - Container logs + - `events.json` - Kubernetes events + - `nodes.json` - Node resource info + +3. Look for patterns in `references/diagnostics.md` + +## Escalation Handoff Template + +When an issue cannot be resolved, produce this summary: + +``` +## Issue Summary +**Problem:** [One-line description] +**Duration:** [When it started] +**Impact:** [Who is affected] + +## Symptoms Observed +- [Symptom 1] +- [Symptom 2] + +## Diagnostic Steps Taken +1. [Step 1] +2. [Step 2] + +## Logs / Evidence +``` +[Relevant log excerpts] +``` + +## Resolution Attempts +- [Attempt 1] - [Result] +- [Attempt 2] - [Result] + +## Likely Root Cause +[Analysis] +``` + +## Additional Resources + +- **Diagnostic Reference:** `references/diagnostics.md` - Detailed commands and log interpretation for each failure mode +- **Replicated Docs:** https://docs.replicated.com/vendor/support-bundle-generating +- **OHE Architecture:** Internal docs on OHE components and their relationships + +## Maintenance + +As new failure modes are discovered in the field, add them to this skill. Update `references/diagnostics.md` with new patterns and resolution steps. diff --git a/skills/openhands-enterprise-troubleshooting/references/diagnostics.md b/skills/openhands-enterprise-troubleshooting/references/diagnostics.md new file mode 100644 index 00000000..84b05737 --- /dev/null +++ b/skills/openhands-enterprise-troubleshooting/references/diagnostics.md @@ -0,0 +1,423 @@ +# OHE Diagnostics Reference + +Detailed diagnostic procedures for each OpenHands Enterprise failure mode. Run these commands on the VM via SSH. + +## Sandbox Startup + +### Check Sandbox Pod Status + +```bash +kubectl get pods -n openhands -l app=sandbox --watch +``` + +Look for: `Running` status, multiple restarts, `ImagePullBackOff`, `CrashLoopBackOff` + +### Check Sandbox Logs + +```bash +# Get sandbox pod name +SANDBOX_POD=$(kubectl get pods -n openhands -l app=sandbox -o jsonpath='{.items[0].metadata.name}') + +# View recent logs +kubectl logs -n openhands $SANDBOX_POD --tail=100 + +# View previous log (if pod restarted) +kubectl logs -n openhands $SANDBOX_POD --previous +``` + +### Common Sandbox Startup Errors + +| Error Pattern | Likely Cause | Check | +|---------------|--------------|-------| +| `ImagePullBackOff` | Registry auth, network | `kubectl describe pod` for image pull error | +| `CrashLoopBackOff` | Config error, missing secret | `kubectl logs --previous` | +| `Init:Error` | Init container failed | `kubectl describe pod` for init container status | +| `Timeout` | Resource exhaustion, runtime issue | `kubectl top pods` | + +### Sandbox Runtime Check + +```bash +# Check if container runtime is responsive +kubectl exec -n openhands deploy/sandbox -- crictl info + +# Check sandbox disk space +kubectl exec -n openhands deploy/sandbox -- df -h + +# Check sandbox file descriptors +kubectl exec -n openhands deploy/sandbox -- ls /proc/self/fd | wc -l +``` + +--- + +## Git Provider Auth + +### Check GitHub App Status + +```bash +kubectl get pods -n openhands -l app=github-app + +# Check GitHub App secret exists +kubectl get secret -n openhands -o yaml | grep -i github +``` + +### Check Git Provider Secrets + +```bash +# List git provider secrets +kubectl get secrets -n openhands | grep -i git + +# Check if secret has data +kubectl get secret -n openhands git-provider-secret -o yaml +``` + +### Validate GitHub Token + +```bash +# Get the token from secret (decode base64) +GITHUB_TOKEN=$(kubectl get secret -n openhands git-provider-secret -o jsonpath='{.data.token}' | base64 -d) + +# Test token validity +curl -s -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/app + +# Check GitHub App installation +curl -s -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/app/installations +``` + +### Check GitLab Token + +```bash +# Get GitLab token +GITLAB_TOKEN=$(kubectl get secret -n openhands git-provider-secret -o jsonpath='{.data.gitlab_token}' | base64 -d) + +# Test token validity +curl -s -H "PRIVATE-TOKEN: $GITLAB_TOKEN" "https://gitlab.com/api/v4/user" +``` + +--- + +## Certificate Issues + +### Check Certificate Expiry + +```bash +# For a specific host +HOST="your-openhands-domain.com" +echo | openssl s_client -connect $HOST:443 -servername $HOST 2>/dev/null | openssl x509 -noout -dates + +# Check all certs in kubernetes secret +kubectl get secret -n openhands -l app=ingress-tls -o jsonpath='{.items[*]}' | jq -r '.[].data."tls.crt"' | base64 -d | openssl x509 -noout -dates +``` + +### Check Certificate Chain + +```bash +# Get full certificate chain +echo | openssl s_client -connect $HOST:443 -servername $HOST -showcerts 2>/dev/null + +# Check chain completeness +echo | openssl s_client -connect $HOST:443 -servername $HOST 2>/dev/null | grep -A2 "Certificate chain" +``` + +### Common Certificate Errors + +| Error | Cause | Fix | +|-------|-------|-----| +| `CERT_HAS_EXPIRED` | Certificate expired | Renew certificate | +| `self signed certificate` | Self-signed in chain | Install proper chain | +| `UNABLE_TO_VERIFY_LEAF_SIGNATURE` | Intermediate missing | Ensure full chain in ingress | +| `certificate hostname mismatch` | Wrong CN/SAN | Reissue with correct hostname | + +### Ingress TLS Check + +```bash +kubectl get ingress -n openhands -o yaml | grep -A5 "tls:" +``` + +--- + +## LLM Connectivity + +### Check LLM Configuration + +```bash +# Get LLM config (masked) +kubectl get configmap -n openhands -o jsonpath='{.items[?(@.metadata.name=="llm-config")].data}' | jq . + +# Check LLM secret +kubectl get secret -n openhands -o jsonpath='{.items[?(@.metadata.name=="llm-credentials")].data}' | jq -r 'keys' +``` + +### Test LLM Endpoint + +```bash +# Get LLM endpoint from config +LLM_ENDPOINT=$(kubectl get configmap -n openhands llm-config -o jsonpath='{.data.endpoint}') + +# Get API key +LLM_API_KEY=$(kubectl get secret -n openhands llm-credentials -o jsonpath='{.data.api_key}' | base64 -d) + +# Test connectivity (example for OpenAI-compatible endpoint) +curl -s -X POST $LLM_ENDPOINT/v1/models \ + -H "Authorization: Bearer $LLM_API_KEY" \ + -w "\nHTTP_CODE:%{http_code}" +``` + +### Network Policy Check + +```bash +# Check if pods have network policies +kubectl get networkpolicy -n openhands + +# Test DNS resolution from pod +kubectl exec -n openhands deploy/agent-server -- nslookup api.openai.com +``` + +### Common LLM Errors + +| Error Pattern | Cause | Fix | +|---------------|-------|-----| +| `connection refused` | Wrong endpoint | Verify LLM endpoint URL | +| `401 Unauthorized` | Bad API key | Re-create/rotate API key | +| `403 Forbidden` | Insufficient permissions | Check model access | +| `connection timeout` | Network policy/firewall | Check network policies | + +--- + +## Keycloak + +### Check Keycloak Pods + +```bash +kubectl get pods -n keycloak --watch + +# Check Keycloak logs +KEYCLOAK_POD=$(kubectl get pods -n keycloak -l app=keycloak -o jsonpath='{.items[0].metadata.name}') +kubectl logs -n keycloak $KEYCLOAK_POD --tail=200 +``` + +### Check Keycloak Database Connectivity + +```bash +# Keycloak requires database - check DB pod +kubectl get pods -n keycloak | grep -E "postgres|mysql|database" + +# Check DB connectivity from Keycloak pod +kubectl exec -n keycloak $KEYCLOAK_POD -- bash -c 'nc -zv $DB_HOST $DB_PORT || echo "DB unreachable"' +``` + +### Check Keycloak Realm Configuration + +```bash +# Get Keycloak admin credentials +KEYCLOAK_ADMIN=$(kubectl get secret -n keycloak keycloak-admin -o jsonpath='{.data.username}' | base64 -d) +KEYCLOAK_PASS=$(kubectl get secret -n keycloak keycloak-admin -o jsonpath='{.data.password}' | base64 -d) + +# Get Keycloak URL +KEYCLOAK_URL=$(kubectl get ingress -n keycloak -o jsonpath='{.items[0].spec.rules[0].host}') + +# Test Keycloak admin access +curl -s -o /dev/null -w "%{http_code}" \ + -d "username=$KEYCLOAK_ADMIN" \ + -d "password=$KEYCLOAK_PASS" \ + -d "grant_type=password" \ + "$KEYCLOAK_URL/realms/master/protocol/openid-connect/token" +``` + +### Keycloak Health Check + +```bash +kubectl exec -n keycloak deploy/keycloak -- /opt/keycloak/bin/kc.sh health --metrics +``` + +--- + +## Replicated Admin Console + +### Check Replicated Operator + +```bash +kubectl get pods -n replicated --watch + +# Check operator logs +kubectl logs -n replicated -l app=replicated-operator --tail=100 --follow +``` + +### Check Replicated Services + +```bash +kubectl get svc -n replicated + +# Check if operator service is exposed +kubectl get ingress -n replicated +``` + +### Common Replicated Issues + +| Symptom | Check | Fix | +|---------|-------|-----| +| "Connection refused" on admin console | Operator pod status | Restart operator pod | +| Admin console shows blank page | Operator logs | Check for migration errors | +| Can't run admin commands | `replicated` CLI version | Update replicated CLI | + +### Replicated CLI Diagnostics + +```bash +# SSH to the VM, then: +replicated admin status +replicated admin console logs --since 1h +replicated apps list +``` + +--- + +## Upgrade Issues + +### Check Failed Upgrade Jobs + +```bash +kubectl get jobs -n openhands | grep -E "upgrade|migrate" + +# Check failed job logs +UPGRADE_JOB=$(kubectl get jobs -n openhands -o jsonpath='{.items[?(@.status.failed)].metadata.name}' | awk '{print $1}') +kubectl logs -n openhands job/$UPGRADE_JOB +``` + +### Check Pre-flight Status + +```bash +# Run pre-flight checks manually +replicated admin preflight --kubecontext=KUBE_CONTEXT --namespace=openHands + +# Check pre-flight results +kubectl get configmap -n replicated -o jsonpath='{.items[?(@.metadata.name=="preflight-results")].data}' +``` + +### Rollback Procedure + +```bash +# List available releases +replicated releases --app=APP_NAME + +# Rollback to previous release +replicated release rollback --app=APP_NAME --sequence=PREVIOUS_SEQUENCE +``` + +### Common Upgrade Failures + +| Error | Cause | Fix | +|-------|-------|-----| +| Migration job failed | Database schema change | Check job logs, retry | +| Pods crash on new version | Config incompatibility | Review changelog, adjust config | +| Pre-flight failed | Resource insufficient | Add resources, retry | +| Helm error | Values incompatible | Review helm values diff | + +--- + +## Resource Exhaustion + +### Check Node Resources + +```bash +# Node CPU/memory +kubectl top nodes + +# Node disk usage +kubectl debug node/NODE_NAME -it -- df -h + +# Check if OOMKilled +kubectl get events -n openhands | grep -i "oom\|killed" +``` + +### Check Pod Resource Usage + +```bash +# Per-pod resource usage +kubectl top pods -n openhands + +# Check pod resource limits +kubectl get pods -n openhands -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.containers[*].resources.limits.memory}{"\n"}' +``` + +### Check File Descriptor Usage + +```bash +# Check fd limit on node +cat /proc/sys/fs/file-max +ulimit -n + +# Check pod fd usage +kubectl exec -n openhands deploy/agent-server -- ls /proc/self/fd | wc -l +``` + +### Check Disk Space + +```bash +# Node disk pressure +kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.conditions[?(@.type=="DiskPressure")].status}{"\n"}' + +# Find large directories +kubectl exec -n openhands deploy/agent-server -- du -sh /var/* +``` + +### Common Resource Exhaustion Fixes + +| Resource | Check | Fix | +|----------|-------|-----| +| Memory OOM | `kubectl top pods` | Increase pod memory limits | +| Disk full | `du -sh` | Clean up logs, increase PV size | +| FD exhaustion | `ls /proc/*/fd \| wc` | Increase ulimit | +| CPU throttling | `kubectl top pods` | Adjust CPU limits | + +--- + +## Log Pattern Quick Reference + +### Search for Common Error Patterns + +```bash +# In pod logs, search for these patterns: +grep -E "ERROR|FATAL|Exception|Traceback" /path/to/logs + +# Search for timeout patterns +grep -E "timeout|timed out|deadline" /path/to/logs + +# Search for connection errors +grep -E "connection refused|connection reset|dial tcp" /path/to/logs + +# Search for auth errors +grep -E "unauthorized|forbidden|authentication" /path/to/logs +``` + +### Kubernetes Events + +```bash +# Get recent events in namespace +kubectl get events -n openhands --sort-by='.lastTimestamp' | tail -50 + +# Filter events by type +kubectl get events -n openhands --field-selector type=Warning +``` + +--- + +## Useful One-Liners + +```bash +# Get all pod statuses at once +kubectl get pods -n openhands -o wide + +# Tail logs from all pods with a label +kubectl logs -n openhands -l app=sandbox --tail=50 -f + +# Get pod restart count +kubectl get pods -n openhands -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.containerStatuses[*].restartCount}{"\n"}' + +# Check pod age and status +kubectl get pods -n openhands -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{.metadata.creationTimestamp}{"\n"}' + +# Extract error messages from all pods +for pod in $(kubectl get pods -n openhands -o name); do + echo "=== $pod ==="; + kubectl logs -n openhands $pod --tail=20 2>&1 | grep -iE "error|fatal" | head -5; +done +```