Add EBS VolumeId to disk metrics

.github/workflows/test-metrics.yml

@@ -32,6 +32,7 @@ jobs:
         if: matrix.configuration == 'cw-agent-stopped'
         run: sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop || true
       - uses: ./
+        id: runs_on
         with:
           metrics: cpu,network,memory,disk,io
       - name: Setup
@@ -56,4 +57,73 @@ jobs:
             sleep 10
           done &
 
-          wait
+          wait
+      - name: Verify VolumeId disk metric
+        env:
+          VOLUME_ID: ${{ steps.runs_on.outputs.volume_id }}
+        run: |
+          set -euo pipefail
+
+          if [[ -z "$VOLUME_ID" ]]; then
+            echo "::error::The action did not resolve an EBS VolumeId"
+            exit 1
+          fi
+
+          # Match the action's credential isolation so preconfigured workflow credentials cannot query a different account.
+          unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_PROFILE AWS_DEFAULT_PROFILE
+          unset AWS_WEB_IDENTITY_TOKEN_FILE AWS_ROLE_ARN AWS_ROLE_SESSION_NAME AWS_EC2_METADATA_DISABLED
+
+          metric_queries=$(jq -nc \
+            --arg instance_id "$RUNS_ON_INSTANCE_ID" \
+            --arg volume_id "$VOLUME_ID" \
+            '[{
+              Id: "diskused",
+              MetricStat: {
+                Metric: {
+                  Namespace: "CWAgent",
+                  MetricName: "disk_used_percent",
+                  Dimensions: [
+                    {Name: "InstanceId", Value: $instance_id},
+                    {Name: "VolumeId", Value: $volume_id},
+                    {Name: "fstype", Value: "ext4"},
+                    {Name: "path", Value: "/"}
+                  ]
+                },
+                Period: 10,
+                Stat: "Average"
+              },
+              ReturnData: true
+            }]')
+
+          for attempt in {1..6}; do
+            start_time=$(date -u -d '10 minutes ago' +%Y-%m-%dT%H:%M:%SZ)
+            end_time=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+            if value_count=$(aws cloudwatch get-metric-data \
+              --region "$RUNS_ON_AWS_REGION" \
+              --metric-data-queries "$metric_queries" \
+              --start-time "$start_time" \
+              --end-time "$end_time" \
+              --query 'length(MetricDataResults[0].Values)' \
+              --output text); then
+              if [[ "$value_count" =~ ^[1-9][0-9]*$ ]]; then
+                echo "Verified disk_used_percent for $VOLUME_ID ($value_count datapoints)"
+                exit 0
+              fi
+            fi
+
+            if [[ "$attempt" -lt 6 ]]; then
+              sleep 10
+            fi
+          done
+
+          echo "::error::No VolumeId-qualified disk_used_percent datapoints were published for $VOLUME_ID"
+          while IFS= read -r config_file; do
+            echo "CloudWatch configuration: $config_file"
+            cat "$config_file"
+          done < <(find /tmp "$RUNNER_TEMP" -maxdepth 2 -type f -name 'runs-on-metrics-*.json' 2>/dev/null)
+          aws cloudwatch list-metrics \
+            --region "$RUNS_ON_AWS_REGION" \
+            --namespace CWAgent \
+            --metric-name disk_used_percent \
+            --dimensions Name=InstanceId,Value="$RUNS_ON_INSTANCE_ID" || true
+          exit 1

action.yml

@@ -29,4 +29,8 @@ inputs:
   sccache:
     description: 'Enable sccache. Can take either "s3" (RunsOn S3 cache bucket) or be empty (disabled). You still need to setup sccache in your workflow, for instance with mozilla-actions/sccache-action.'
     required: false
-    default: ''
+    default: ''
+
+outputs:
+  volume_id:
+    description: 'EBS volume ID backing the configured disk metrics, when available'

internal/monitoring/agent.go

@@ -13,6 +13,11 @@ import (
 	"github.com/sethvargo/go-githubactions"
 )
 
+const (
+	sysBlockRoot     = "/sys/block"
+	volumeIDStateKey = "disk_volume_id"
+)
+
 // https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html
 func GenerateCloudWatchConfig(action *githubactions.Action, metrics []string, networkInterface, diskDevice string) error {
 	if len(metrics) == 0 {
@@ -27,11 +32,63 @@ func GenerateCloudWatchConfig(action *githubactions.Action, metrics []string, ne
 	// Get network interface and disk device based on config
 	primaryInterface := getNetworkInterface(networkInterface)
 	rootDisk := getDiskDevice(diskDevice)
+	volumeID := ""
+	for _, metric := range metrics {
+		if !strings.EqualFold(metric, "disk") {
+			continue
+		}
+
+		resolvedVolumeID, err := getEBSVolumeID(sysBlockRoot, rootDisk)
+		if err != nil {
+			action.Warningf("Failed to resolve EBS volume ID for disk device %s: %v", rootDisk, err)
+		} else {
+			volumeID = resolvedVolumeID
+			action.Infof("Using EBS volume ID: %s", volumeID)
+		}
+		break
+	}
 
 	action.Infof("Using network interface: %s", primaryInterface)
 	action.Infof("Using disk device: %s", rootDisk)
 
-	config := CloudWatchConfig{
+	config := buildCloudWatchConfig(metrics, primaryInterface, rootDisk, volumeID)
+
+	// Write config file
+	configFile, err := os.CreateTemp("", "runs-on-metrics-*.json")
+	if err != nil {
+		return fmt.Errorf("failed to create temp file: %w", err)
+	}
+	configPath := configFile.Name()
+	defer configFile.Close()
+
+	configJSON, err := json.MarshalIndent(config, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to marshal config: %w", err)
+	}
+
+	if err := os.WriteFile(configPath, configJSON, 0644); err != nil {
+		return fmt.Errorf("failed to write config file: %w", err)
+	}
+
+	action.Infof("Generated CloudWatch config with metrics: %v", metrics)
+	action.Infof("Config file: %s", configPath)
+	action.Infof("Config content: %s", string(configJSON))
+
+	if err := applyCloudWatchConfig(action, configPath); err != nil {
+		return err
+	}
+
+	if volumeID != "" {
+		action.SetOutput("volume_id", volumeID)
+		// GitHub action setup and post run in separate processes, so persist the exact published dimension.
+		action.SaveState(volumeIDStateKey, volumeID)
+	}
+
+	return nil
+}
+
+func buildCloudWatchConfig(metrics []string, primaryInterface, rootDisk, volumeID string) CloudWatchConfig {
+	cloudWatchConfig := CloudWatchConfig{
 		Metrics: MetricsConfig{
 			Namespace:        NAMESPACE,
 			MetricsCollected: make(map[string]interface{}),
@@ -58,7 +115,7 @@ func GenerateCloudWatchConfig(action *githubactions.Action, metrics []string, ne
 			for _, measurement := range measurements {
 				cpuConfig["measurement"] = append(cpuConfig["measurement"].([]string), measurement.Name)
 			}
-			config.Metrics.MetricsCollected["cpu"] = cpuConfig
+			cloudWatchConfig.Metrics.MetricsCollected["cpu"] = cpuConfig
 		case "network":
 			netConfig := map[string]interface{}{
 				"drop_original_metrics": true,
@@ -68,7 +125,7 @@ func GenerateCloudWatchConfig(action *githubactions.Action, metrics []string, ne
 			for _, measurement := range measurements {
 				netConfig["measurement"] = append(netConfig["measurement"].([]string), measurement.Name)
 			}
-			config.Metrics.MetricsCollected["net"] = netConfig
+			cloudWatchConfig.Metrics.MetricsCollected["net"] = netConfig
 		case "memory":
 			memConfig := map[string]interface{}{
 				"drop_original_metrics": true,
@@ -77,7 +134,7 @@ func GenerateCloudWatchConfig(action *githubactions.Action, metrics []string, ne
 			for _, measurement := range measurements {
 				memConfig["measurement"] = append(memConfig["measurement"].([]string), measurement.Name)
 			}
-			config.Metrics.MetricsCollected["mem"] = memConfig
+			cloudWatchConfig.Metrics.MetricsCollected["mem"] = memConfig
 		case "disk":
 			diskConfig := map[string]interface{}{
 				"drop_original_metrics": true,
@@ -91,7 +148,12 @@ func GenerateCloudWatchConfig(action *githubactions.Action, metrics []string, ne
 			for _, measurement := range measurements {
 				diskConfig["measurement"] = append(diskConfig["measurement"].([]string), measurement.Name)
 			}
-			config.Metrics.MetricsCollected["disk"] = diskConfig
+			if volumeID != "" {
+				diskConfig["append_dimensions"] = map[string]string{
+					"VolumeId": volumeID,
+				}
+			}
+			cloudWatchConfig.Metrics.MetricsCollected["disk"] = diskConfig
 		case "io":
 			diskioConfig := map[string]interface{}{
 				"drop_original_metrics": true,
@@ -101,33 +163,11 @@ func GenerateCloudWatchConfig(action *githubactions.Action, metrics []string, ne
 			for _, measurement := range measurements {
 				diskioConfig["measurement"] = append(diskioConfig["measurement"].([]string), measurement.Name)
 			}
-			config.Metrics.MetricsCollected["diskio"] = diskioConfig
+			cloudWatchConfig.Metrics.MetricsCollected["diskio"] = diskioConfig
 		}
 	}
 
-	// Write config file
-	configFile, err := os.CreateTemp("", "runs-on-metrics-*.json")
-	if err != nil {
-		return fmt.Errorf("failed to create temp file: %w", err)
-	}
-	configPath := configFile.Name()
-	defer configFile.Close()
-
-	configJSON, err := json.MarshalIndent(config, "", "  ")
-	if err != nil {
-		return fmt.Errorf("failed to marshal config: %w", err)
-	}
-
-	if err := os.WriteFile(configPath, configJSON, 0644); err != nil {
-		return fmt.Errorf("failed to write config file: %w", err)
-	}
-
-	action.Infof("Generated CloudWatch config with metrics: %v", metrics)
-	action.Infof("Config file: %s", configPath)
-	action.Infof("Config content: %s", string(configJSON))
-
-	// Apply the config to the CloudWatch agent (start if needed, or append if already running)
-	return applyCloudWatchConfig(action, configPath)
+	return cloudWatchConfig
 }
 
 const agentCtl = "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl"

internal/monitoring/agent_test.go

@@ -0,0 +1,35 @@
+package monitoring
+
+import "testing"
+
+func TestBuildCloudWatchConfigAddsVolumeIDToDiskMetrics(t *testing.T) {
+	t.Parallel()
+
+	config := buildCloudWatchConfig([]string{"disk"}, "ens5", "nvme0n1p1", "vol-0123456789abcdef0")
+	diskConfig := config.Metrics.MetricsCollected["disk"].(map[string]interface{})
+
+	if got := diskConfig["drop_device"]; got != true {
+		t.Fatalf("drop_device = %v, want true", got)
+	}
+	dimensions, ok := diskConfig["append_dimensions"].(map[string]string)
+	if !ok {
+		t.Fatalf("append_dimensions has type %T, want map[string]string", diskConfig["append_dimensions"])
+	}
+	if got := dimensions["VolumeId"]; got != "vol-0123456789abcdef0" {
+		t.Fatalf("VolumeId = %q, want %q", got, "vol-0123456789abcdef0")
+	}
+	if _, exists := dimensions["volume_id"]; exists {
+		t.Fatal("unexpected lowercase volume_id dimension")
+	}
+}
+
+func TestBuildCloudWatchConfigOmitsVolumeIDWhenUnavailable(t *testing.T) {
+	t.Parallel()
+
+	config := buildCloudWatchConfig([]string{"disk"}, "ens5", "nvme0n1p1", "")
+	diskConfig := config.Metrics.MetricsCollected["disk"].(map[string]interface{})
+
+	if _, exists := diskConfig["append_dimensions"]; exists {
+		t.Fatal("append_dimensions should be omitted without a volume ID")
+	}
+}

internal/monitoring/helpers.go

@@ -2,15 +2,23 @@ package monitoring
 
 import (
 	"bufio"
+	"fmt"
 	"math"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"regexp"
 	"strings"
 )
 
 const DEFAULT_NETWORK_INTERFACE = "enp39s0"
 const DEFAULT_DISK_DEVICE = "nvme0n1p1"
 
+var (
+	nvmeDevicePattern  = regexp.MustCompile(`^(nvme\d+n\d+)(?:p\d+)?$`)
+	ebsVolumeIDPattern = regexp.MustCompile(`^vol-?(?:[0-9a-f]{8}|[0-9a-f]{17})$`)
+)
+
 // detectPrimaryNetworkInterface finds the primary network interface (excluding loopback and docker)
 func detectPrimaryNetworkInterface() string {
 	// Try to get the interface used for the default route
@@ -110,6 +118,35 @@ func getDiskDevice(diskDevice string) string {
 	return diskDevice
 }
 
+// getEBSVolumeID resolves the EBS volume backing a Nitro NVMe device.
+func getEBSVolumeID(sysBlockRoot, diskDevice string) (string, error) {
+	// Restrict the user-provided device name before joining it to a privileged sysfs path.
+	if diskDevice == "" || filepath.Base(diskDevice) != diskDevice {
+		return "", fmt.Errorf("invalid disk device %q", diskDevice)
+	}
+
+	matches := nvmeDevicePattern.FindStringSubmatch(diskDevice)
+	if matches == nil {
+		return "", fmt.Errorf("disk device %q is not a supported NVMe device", diskDevice)
+	}
+
+	serialPath := filepath.Join(sysBlockRoot, matches[1], "device", "serial")
+	serialBytes, err := os.ReadFile(serialPath)
+	if err != nil {
+		return "", fmt.Errorf("read EBS volume serial: %w", err)
+	}
+
+	serial := strings.ToLower(strings.Join(strings.Fields(string(serialBytes)), ""))
+	if !ebsVolumeIDPattern.MatchString(serial) {
+		return "", fmt.Errorf("unexpected EBS volume serial %q", serial)
+	}
+	if strings.HasPrefix(serial, "vol-") {
+		return serial, nil
+	}
+
+	return "vol-" + strings.TrimPrefix(serial, "vol"), nil
+}
+
 // calculateStats computes min, max, and average of a slice of floats
 func calculateStats(data []float64) (min, max, avg float64) {
 	data = sanitizeFloatSeries(data)