Feat/picod prometheus metrics

go.mod

@@ -11,6 +11,8 @@ require (
 	github.com/gin-gonic/gin v1.10.0
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/uuid v1.6.0
+	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/redis/go-redis/v9 v9.17.1
 	github.com/stretchr/testify v1.11.1
 	github.com/valkey-io/valkey-go v1.0.69
@@ -75,8 +77,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.1 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect

pkg/picod/execute.go

@@ -53,8 +53,12 @@ type ExecuteResponse struct {
 
 // ExecuteHandler handles command execution requests
 func (s *Server) ExecuteHandler(c *gin.Context) {
+	s.metrics.ActiveExecutions.Inc()
+	defer s.metrics.ActiveExecutions.Dec()
+
 	var req ExecuteRequest
 	if err := c.ShouldBindJSON(&req); err != nil {
+		s.metrics.ExecuteRequestsTotal.WithLabelValues("invalid").Inc()
 		c.JSON(http.StatusBadRequest, gin.H{
 			"error": err.Error(),
 			"code":  http.StatusBadRequest,
@@ -63,6 +67,7 @@ func (s *Server) ExecuteHandler(c *gin.Context) {
 	}
 
 	if len(req.Command) == 0 {
+		s.metrics.ExecuteRequestsTotal.WithLabelValues("invalid").Inc()
 		c.JSON(http.StatusBadRequest, gin.H{
 			"error": "command cannot be empty",
 			"code":  http.StatusBadRequest,
@@ -76,6 +81,7 @@ func (s *Server) ExecuteHandler(c *gin.Context) {
 		var err error
 		timeoutDuration, err = time.ParseDuration(req.Timeout)
 		if err != nil {
+			s.metrics.ExecuteRequestsTotal.WithLabelValues("invalid").Inc()
 			c.JSON(http.StatusBadRequest, gin.H{
 				"error": fmt.Sprintf("Invalid timeout format: %v", err),
 				"code":  http.StatusBadRequest,
@@ -92,28 +98,11 @@ func (s *Server) ExecuteHandler(c *gin.Context) {
 	// Use the first element as the command and the rest as arguments
 	cmd := exec.CommandContext(ctx, req.Command[0], req.Command[1:]...) //nolint:gosec // This is an agent designed to execute arbitrary commands
 
-	// Default working directory to workspace; override if the request specifies one.
-	cmd.Dir = s.workspaceDir
-	if req.WorkingDir != "" {
-		safeWorkingDir, err := s.sanitizePath(req.WorkingDir)
-		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{
-				"error": fmt.Sprintf("Invalid working directory: %v", err),
-				"code":  http.StatusBadRequest,
-			})
-			return
-		}
-		if _, statErr := os.Stat(safeWorkingDir); os.IsNotExist(statErr) {
-			if err := s.mkdirSafe(safeWorkingDir); err != nil {
-				c.JSON(http.StatusInternalServerError, gin.H{
-					"error": fmt.Sprintf("Failed to create working directory: %v", err),
-					"code":  http.StatusInternalServerError,
-				})
-				return
-			}
-		}
-		cmd.Dir = safeWorkingDir
+	workingDir, ok := s.prepareWorkingDir(c, req.WorkingDir)
+	if !ok {
+		return
 	}
+	cmd.Dir = workingDir
 
 	// Set environment variables
 	if len(req.Env) > 0 {
@@ -134,13 +123,21 @@ func (s *Server) ExecuteHandler(c *gin.Context) {
 	endTime := time.Now()
 
 	var exitCode int
+	var outcomeStatus string
 	if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 		exitCode = TimeoutExitCode
+		outcomeStatus = "timeout"
 		stderr.WriteString(fmt.Sprintf("Command timed out after %.0f seconds", timeoutDuration.Seconds()))
 	} else if cmd.ProcessState != nil {
 		exitCode = cmd.ProcessState.ExitCode()
+		if exitCode == 0 {
+			outcomeStatus = "success"
+		} else {
+			outcomeStatus = "error"
+		}
 	} else {
 		exitCode = 1
+		outcomeStatus = "error"
 		if stderr.Len() > 0 {
 			stderr.WriteString("\n")
 		}
@@ -150,6 +147,8 @@ func (s *Server) ExecuteHandler(c *gin.Context) {
 		}
 	}
 
+	s.metrics.ExecuteRequestsTotal.WithLabelValues(outcomeStatus).Inc()
+
 	c.JSON(http.StatusOK, ExecuteResponse{
 		Stdout:    stdout.String(),
 		Stderr:    stderr.String(),
@@ -159,3 +158,29 @@ func (s *Server) ExecuteHandler(c *gin.Context) {
 		EndTime:   endTime,
 	})
 }
+
+func (s *Server) prepareWorkingDir(c *gin.Context, reqWorkingDir string) (string, bool) {
+	if reqWorkingDir == "" {
+		return s.workspaceDir, true
+	}
+	safeWorkingDir, err := s.sanitizePath(reqWorkingDir)
+	if err != nil {
+		s.metrics.ExecuteRequestsTotal.WithLabelValues("invalid").Inc()
+		c.JSON(http.StatusBadRequest, gin.H{
+			"error": fmt.Sprintf("Invalid working directory: %v", err),
+			"code":  http.StatusBadRequest,
+		})
+		return "", false
+	}
+	if _, statErr := os.Stat(safeWorkingDir); os.IsNotExist(statErr) {
+		if err := s.mkdirSafe(safeWorkingDir); err != nil {
+			s.metrics.ExecuteRequestsTotal.WithLabelValues("error").Inc()
+			c.JSON(http.StatusInternalServerError, gin.H{
+				"error": fmt.Sprintf("Failed to create working directory: %v", err),
+				"code":  http.StatusInternalServerError,
+			})
+			return "", false
+		}
+	}
+	return safeWorkingDir, true
+}

pkg/picod/metrics.go

@@ -0,0 +1,114 @@
+/*
+Copyright The Volcano Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package picod
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// Metrics holds the Prometheus collectors for PicoD.
+type Metrics struct {
+	Registry             *prometheus.Registry
+	ActiveExecutions     prometheus.Gauge
+	ExecuteRequestsTotal *prometheus.CounterVec
+	HTTPRequestsTotal    *prometheus.CounterVec
+	HTTPRequestDuration  *prometheus.HistogramVec
+}
+
+// NewMetrics creates and registers metrics collectors with a private registry.
+func NewMetrics() *Metrics {
+	reg := prometheus.NewRegistry()
+
+	activeExecutions := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "picod_active_executions",
+		Help: "Number of execute handler invocations currently in flight (including validation).",
+	})
+
+	executeRequestsTotal := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "picod_execute_requests_total",
+			Help: "Total number of execute requests, partitioned by status (success, error, timeout, invalid).",
+		},
+		[]string{"status"},
+	)
+
+	httpRequestsTotal := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "picod_http_requests_total",
+			Help: "Total number of HTTP requests processed, partitioned by method, path, and status code.",
+		},
+		[]string{"method", "path", "status_code"},
+	)
+
+	httpRequestDuration := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "picod_http_request_duration_seconds",
+			Help:    "Latency of HTTP requests, partitioned by method and path.",
+			Buckets: prometheus.DefBuckets,
+		},
+		[]string{"method", "path"},
+	)
+
+	reg.MustRegister(activeExecutions)
+	reg.MustRegister(executeRequestsTotal)
+	reg.MustRegister(httpRequestsTotal)
+	reg.MustRegister(httpRequestDuration)
+
+	return &Metrics{
+		Registry:             reg,
+		ActiveExecutions:     activeExecutions,
+		ExecuteRequestsTotal: executeRequestsTotal,
+		HTTPRequestsTotal:    httpRequestsTotal,
+		HTTPRequestDuration:  httpRequestDuration,
+	}
+}
+
+// Handler returns an HTTP handler for the metrics registry.
+func (m *Metrics) Handler() gin.HandlerFunc {
+	h := promhttp.HandlerFor(m.Registry, promhttp.HandlerOpts{})
+	return func(c *gin.Context) {
+		h.ServeHTTP(c.Writer, c.Request)
+	}
+}
+
+// Middleware returns a Gin middleware that records HTTP request metrics.
+func (m *Metrics) Middleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		path := c.FullPath()
+		if path == "" {
+			path = "unmatched"
+		}
+		// Do not record requests to /metrics or /health to avoid noise
+		if path == "/metrics" || path == "/health" {
+			c.Next()
+			return
+		}
+
+		start := time.Now()
+		c.Next()
+		duration := time.Since(start).Seconds()
+
+		status := strconv.Itoa(c.Writer.Status())
+		m.HTTPRequestsTotal.WithLabelValues(c.Request.Method, path, status).Inc()
+		m.HTTPRequestDuration.WithLabelValues(c.Request.Method, path).Observe(duration)
+	}
+}