feat: add custom metric to track Sandbox creation time latency

Author: barney-s

cmd/agent-sandbox-controller/main.go

@@ -22,10 +22,13 @@ import (
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	"github.com/prometheus/client_golang/prometheus"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"sigs.k8s.io/agent-sandbox/controllers"
 	extensionsv1alpha1 "sigs.k8s.io/agent-sandbox/extensions/api/v1alpha1"
@@ -35,8 +38,21 @@ import (
 
 var (
 	setupLog = ctrl.Log.WithName("setup")
+	// sandboxCreationLatency measures the time it takes for a sandbox to be created and ready.
+	sandboxCreationLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "sandbox_creation_latency_seconds",
+			Help:    "Time it takes for a sandbox to be created and ready.",
+			Buckets: []float64{1, 2, 5, 10, 15, 20, 30, 60},
+		},
+	)
 )
 
+func init() {
+	// Register custom metrics with the global prometheus registry
+	metrics.Registry.MustRegister(sandboxCreationLatency)
+}
+
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
@@ -62,7 +78,10 @@ func main() {
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                 scheme,
+		Scheme: scheme,
+		Metrics: metricsserver.Options{
+			BindAddress: metricsAddr,
+		},
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "a3317529.x-k8s.io",
@@ -73,8 +92,9 @@ func main() {
 	}
 
 	if err = (&controllers.SandboxReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:                   mgr.GetClient(),
+		Scheme:                   mgr.GetScheme(),
+		SandboxCreationLatency: sandboxCreationLatency,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Sandbox")
 		os.Exit(1)

controllers/sandbox_controller.go

@@ -22,6 +22,7 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -58,7 +59,8 @@ func init() {
 // SandboxReconciler reconciles a Sandbox object
 type SandboxReconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
+	Scheme                 *runtime.Scheme
+	SandboxCreationLatency prometheus.Histogram
 }
 
 //+kubebuilder:rbac:groups=agents.x-k8s.io,resources=sandboxes,verbs=get;list;watch;create;update;patch;delete
@@ -225,6 +227,16 @@ func (r *SandboxReconciler) updateStatus(ctx context.Context, oldStatus *sandbox
 		return err
 	}
 
+	// If the sandbox is ready, record the creation latency.
+	if meta.IsStatusConditionTrue(sandbox.Status.Conditions, string(sandboxv1alpha1.SandboxConditionReady)) {
+		// Check if the old status was not ready.
+		if !meta.IsStatusConditionTrue(oldStatus.Conditions, string(sandboxv1alpha1.SandboxConditionReady)) {
+			latency := time.Since(sandbox.CreationTimestamp.Time).Seconds()
+			r.SandboxCreationLatency.Observe(latency)
+			log.Info("sandbox is ready, recorded creation latency", "latency", latency)
+		}
+	}
+
 	// Surface error
 	return nil
 }

controllers/sandbox_controller_test.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -649,3 +651,40 @@ func TestSandboxExpiry(t *testing.T) {
 		})
 	}
 }
+
+func TestRecordCreationLatency(t *testing.T) {
+	sandboxName := "sandbox-name"
+	sandboxNs := "sandbox-ns"
+	creationLatencyMetric := prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "sandbox_creation_latency_seconds",
+			Help:    "Time it takes for a sandbox to be created and ready.",
+			Buckets: []float64{1, 2, 5, 10, 15, 20, 30, 60},
+		},
+	)
+
+	sb := &sandboxv1alpha1.Sandbox{}
+	sb.Name = sandboxName
+	sb.Namespace = sandboxNs
+	sb.Generation = 1
+	sb.CreationTimestamp = metav1.NewTime(time.Now().Add(-10 * time.Second))
+
+	r := SandboxReconciler{
+		Client:                 newFakeClient(sb),
+		Scheme:                 Scheme,
+		SandboxCreationLatency: creationLatencyMetric,
+	}
+
+	oldStatus := &sandboxv1alpha1.SandboxStatus{}
+	sb.Status.Conditions = []metav1.Condition{
+		{
+			Type:   string(sandboxv1alpha1.SandboxConditionReady),
+			Status: metav1.ConditionTrue,
+		},
+	}
+
+	err := r.updateStatus(t.Context(), oldStatus, sb)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, testutil.CollectAndCount(creationLatencyMetric))
+}

go.mod

@@ -4,6 +4,7 @@ go 1.24.4
 
 require (
 	github.com/google/go-cmp v0.7.0
+	github.com/prometheus/client_golang v1.23.2
 	github.com/stretchr/testify v1.11.1
 	k8s.io/api v0.34.1
 	k8s.io/apiextensions-apiserver v0.34.1
@@ -43,13 +44,13 @@ require (
 	github.com/google/gnostic-models v0.7.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/onsi/ginkgo/v2 v2.23.3 // indirect
 	github.com/onsi/gomega v1.37.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.1 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect