Skip to content

Commit e79aaca

Browse files
barney-sbarney-s
committed
feat: add custom metric to track Sandbox creation time latency
1 parent e568f6c commit e79aaca

File tree

6 files changed

+110
-13
lines changed

6 files changed

+110
-13
lines changed

‎cmd/agent-sandbox-controller/main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package main
1717
import (
1818
"flag"
1919
"os"
20+
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
2021

2122
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
2223
// to ensure that exec-entrypoint and run can make use of them.
@@ -63,6 +64,7 @@ func main() {
6364

6465
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
6566
Scheme: scheme,
67+
Metrics: metricsserver.Options{BindAddress: metricsAddr},
6668
HealthProbeBindAddress: probeAddr,
6769
LeaderElection: enableLeaderElection,
6870
LeaderElectionID: "a3317529.x-k8s.io",

‎controllers/metrics.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright 2025 The Kubernetes Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package controllers
16+
17+
import (
18+
"github.com/prometheus/client_golang/prometheus"
19+
"sigs.k8s.io/controller-runtime/pkg/metrics"
20+
)
21+
22+
var (
23+
sandboxCreationLatency = prometheus.NewHistogram(
24+
prometheus.HistogramOpts{
25+
Name: "sandbox_creation_latency_seconds",
26+
Help: "Time it takes for a sandbox to become ready after creation.",
27+
Buckets: []float64{1, 2, 5, 10, 15, 30, 60, 120, 300},
28+
},
29+
)
30+
)
31+
32+
func init() {
33+
metrics.Registry.MustRegister(sandboxCreationLatency)
34+
}

‎controllers/sandbox_controller.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,8 +148,15 @@ func (r *SandboxReconciler) reconcileChildResources(ctx context.Context, sandbox
148148

149149
// compute and set overall Ready condition
150150
readyCondition := r.computeReadyCondition(sandbox, allErrors, svc, pod)
151+
oldReadyCondition := meta.FindStatusCondition(sandbox.Status.Conditions, string(sandboxv1alpha1.SandboxConditionReady))
151152
meta.SetStatusCondition(&sandbox.Status.Conditions, readyCondition)
152153

154+
// record metric if we are marking sandbox as ready
155+
if readyCondition.Status == metav1.ConditionTrue && (oldReadyCondition == nil || oldReadyCondition.Status == metav1.ConditionFalse) {
156+
latency := time.Since(sandbox.CreationTimestamp.Time)
157+
sandboxCreationLatency.Observe(latency.Seconds())
158+
}
159+
153160
return allErrors
154161
}
155162

‎controllers/sandbox_controller_test.go‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,13 @@ package controllers
1616

1717
import (
1818
"errors"
19+
"strings"
1920
"testing"
2021
"time"
2122

2223
"github.com/google/go-cmp/cmp"
2324
"github.com/google/go-cmp/cmp/cmpopts"
25+
"github.com/prometheus/client_golang/prometheus/testutil"
2426
"github.com/stretchr/testify/require"
2527
corev1 "k8s.io/api/core/v1"
2628
k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -406,7 +408,52 @@ func TestReconcile(t *testing.T) {
406408
},
407409
},
408410
},
409-
}
411+
{
412+
name: "metric is recorded when sandbox becomes ready",
413+
initialObjs: []runtime.Object{
414+
&corev1.Pod{
415+
ObjectMeta: metav1.ObjectMeta{
416+
Name: sandboxName,
417+
Namespace: sandboxNs,
418+
},
419+
Status: corev1.PodStatus{
420+
Phase: corev1.PodRunning,
421+
Conditions: []corev1.PodCondition{
422+
{
423+
Type: corev1.PodReady,
424+
Status: corev1.ConditionTrue,
425+
},
426+
},
427+
},
428+
},
429+
},
430+
sandboxSpec: sandboxv1alpha1.SandboxSpec{
431+
PodTemplate: sandboxv1alpha1.PodTemplate{
432+
Spec: corev1.PodSpec{
433+
Containers: []corev1.Container{
434+
{
435+
Name: "test-container",
436+
},
437+
},
438+
},
439+
},
440+
},
441+
wantStatus: sandboxv1alpha1.SandboxStatus{
442+
Service: sandboxName,
443+
ServiceFQDN: "sandbox-name.sandbox-ns.svc.cluster.local",
444+
Replicas: 1,
445+
LabelSelector: "agents.x-k8s.io/sandbox-name-hash=ab179450", // Pre-computed hash of "sandbox-name"
446+
Conditions: []metav1.Condition{
447+
{
448+
Type: "Ready",
449+
Status: "True",
450+
ObservedGeneration: 1,
451+
Reason: "DependenciesReady",
452+
Message: "Pod is Ready; Service Exists",
453+
},
454+
},
455+
},
456+
}, }
410457

411458
for _, tc := range testCases {
412459
t.Run(tc.name, func(t *testing.T) {
@@ -443,6 +490,9 @@ func TestReconcile(t *testing.T) {
443490
require.NoError(t, err)
444491
require.Equal(t, obj, liveObj)
445492
}
493+
if strings.Contains(tc.name, "metric") {
494+
require.Equal(t, 1, testutil.CollectAndCount(sandboxCreationLatency))
495+
}
446496
})
447497
}
448498
}

‎go.mod‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.24.4
44

55
require (
66
github.com/google/go-cmp v0.7.0
7+
github.com/prometheus/client_golang v1.22.0
78
github.com/stretchr/testify v1.11.1
89
k8s.io/api v0.34.1
910
k8s.io/apiextensions-apiserver v0.34.1
@@ -17,8 +18,9 @@ require (
1718
require (
1819
github.com/beorn7/perks v1.0.1 // indirect
1920
github.com/cespare/xxhash/v2 v2.3.0 // indirect
20-
github.com/davecgh/go-spew v1.1.1 // indirect
21+
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
2122
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
23+
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
2224
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
2325
github.com/fsnotify/fsnotify v1.9.0 // indirect
2426
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
@@ -43,15 +45,15 @@ require (
4345
github.com/google/gnostic-models v0.7.0 // indirect
4446
github.com/google/uuid v1.6.0 // indirect
4547
github.com/json-iterator/go v1.1.12 // indirect
48+
github.com/kylelemons/godebug v1.1.0 // indirect
4649
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
4750
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
4851
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
4952
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
5053
github.com/onsi/gomega v1.37.0 // indirect
51-
github.com/pmezard/go-difflib v1.0.0 // indirect
52-
github.com/prometheus/client_golang v1.23.2 // indirect
54+
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
5355
github.com/prometheus/client_model v0.6.2 // indirect
54-
github.com/prometheus/common v0.67.1 // indirect
56+
github.com/prometheus/common v0.62.0 // indirect
5557
github.com/prometheus/procfs v0.17.0 // indirect
5658
github.com/rogpeppe/go-internal v1.14.1 // indirect
5759
github.com/spf13/pflag v1.0.10 // indirect

‎go.sum‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
33
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
44
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
55
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6-
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
76
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7+
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
8+
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
89
github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
910
github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
10-
github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
11-
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
11+
github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
12+
github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
1213
github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
1314
github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
1415
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
@@ -92,14 +93,15 @@ github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
9293
github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
9394
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
9495
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
95-
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
9696
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
97-
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
98-
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
97+
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
98+
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
99+
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
100+
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
99101
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
100102
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
101-
github.com/prometheus/common v0.67.1 h1:OTSON1P4DNxzTg4hmKCc37o4ZAZDv0cfXLkOt0oEowI=
102-
github.com/prometheus/common v0.67.1/go.mod h1:RpmT9v35q2Y+lsieQsdOh5sXZ6ajUGC8NjZAmr8vb0Q=
103+
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
104+
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
103105
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
104106
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
105107
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=

0 commit comments

Comments
 (0)