Warn about conflicts with ksm or node exporter service monitors (#2211)

Signed-off-by: Pete Wall <[email protected]>

Author: petewall

charts/k8s-monitoring/CHANGELOG.md

@@ -2,8 +2,11 @@
 
 ## 3.7.0
 
+*   Add an integration for PostgreSQL, including support for Database Observability (@petewall)
 *   Add the ability to set otel_annotations flag for the k8sattributes processor (@petewall)
 *   Add more options to the secretFilter component in the pod logs features (@petewall)
+*   Check for the presence of kube-state-metrics or Node Exporter ServiceMonitors if clusterMetrics and
+    prometheusOperatorObjects features are enabled (@petewall)
 
 ## 3.6.2
 

charts/k8s-monitoring/charts/feature-cluster-metrics/templates/_notes.tpl

@@ -17,9 +17,42 @@
 Scrape Kubernetes Cluster metrics
 {{- end }}
 
-{{- define "feature.clusterMetrics.notes.actions" }}{{- end }}
 
-{{- define "feature.clusterMetrics.summary" -}}
+{{- define "feature.clusterMetrics.notes.actions" }}
+{{- $serviceMonitorScrapingEnabled := and .Values.prometheusOperatorObjects.enabled (dig "serviceMonitors" "enabled" true .Values.prometheusOperatorObjects)}}
+{{- if $serviceMonitorScrapingEnabled }}
+  {{- $values := .Values.clusterMetrics }}
+  {{- if (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+    {{- $namespaces := list }}
+    {{- range $serviceMonitor := (lookup "monitoring.coreos.com/v1" "ServiceMonitor" "" "").items }}
+      {{- if contains "kube-state-metrics" $serviceMonitor.metadata.name }}
+        {{- if (index $values "kube-state-metrics").enabled }}
+          {{- $namespaces = append $namespaces $serviceMonitor.metadata.namespace }}
+⚠️ Detected a ServiceMonitor named {{ $serviceMonitor.metadata.name }} in namespace {{ $serviceMonitor.metadata.namespace }}, but this chart has already enabled the Cluster Metrics feature. This might result in duplicated metrics from kube-state-metrics.
+        {{- end }}
+      {{- end }}
+      {{- if contains "node-exporter" $serviceMonitor.metadata.name }}
+        {{- if (index $values "node-exporter").enabled }}
+          {{- $namespaces = append $namespaces $serviceMonitor.metadata.namespace }}
+⚠️ Detected a ServiceMonitor named {{ $serviceMonitor.metadata.name }} in namespace {{ $serviceMonitor.metadata.namespace }}, but this chart has already enabled the Cluster Metrics feature. This might result in duplicated metrics from Node Exporter.
+        {{- end }}
+      {{- end }}
+    {{- end }}
+  {{- if $namespaces }}
+To prevent duplicate metrics, either delete the ServiceMonitor, or filter them out with:
+prometheusOperatorObjects:
+  serviceMonitors:
+    excludeNamespaces:{{ $namespaces | uniq | toYaml | nindent 6 }}
+OR
+prometheusOperatorObjects:
+  serviceMonitors:
+    labelExpressions: <label expression to exclude the ServiceMonitor>
+  {{- end }}
+  {{- end }}
+{{- end }}
+{{- end }}
+
+{{- define "feature.clusterMetrics.summary" }}
 {{- $sources := list }}
 {{- if .Values.controlPlane.enabled }}{{- $sources = append $sources "controlPlane" }}{{ end }}
 {{- if .Values.kubelet.enabled }}{{- $sources = append $sources "kubelet" }}{{ end }}

charts/k8s-monitoring/templates/NOTES.txt

@@ -24,9 +24,9 @@ It will:
   {{- $collectorName := (index $.Values $feature).collector }}
   {{- $Collector := dict "Namespace" $.Release.Namespace }}
   {{- if $collectorName }}
-    {{- $Collector = merge $Collector (dict "ServiceName" (include "collector.alloy.fullname" (merge $ (dict "collectorName" $collectorName)))) }}
+    {{- $Collector = merge $Collector (dict "ServiceName" (include "collector.alloy.fullname" (merge (dict "collectorName" $collectorName) $))) }}
   {{- end }}
-  {{- include (printf "feature.%s.notes.actions" $feature) (dict "Values" $.Values "Collector" $Collector) | indent 0 }}
+  {{- include (printf "feature.%s.notes.actions" $feature) (merge (dict "Collector" $Collector) $) | indent 0 }}
 {{- end }}
 {{- if $isTranslating }}
 

charts/k8s-monitoring/templates/features/_feature_cluster_metrics.tpl

@@ -40,15 +40,23 @@ cluster_metrics "feature" {
 {{- define "features.clusterMetrics.collector.values" }}{{- end -}}
 
 {{- define "features.clusterMetrics.validate" }}
-{{- if .Values.clusterMetrics.enabled -}}
-{{- $featureName := "Kubernetes Cluster metrics" }}
-{{- $destinations := include "features.clusterMetrics.destinations" . | fromYamlArray }}
-{{- include "destinations.validate_destination_list" (dict "destinations" $destinations "type" "metrics" "ecosystem" "prometheus" "feature" $featureName) }}
-{{- range $collector := include "features.clusterMetrics.collectors" . | fromYamlArray }}
-  {{- include "collectors.require_collector" (dict "Values" $.Values "name" $collector "feature" $featureName) }}
-{{- end -}}
+{{- if .Values.clusterMetrics.enabled }}
+  {{- $featureName := "Kubernetes Cluster metrics" }}
+  {{- $destinations := include "features.clusterMetrics.destinations" . | fromYamlArray }}
+  {{- include "destinations.validate_destination_list" (dict "destinations" $destinations "type" "metrics" "ecosystem" "prometheus" "feature" $featureName) }}
 
+  {{- range $collector := include "features.clusterMetrics.collectors" . | fromYamlArray }}
+    {{- include "collectors.require_collector" (dict "Values" $.Values "name" $collector "feature" $featureName) }}
+  {{- end }}
+
+  {{- include "features.clusterMetrics.validate.opencost" . }}
+  {{- include "feature.clusterMetrics.validate" (dict "Values" $.Values.clusterMetrics) }}
+{{- end }}
+{{- end }}
+
+{{- define "features.clusterMetrics.validate.opencost" }}
 {{- if .Values.clusterMetrics.opencost.enabled}}
+  {{- $destinations := include "features.clusterMetrics.destinations" . | fromYamlArray }}
   {{- if ne .Values.cluster.name .Values.clusterMetrics.opencost.opencost.exporter.defaultClusterId }}
     {{- $msg := list "" "The OpenCost default cluster id should match the cluster name." }}
     {{- $msg = append $msg "Please set:" }}
@@ -73,7 +81,7 @@ cluster_metrics "feature" {
       {{- $msg = append $msg (printf "Where <metrics destination name> is one of %s" (include "english_list_or" $destinations)) }}
       {{- end }}
       {{- fail (join "\n" $msg) }}
-    {{- end -}}
+    {{- end }}
 
     {{- $destinationFound := false }}
     {{- range $index, $destinationName := $destinations }}
@@ -178,7 +186,7 @@ cluster_metrics "feature" {
           {{- fail (join "\n" $msg) }}
         {{- end }}
       {{- end }}
-    {{- end -}}
+    {{- end }}
 
     {{- if eq $destinationFound false }}
       {{- $msg := list "" (printf "The destination \"%s\" is not a Prometheus data source." $.Values.clusterMetrics.opencost.metricsSource) }}
@@ -190,9 +198,7 @@ cluster_metrics "feature" {
       {{- $msg = append $msg "    metricsSource:  <metrics destination name>" }}
       {{- $msg = append $msg (printf "Where <metrics destination name> is one of %s" (include "english_list_or" $destinations)) }}
       {{- fail (join "\n" $msg) }}
-    {{- end -}}
-  {{- end -}}
-{{- end -}}
-{{- include "feature.clusterMetrics.validate" (dict "Values" $.Values.clusterMetrics) }}
+    {{- end }}
+  {{- end }}
 {{- end -}}
 {{- end -}}

charts/k8s-monitoring/tests/__snapshot__/feature_notes_test.yaml.snap

@@ -13,4 +13,3 @@ print the warning about ecosystem translation if the destinations don't match:
       * One or more features will have their data translated to a different storage ecosystem. This may result in an imperfect
         translation of labels and attributes. For more information, consult:
           https://github.com/grafana/k8s-monitoring-helm/tree/main/charts/k8s-monitoring/docs/destinations#ecosystem-translation
-

charts/k8s-monitoring/tests/feature_notes_test.yaml

@@ -18,3 +18,45 @@ tests:
         enabled: true
     asserts:
       - matchSnapshotRaw: {}
+
+#  - it: print the warning about ServiceMonitors if they exist on the system
+#    capabilities:
+#      apiVersions: [monitoring.coreos.com/v1/ServiceMonitor]
+#    kubernetesProvider:
+#      scheme:
+#        "monitoring.coreos.com/v1/ServiceMonitor":
+#          gvr:
+#            version: "monitoring.coreos.com/v1"
+#            resource: "servicemonitors"
+#          namespaced: true
+#      objects:
+#        - apiVersion: "monitoring.coreos.com/v1"
+#          kind: ServiceMonitor
+#          metadata:
+#            name: my-kube-state-metrics-service-monitor
+#            namespace: monitoring
+#        - apiVersion: "monitoring.coreos.com/v1"
+#          kind: ServiceMonitor
+#          metadata:
+#            name: prometheus-node-exporter
+#            namespace: monitoring
+#        - apiVersion: "monitoring.coreos.com/v1"
+#          kind: ServiceMonitor
+#          metadata:
+#            name: redis
+#            namespace: monitoring
+#    set:
+#      cluster:
+#        name: ci-test-cluster
+#      destinations:
+#        - name: test
+#          type: prometheus
+#          url: https://prometheus.example.com
+#      clusterMetrics:
+#        enabled: true
+#      prometheusOperatorObjects:
+#        enabled: true
+#      alloy-metrics:
+#        enabled: true
+#    asserts:
+#      - matchSnapshotRaw: {}

charts/k8s-monitoring/tests/platform_validations_openshift_test.yaml

@@ -5,7 +5,8 @@ templates:
 tests:
   - it: asks you to set the platform to OpenShift
     set:
-      cluster: {name: openshift-cluster}
+      cluster:
+        name: openshift-cluster
     capabilities:
       apiVersions:
         - security.openshift.io/v1/SecurityContextConstraints
@@ -15,3 +16,16 @@ tests:
             This Kubernetes cluster appears to be OpenShift. Please set the platform to enable compatibility:
             global:
               platform: openshift
+
+  - it: does not ask when the platform is set
+    set:
+      global:
+        platform: openshift
+      cluster:
+        name: openshift-cluster
+      alloy-metrics: {enabled: true, extraConfig: "// Enabled"}
+    capabilities:
+      apiVersions:
+        - security.openshift.io/v1/SecurityContextConstraints
+    asserts:
+      - notFailedTemplate: {}