adds podTemplateSpec field to MCPRegistry CRD that allows user to customise registry api deployment (#2777)

* adds `podTemplateSpec` to MCPRegistry CRD

users will want to override the templatespec for the registry server and
adding a podTemplateSpec override enables this.

Signed-off-by: Chris Burns <[email protected]>

* bumps crds chart

Signed-off-by: Chris Burns <[email protected]>

---------

Signed-off-by: Chris Burns <[email protected]>

Author: ChrisJBurns

cmd/thv-operator/api/v1alpha1/mcpregistry_types.go

@@ -5,6 +5,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
 const (
@@ -43,6 +44,16 @@ type MCPRegistrySpec struct {
 	// +kubebuilder:default=false
 	// +optional
 	EnforceServers bool `json:"enforceServers,omitempty"`
+
+	// PodTemplateSpec defines the pod template to use for the registry API server
+	// This allows for customizing the pod configuration beyond what is provided by the other fields.
+	// Note that to modify the specific container the registry API server runs in, you must specify
+	// the `registry-api` container name in the PodTemplateSpec.
+	// This field accepts a PodTemplateSpec object as JSON/YAML.
+	// +optional
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Type=object
+	PodTemplateSpec *runtime.RawExtension `json:"podTemplateSpec,omitempty"`
 }
 
 // MCPRegistryConfig defines the configuration for a registry data source
@@ -350,6 +361,18 @@ const (
 
 	// ConditionAPIReady indicates whether the registry API is ready
 	ConditionAPIReady = "APIReady"
+
+	// ConditionRegistryPodTemplateValid indicates whether the PodTemplateSpec is valid
+	ConditionRegistryPodTemplateValid = "PodTemplateValid"
+)
+
+// Condition reasons for MCPRegistry PodTemplateSpec validation
+const (
+	// ConditionReasonRegistryPodTemplateValid indicates PodTemplateSpec validation succeeded
+	ConditionReasonRegistryPodTemplateValid = "ValidPodTemplateSpec"
+
+	// ConditionReasonRegistryPodTemplateInvalid indicates PodTemplateSpec validation failed
+	ConditionReasonRegistryPodTemplateInvalid = "InvalidPodTemplateSpec"
 )
 
 //+kubebuilder:object:root=true
@@ -437,3 +460,13 @@ func (r *MCPRegistry) DeriveOverallPhase() MCPRegistryPhase {
 func init() {
 	SchemeBuilder.Register(&MCPRegistry{}, &MCPRegistryList{})
 }
+
+// HasPodTemplateSpec returns true if the MCPRegistry has a PodTemplateSpec
+func (r *MCPRegistry) HasPodTemplateSpec() bool {
+	return r.Spec.PodTemplateSpec != nil
+}
+
+// GetPodTemplateSpecRaw returns the raw PodTemplateSpec
+func (r *MCPRegistry) GetPodTemplateSpecRaw() *runtime.RawExtension {
+	return r.Spec.PodTemplateSpec
+}

cmd/thv-operator/api/v1alpha1/zz_generated.deepcopy.go

@@ -8,6 +8,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -95,6 +96,17 @@ func (r *MCPRegistryReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	ctxLogger.Info("Reconciling MCPRegistry", "MCPRegistry.Name", mcpRegistry.Name, "phase", mcpRegistry.Status.Phase,
 		"apiPhase", apiPhase, "apiEndpoint", apiEndpoint)
 
+	// Validate PodTemplateSpec early - before other operations
+	if mcpRegistry.HasPodTemplateSpec() {
+		// Validate PodTemplateSpec early - before other operations
+		// This ensures we fail fast if the spec is invalid
+		if !r.validateAndUpdatePodTemplateStatus(ctx, mcpRegistry) {
+			// Invalid PodTemplateSpec - return without error to avoid infinite retries
+			// The user must fix the spec and the next reconciliation will retry
+			return ctrl.Result{}, nil
+		}
+	}
+
 	// 2. Handle deletion if DeletionTimestamp is set
 	if mcpRegistry.GetDeletionTimestamp() != nil {
 		// The object is being deleted
@@ -301,3 +313,53 @@ func (*MCPRegistryReconciler) applyStatusUpdates(
 
 	return nil
 }
+
+// validateAndUpdatePodTemplateStatus validates the PodTemplateSpec and updates the MCPRegistry status
+// with appropriate conditions. Returns true if validation passes, false otherwise.
+func (r *MCPRegistryReconciler) validateAndUpdatePodTemplateStatus(
+	ctx context.Context, mcpRegistry *mcpv1alpha1.MCPRegistry,
+) bool {
+	ctxLogger := log.FromContext(ctx)
+
+	// Validate the PodTemplateSpec by attempting to parse it
+	err := registryapi.ValidatePodTemplateSpec(mcpRegistry.GetPodTemplateSpecRaw())
+	if err != nil {
+		// Set phase and message
+		mcpRegistry.Status.Phase = mcpv1alpha1.MCPRegistryPhaseFailed
+		mcpRegistry.Status.Message = fmt.Sprintf("Invalid PodTemplateSpec: %v", err)
+
+		// Set condition for invalid PodTemplateSpec
+		meta.SetStatusCondition(&mcpRegistry.Status.Conditions, metav1.Condition{
+			Type:               mcpv1alpha1.ConditionRegistryPodTemplateValid,
+			Status:             metav1.ConditionFalse,
+			ObservedGeneration: mcpRegistry.Generation,
+			Reason:             mcpv1alpha1.ConditionReasonRegistryPodTemplateInvalid,
+			Message:            fmt.Sprintf("Failed to parse PodTemplateSpec: %v. Deployment blocked until fixed.", err),
+		})
+
+		// Update status with the condition
+		if statusErr := r.Status().Update(ctx, mcpRegistry); statusErr != nil {
+			ctxLogger.Error(statusErr, "Failed to update MCPRegistry status with PodTemplateSpec validation")
+			return false
+		}
+
+		ctxLogger.Error(err, "PodTemplateSpec validation failed")
+		return false
+	}
+
+	// Set condition for valid PodTemplateSpec
+	meta.SetStatusCondition(&mcpRegistry.Status.Conditions, metav1.Condition{
+		Type:               mcpv1alpha1.ConditionRegistryPodTemplateValid,
+		Status:             metav1.ConditionTrue,
+		ObservedGeneration: mcpRegistry.Generation,
+		Reason:             mcpv1alpha1.ConditionReasonRegistryPodTemplateValid,
+		Message:            "PodTemplateSpec is valid",
+	})
+
+	// Update status with the condition
+	if statusErr := r.Status().Update(ctx, mcpRegistry); statusErr != nil {
+		ctxLogger.Error(statusErr, "Failed to update MCPRegistry status with PodTemplateSpec validation")
+	}
+
+	return true
+}

cmd/thv-operator/controllers/mcpregistry_controller.go

@@ -77,7 +77,7 @@ func (m *manager) ensureDeployment(
 	ctxLogger := log.FromContext(ctx).WithValues("mcpregistry", mcpRegistry.Name)
 
 	// Build the desired deployment configuration
-	deployment := m.buildRegistryAPIDeployment(mcpRegistry, configManager)
+	deployment := m.buildRegistryAPIDeployment(ctx, mcpRegistry, configManager)
 	deploymentName := deployment.Name
 
 	// Set owner reference for automatic garbage collection
@@ -119,18 +119,30 @@ func (m *manager) ensureDeployment(
 // This function handles all deployment configuration including labels, container specs, probes,
 // and storage manager integration. It returns a fully configured deployment ready for Kubernetes API operations.
 func (*manager) buildRegistryAPIDeployment(
+	ctx context.Context,
 	mcpRegistry *mcpv1alpha1.MCPRegistry,
 	configManager config.ConfigManager,
 ) *appsv1.Deployment {
+	ctxLogger := log.FromContext(ctx).WithValues("mcpregistry", mcpRegistry.Name)
 	// Generate deployment name using the established pattern
 	deploymentName := mcpRegistry.GetAPIResourceName()
 
 	// Define labels using common function
 	labels := labelsForRegistryAPI(mcpRegistry, deploymentName)
 
-	// Build the PodTemplateSpec using the functional options pattern
-	// This includes all mount configurations via the builder pattern
-	builder := NewPodTemplateSpecBuilder()
+	// Parse user-provided PodTemplateSpec if present
+	var userPTS *corev1.PodTemplateSpec
+	if mcpRegistry.HasPodTemplateSpec() {
+		var err error
+		userPTS, err = ParsePodTemplateSpec(mcpRegistry.GetPodTemplateSpecRaw())
+		if err != nil {
+			ctxLogger.Error(err, "Failed to parse PodTemplateSpec")
+			return nil
+		}
+	}
+
+	// Build PodTemplateSpec with defaults and user customizations merged
+	builder := NewPodTemplateSpecBuilderFrom(userPTS)
 	podTemplateSpec := builder.Apply(
 		WithLabels(labels),
 		WithAnnotations(map[string]string{

cmd/thv-operator/pkg/registryapi/deployment.go

@@ -1,6 +1,7 @@
 package registryapi
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -10,6 +11,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	mcpv1alpha1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1alpha1"
@@ -121,6 +123,47 @@ func TestManagerBuildRegistryAPIDeployment(t *testing.T) {
 
 			},
 		},
+		{
+			name: "user PodTemplateSpec merged correctly",
+			mcpRegistry: &mcpv1alpha1.MCPRegistry{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-registry",
+					Namespace: "test-namespace",
+				},
+				Spec: mcpv1alpha1.MCPRegistrySpec{
+					Registries: []mcpv1alpha1.MCPRegistryConfig{
+						{
+							Name:   "default",
+							Format: mcpv1alpha1.RegistryFormatToolHive,
+							ConfigMapRef: &corev1.ConfigMapKeySelector{
+								LocalObjectReference: corev1.LocalObjectReference{
+									Name: "test-configmap",
+								},
+								Key: "registry.json",
+							},
+						},
+					},
+					PodTemplateSpec: &runtime.RawExtension{
+						Raw: []byte(`{"spec":{"serviceAccountName":"custom-sa"}}`),
+					},
+				},
+			},
+			setupMocks: func() {
+			},
+			validateResult: func(t *testing.T, deployment *appsv1.Deployment) {
+				t.Helper()
+				require.NotNil(t, deployment)
+
+				// User-provided service account name should take precedence
+				assert.Equal(t, "custom-sa", deployment.Spec.Template.Spec.ServiceAccountName)
+
+				// Default volumes and mounts should still be present
+				volumes := deployment.Spec.Template.Spec.Volumes
+				assert.True(t, hasVolume(volumes, RegistryServerConfigVolumeName))
+				assert.True(t, hasVolume(volumes, "storage-data"))
+				assert.True(t, hasVolume(volumes, "registry-data-source-default"))
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -134,7 +177,7 @@ func TestManagerBuildRegistryAPIDeployment(t *testing.T) {
 			manager := &manager{}
 
 			configManager := config.NewConfigManagerForTesting(tt.mcpRegistry)
-			deployment := manager.buildRegistryAPIDeployment(tt.mcpRegistry, configManager)
+			deployment := manager.buildRegistryAPIDeployment(context.Background(), tt.mcpRegistry, configManager)
 			tt.validateResult(t, deployment)
 		})
 	}

cmd/thv-operator/pkg/registryapi/deployment_test.go

@@ -2,11 +2,13 @@
 package registryapi
 
 import (
+	"encoding/json"
 	"fmt"
 	"path/filepath"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	mcpv1alpha1 "github.com/stacklok/toolhive/cmd/thv-operator/api/v1alpha1"
@@ -17,15 +19,45 @@ import (
 type PodTemplateSpecOption func(*corev1.PodTemplateSpec)
 
 // PodTemplateSpecBuilder builds a PodTemplateSpec using the functional options pattern.
+// When created with NewPodTemplateSpecBuilderFrom, the builder stores the user's template
+// and applies options as defaults. Build() merges them with user values taking precedence.
 type PodTemplateSpecBuilder struct {
-	podTemplateSpec *corev1.PodTemplateSpec
+	// userTemplate is the user-provided PodTemplateSpec (if any)
+	userTemplate *corev1.PodTemplateSpec
+	// defaultSpec is built up via Apply() with options acting as defaults
+	defaultSpec *corev1.PodTemplateSpec
 }
 
-// NewPodTemplateSpecBuilder creates a new PodTemplateSpecBuilder with default values.
-func NewPodTemplateSpecBuilder() *PodTemplateSpecBuilder {
+// NewPodTemplateSpecBuilderFrom creates a new PodTemplateSpecBuilder with a user-provided template.
+// The user template is deep-copied to avoid mutating the original.
+// Options applied via Apply() act as defaults - Build() will merge them with user values,
+// where user values take precedence over defaults.
+func NewPodTemplateSpecBuilderFrom(userTemplate *corev1.PodTemplateSpec) *PodTemplateSpecBuilder {
+	var userCopy *corev1.PodTemplateSpec
+	if userTemplate != nil {
+		userCopy = userTemplate.DeepCopy()
+	}
 	return &PodTemplateSpecBuilder{
-		podTemplateSpec: &corev1.PodTemplateSpec{},
+		userTemplate: userCopy,
+		defaultSpec:  &corev1.PodTemplateSpec{},
+	}
+}
+
+// Apply applies the given options to build up the default PodTemplateSpec.
+func (b *PodTemplateSpecBuilder) Apply(opts ...PodTemplateSpecOption) *PodTemplateSpecBuilder {
+	for _, opt := range opts {
+		opt(b.defaultSpec)
+	}
+	return b
+}
+
+// Build returns the final PodTemplateSpec.
+// If a user template was provided, merges defaults with user values (user takes precedence).
+func (b *PodTemplateSpecBuilder) Build() corev1.PodTemplateSpec {
+	if b.userTemplate == nil {
+		return *b.defaultSpec
 	}
+	return MergePodTemplateSpecs(b.defaultSpec, b.userTemplate)
 }
 
 // WithLabels sets the labels on the PodTemplateSpec.
@@ -196,17 +228,28 @@ func WithRegistrySourceMounts(containerName string, registries []mcpv1alpha1.MCP
 	}
 }
 
-// Apply applies the given options to the PodTemplateSpecBuilder.
-func (b *PodTemplateSpecBuilder) Apply(opts ...PodTemplateSpecOption) *PodTemplateSpecBuilder {
-	for _, opt := range opts {
-		opt(b.podTemplateSpec)
+// ParsePodTemplateSpec parses a runtime.RawExtension into a PodTemplateSpec.
+// Returns nil if the raw extension is nil or empty.
+// Returns an error if the raw extension contains invalid PodTemplateSpec data.
+func ParsePodTemplateSpec(raw *runtime.RawExtension) (*corev1.PodTemplateSpec, error) {
+	if raw == nil || raw.Raw == nil || len(raw.Raw) == 0 {
+		return nil, nil
 	}
-	return b
+
+	var pts corev1.PodTemplateSpec
+	if err := json.Unmarshal(raw.Raw, &pts); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal PodTemplateSpec: %w", err)
+	}
+
+	return &pts, nil
 }
 
-// Build returns the configured PodTemplateSpec.
-func (b *PodTemplateSpecBuilder) Build() corev1.PodTemplateSpec {
-	return *b.podTemplateSpec
+// ValidatePodTemplateSpec validates a runtime.RawExtension contains valid PodTemplateSpec data.
+// Returns nil if the raw extension is nil, empty, or contains valid data.
+// Returns an error if the raw extension contains invalid PodTemplateSpec data.
+func ValidatePodTemplateSpec(raw *runtime.RawExtension) error {
+	_, err := ParsePodTemplateSpec(raw)
+	return err
 }
 
 // BuildRegistryAPIContainer creates the registry-api container with default configuration.
@@ -257,19 +300,6 @@ func BuildRegistryAPIContainer(image string) corev1.Container {
 	}
 }
 
-// DefaultRegistryAPIPodTemplateSpec creates a default PodTemplateSpec for the registry-api.
-func DefaultRegistryAPIPodTemplateSpec(labels map[string]string, configHash string) corev1.PodTemplateSpec {
-	builder := NewPodTemplateSpecBuilder()
-	return builder.Apply(
-		WithLabels(labels),
-		WithAnnotations(map[string]string{
-			"toolhive.stacklok.dev/config-hash": configHash,
-		}),
-		WithServiceAccountName(DefaultServiceAccountName),
-		WithContainer(BuildRegistryAPIContainer(getRegistryAPIImage())),
-	).Build()
-}
-
 // MergePodTemplateSpecs merges a default PodTemplateSpec with a user-provided one.
 // User-provided values take precedence over defaults. This allows users to customize
 // infrastructure concerns while ensuring sensible defaults are applied where values