feat(storage): Add SigV4 authentication support for Elasticsearch/OpenSearch storage backends (#7611)

This PR enables Jaeger to use AWS Managed Elasticsearch/OpenSearch for
trace and metrics storage by adding SigV4 HTTP authentication support to
Elasticsearch and OpenSearch backends.

## Summary of changes

**Configuration**
- Add
`jaeger_storage.backends.<name>.<elasticsearch|opensearch>.auth_extension.authenticator`
to reference an OpenTelemetry HTTP authenticator extension by name
- Add
`jaeger_storage.metric_backends.<name>.<elasticsearch|opensearch>.auth_extension.authenticator`
for metric storage backends

**Elasticsearch/OpenSearch backends**
- Thread the resolved HTTP authenticator through the factory chain
(v1/v2 trace storage and metrics storage)
- Wrap the HTTP RoundTripper used by ES/OS clients with the extension's
RoundTripper (applies SigV4 signing when using `sigv4authextension` )
- Updated `GetHTTPRoundTripper()` to accept and apply the HTTP
authenticator

## Configuration example

```yaml
extensions:
sigv4auth:
region: us-east-1
service: es # or 'aoss' for OpenSearch Serverless
# credentials/assume-role configuration per the extension's documentation

service:
extensions: [sigv4auth]

jaeger_storage:
backends:
es-aws:
elasticsearch:
servers: ["https://my-domain.us-east-1.es.amazonaws.com/"]
auth_extension:
authenticator: sigv4auth
indices:
spans:
shards: 5
replicas: 1

metric_backends:
es-metrics:
elasticsearch:
servers: ["https://my-domain.us-east-1.es.amazonaws.com/"]
auth_extension:
authenticator: sigv4auth
```

## Implementation

- ES/OS backends now support optional HTTP authenticators via
`auth_extension.authenticator`
- The extension's RoundTripper wraps the base transport for SigV4
signing
- Supports trace and metrics storage for Elasticsearch 7.x/8.x and
OpenSearch

## Scope

- Adds authentication support to:
  - Elasticsearch trace storage (v1 and v2)
  - OpenSearch trace storage (v1 and v2)
  - Elasticsearch metrics storage
  - OpenSearch metrics storage
- Backward compatible - authentication is optional

## Related issue

Part of #7468

---------

Signed-off-by: SoumyaRaikwar <[email protected]>
Signed-off-by: Soumya Raikwar <[email protected]>
Co-authored-by: Yuri Shkuro <[email protected]>

Author: SoumyaRaikwar

cmd/jaeger/internal/extension/jaegerstorage/config.go

@@ -51,27 +51,9 @@ type TraceBackend struct {
 	ClickHouse    *clickhouse.Configuration `mapstructure:"clickhouse"`
 }
 
-// AuthConfig represents authentication configuration for metric backends.
-//
-// The Authenticator field expects the ID (name) of an HTTP authenticator
-// extension that is registered in the running binary and implements
-// go.opentelemetry.io/collector/extension/extensionauth.HTTPClient.
-//
-// Valid values:
-//   - "sigv4auth" in the stock Jaeger binary (built-in).
-//   - Any other extension name is valid only if that authenticator extension
-//     is included in the build; otherwise Jaeger will error at startup when
-//     resolving the extension.
-//   - Empty/omitted means no auth (default behavior).
-type AuthConfig struct {
-	// Authenticator is the name (ID) of the HTTP authenticator extension to use.
-	Authenticator string `mapstructure:"authenticator"`
-}
-
-// PrometheusConfiguration wraps the base Prometheus configuration with auth support.
 type PrometheusConfiguration struct {
-	promcfg.Configuration `mapstructure:",squash"`
-	Auth                  *AuthConfig `mapstructure:"auth,omitempty"`
+	Configuration  promcfg.Configuration `mapstructure:",squash"`
+	Authentication escfg.Authentication  `mapstructure:"auth"`
 }
 
 // MetricBackend contains configuration for a single metric storage backend.

cmd/jaeger/internal/extension/jaegerstorage/config_test.go

@@ -116,7 +116,7 @@ metric_backends:
 `)
 	cfg := createDefaultConfig().(*Config)
 	require.NoError(t, conf.Unmarshal(cfg))
-	assert.NotEmpty(t, cfg.MetricBackends["some_metrics_storage"].Prometheus.ServerURL)
+	assert.NotEmpty(t, cfg.MetricBackends["some_metrics_storage"].Prometheus.Configuration.ServerURL)
 }
 
 func TestConfigDefaultElasticsearchAsMetricsBackend(t *testing.T) {

cmd/jaeger/internal/extension/jaegerstorage/extension.go

@@ -14,6 +14,7 @@ import (
 	"go.opentelemetry.io/collector/extension/extensionauth"
 
 	"github.com/jaegertracing/jaeger/internal/metrics"
+	"github.com/jaegertracing/jaeger/internal/storage/elasticsearch/config"
 	esmetrics "github.com/jaegertracing/jaeger/internal/storage/metricstore/elasticsearch"
 	"github.com/jaegertracing/jaeger/internal/storage/metricstore/prometheus"
 	"github.com/jaegertracing/jaeger/internal/storage/v1"
@@ -135,9 +136,9 @@ func findExtension(host component.Host) (Extension, error) {
 	return ext, nil
 }
 
-func newStorageExt(config *Config, telset component.TelemetrySettings) *storageExt {
+func newStorageExt(cfg *Config, telset component.TelemetrySettings) *storageExt {
 	return &storageExt{
-		config:           config,
+		config:           cfg,
 		telset:           telset,
 		factories:        make(map[string]tracestore.Factory),
 		metricsFactories: make(map[string]storage.MetricStoreFactory),
@@ -184,19 +185,30 @@ func (s *storageExt) Start(ctx context.Context, host component.Host) error {
 		case cfg.Elasticsearch != nil:
 			esTelset := telset
 			esTelset.Metrics = scopedMetricsFactory(storageName, "elasticsearch", "tracestore")
+			httpAuth, authErr := s.resolveAuthenticator(host, cfg.Elasticsearch.Authentication, "elasticsearch", storageName)
+			if authErr != nil {
+				return authErr
+			}
 			factory, err = es.NewFactory(
 				ctx,
 				*cfg.Elasticsearch,
 				esTelset,
+				httpAuth,
 			)
+
 		case cfg.Opensearch != nil:
 			osTelset := telset
 			osTelset.Metrics = scopedMetricsFactory(storageName, "opensearch", "tracestore")
-			factory, err = es.NewFactory(
-				ctx,
+			httpAuth, authErr := s.resolveAuthenticator(host, cfg.Opensearch.Authentication, "opensearch", storageName)
+			if authErr != nil {
+				return authErr
+			}
+			factory, err = es.NewFactory(ctx,
 				*cfg.Opensearch,
 				osTelset,
+				httpAuth,
 			)
+
 		case cfg.ClickHouse != nil:
 			chTelset := telset
 			chTelset.Metrics = scopedMetricsFactory(storageName, "clickhouse", "tracestore")
@@ -223,46 +235,43 @@ func (s *storageExt) Start(ctx context.Context, host component.Host) error {
 		case cfg.Prometheus != nil:
 			promTelset := telset
 			promTelset.Metrics = scopedMetricsFactory(metricStorageName, "prometheus", "metricstore")
-
-			// Resolve authenticator if configured
-			var httpAuthenticator extensionauth.HTTPClient
-			if cfg.Prometheus.Auth != nil && cfg.Prometheus.Auth.Authenticator != "" {
-				httpAuthenticator, err = s.getAuthenticator(host, cfg.Prometheus.Auth.Authenticator)
-				if err != nil {
-					return fmt.Errorf("failed to get HTTP authenticator '%s' for metric storage '%s': %w",
-						cfg.Prometheus.Auth.Authenticator, metricStorageName, err)
-				}
-				s.telset.Logger.Sugar().Infof("HTTP auth configured for metric storage '%s' with authenticator '%s'",
-					metricStorageName, cfg.Prometheus.Auth.Authenticator)
+			httpAuth, authErr := s.resolveAuthenticator(host, cfg.Prometheus.Authentication, "prometheus metrics", metricStorageName)
+			if authErr != nil {
+				return authErr
 			}
-
-			// Create factory with optional authenticator (nil if not configured)
 			metricStoreFactory, err = prometheus.NewFactoryWithConfig(
 				cfg.Prometheus.Configuration,
 				promTelset,
-				httpAuthenticator,
+				httpAuth,
 			)
-			if err != nil {
-				return fmt.Errorf("failed to initialize metrics storage '%s': %w", metricStorageName, err)
-			}
 
 		case cfg.Elasticsearch != nil:
 			esTelset := telset
 			esTelset.Metrics = scopedMetricsFactory(metricStorageName, "elasticsearch", "metricstore")
+			httpAuth, authErr := s.resolveAuthenticator(host, cfg.Elasticsearch.Authentication, "elasticsearch metrics", metricStorageName)
+			if authErr != nil {
+				return authErr
+			}
 			metricStoreFactory, err = esmetrics.NewFactory(
 				ctx,
 				*cfg.Elasticsearch,
 				esTelset,
+				httpAuth,
 			)
 
 		case cfg.Opensearch != nil:
 			osTelset := telset
 			osTelset.Metrics = scopedMetricsFactory(metricStorageName, "opensearch", "metricstore")
-			metricStoreFactory, err = esmetrics.NewFactory(
-				ctx,
+			httpAuth, authErr := s.resolveAuthenticator(host, cfg.Opensearch.Authentication, "opensearch metrics", metricStorageName)
+			if authErr != nil {
+				return authErr
+			}
+			metricStoreFactory, err = esmetrics.NewFactory(ctx,
 				*cfg.Opensearch,
 				osTelset,
+				httpAuth,
 			)
+
 		default:
 			err = fmt.Errorf("no metric backend configuration provided for '%s'", metricStorageName)
 		}
@@ -305,11 +314,14 @@ func (s *storageExt) MetricStorageFactory(name string) (storage.MetricStoreFacto
 	return mf, ok
 }
 
-// getAuthenticator retrieves an HTTP authenticator extension from the host by name
-// authentication extension ID, or nil if no extension is configured.
+// getAuthenticator retrieves an HTTP authenticator extension from the host by name.
 func (*storageExt) getAuthenticator(host component.Host, authenticatorName string) (extensionauth.HTTPClient, error) {
+	if authenticatorName == "" {
+		return nil, nil
+	}
+
 	for id, ext := range host.GetExtensions() {
-		if id.Name() == authenticatorName {
+		if id.String() == authenticatorName || id.Name() == authenticatorName {
 			if httpAuth, ok := ext.(extensionauth.HTTPClient); ok {
 				return httpAuth, nil
 			}
@@ -318,3 +330,18 @@ func (*storageExt) getAuthenticator(host component.Host, authenticatorName strin
 	}
 	return nil, fmt.Errorf("authenticator extension '%s' not found", authenticatorName)
 }
+
+// resolveAuthenticator is a helper to resolve and validate HTTP authenticator for a backend
+func (s *storageExt) resolveAuthenticator(host component.Host, authCfg config.Authentication, backendType, backendName string) (extensionauth.HTTPClient, error) {
+	if authCfg.AuthenticatorID.String() == "" {
+		return nil, nil
+	}
+
+	httpAuth, err := s.getAuthenticator(host, authCfg.AuthenticatorID.String())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get HTTP authenticator for %s backend '%s': %w", backendType, backendName, err)
+	}
+	s.telset.Logger.Sugar().Infof("HTTP auth configured for %s backend '%s' with authenticator '%s'",
+		backendType, backendName, authCfg.AuthenticatorID.String())
+	return httpAuth, nil
+}