Add experimental exception_callback configuration (#1355)

Co-authored-by: David Montague <[email protected]>

Author: alexmojaki

logfire/_internal/config.py

@@ -66,6 +66,7 @@
 from logfire.version import VERSION
 
 from ..propagate import NoExtractTraceContextPropagator, WarnOnExtractTraceContextPropagator
+from ..types import ExceptionCallback
 from .client import InvalidProjectName, LogfireClient, ProjectAlreadyExists
 from .config_params import ParamManager, PydanticPluginRecordValues
 from .constants import (
@@ -182,6 +183,11 @@ class AdvancedOptions:
     log_record_processors: Sequence[LogRecordProcessor] = ()
     """Configuration for OpenTelemetry logging. This is experimental and may be removed."""
 
+    exception_callback: ExceptionCallback | None = None
+    """Callback function that is called when an exception is recorded on a span.
+
+    This is experimental and may be modified or removed."""
+
     def generate_base_url(self, token: str) -> str:
         if self.base_url is not None:
             return self.base_url

logfire/_internal/exporters/test.py

@@ -171,7 +171,8 @@ def build_attributes(
         k: process_attribute(k, v, strip_filepaths, fixed_line_number, strip_function_qualname, parse_json_attributes)
         for k, v in attributes.items()
     }
-    attributes.pop(ATTRIBUTES_EXCEPTION_FINGERPRINT_KEY, None)
+    if ATTRIBUTES_EXCEPTION_FINGERPRINT_KEY in attributes:
+        attributes[ATTRIBUTES_EXCEPTION_FINGERPRINT_KEY] = '0' * 64
     if 'telemetry.sdk.version' in attributes:
         attributes['telemetry.sdk.version'] = '0.0.0'
     return attributes

logfire/_internal/main.py

@@ -26,7 +26,7 @@
 from opentelemetry.context import Context
 from opentelemetry.metrics import CallbackT, Counter, Histogram, UpDownCounter
 from opentelemetry.sdk.trace import ReadableSpan, Span
-from opentelemetry.trace import SpanContext, Tracer
+from opentelemetry.trace import SpanContext
 from opentelemetry.util import types as otel_types
 from typing_extensions import LiteralString, ParamSpec
 
@@ -60,7 +60,12 @@
 )
 from .metrics import ProxyMeterProvider
 from .stack_info import get_user_stack_info
-from .tracer import ProxyTracerProvider, _LogfireWrappedSpan, record_exception, set_exception_status  # type: ignore
+from .tracer import (
+    ProxyTracerProvider,
+    _LogfireWrappedSpan,  # type: ignore
+    _ProxyTracer,  # type: ignore
+    set_exception_status,
+)
 from .utils import get_version, handle_internal_errors, log_internal_error, uniquify_sequence
 
 if TYPE_CHECKING:
@@ -157,14 +162,14 @@ def _meter(self):
         return self._meter_provider.get_meter(self._otel_scope, VERSION)
 
     @cached_property
-    def _logs_tracer(self) -> Tracer:
+    def _logs_tracer(self) -> _ProxyTracer:
         return self._get_tracer(is_span_tracer=False)
 
     @cached_property
-    def _spans_tracer(self) -> Tracer:
+    def _spans_tracer(self) -> _ProxyTracer:
         return self._get_tracer(is_span_tracer=True)
 
-    def _get_tracer(self, *, is_span_tracer: bool) -> Tracer:  # pragma: no cover
+    def _get_tracer(self, *, is_span_tracer: bool) -> _ProxyTracer:
         return self._tracer_provider.get_tracer(
             self._otel_scope,
             VERSION,
@@ -759,7 +764,7 @@ def log(
                 if isinstance(exc_info, tuple):
                     exc_info = exc_info[1]
                 if isinstance(exc_info, BaseException):
-                    record_exception(span, exc_info)
+                    span.record_exception(exc_info)
                     if otlp_attributes[ATTRIBUTES_LOG_LEVEL_NUM_KEY] >= LEVEL_NUMBERS['error']:  # type: ignore
                         # Set the status description to the exception message.
                         # OTEL only lets us set the description when the status code is ERROR,
@@ -2328,7 +2333,7 @@ def __init__(
         self,
         span_name: str,
         otlp_attributes: dict[str, otel_types.AttributeValue],
-        tracer: Tracer,
+        tracer: _ProxyTracer,
         json_schema_properties: JsonSchemaProperties,
         links: Sequence[tuple[SpanContext, otel_types.Attributes]],
     ) -> None:
@@ -2340,7 +2345,7 @@ def __init__(
 
         self._added_attributes = False
         self._token: None | Token[Context] = None
-        self._span: None | trace_api.Span = None
+        self._span: None | _LogfireWrappedSpan = None
 
     if not TYPE_CHECKING:  # pragma: no branch
 
@@ -2458,11 +2463,7 @@ def record_exception(
         if not self._span.is_recording():
             return
 
-        span = self._span
-        while isinstance(span, _LogfireWrappedSpan):
-            span = span.span
-        record_exception(
-            span,
+        self._span.record_exception(
             exception,
             attributes=attributes,
             timestamp=timestamp,

logfire/_internal/tracer.py

@@ -16,6 +16,7 @@
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import (
     ReadableSpan,
+    Span as SDKSpan,
     SpanProcessor,
     Tracer as SDKTracer,
     TracerProvider as SDKTracerProvider,
@@ -35,9 +36,13 @@
     ATTRIBUTES_VALIDATION_ERROR_KEY,
     log_level_attributes,
 )
-from .utils import canonicalize_exception_traceback, handle_internal_errors, sha256_string
+from .utils import handle_internal_errors, sha256_string
 
 if TYPE_CHECKING:
+    from starlette.exceptions import HTTPException
+    from typing_extensions import TypeIs
+
+    from ..types import ExceptionCallback
     from .config import LogfireConfig
 
 try:
@@ -146,6 +151,7 @@ class _LogfireWrappedSpan(trace_api.Span, ReadableSpan):
     ns_timestamp_generator: Callable[[], int]
     record_metrics: bool
     metrics: dict[str, SpanMetric] = field(default_factory=lambda: defaultdict(SpanMetric))
+    exception_callback: ExceptionCallback | None = None
 
     def __post_init__(self):
         OPEN_SPANS[self._open_spans_key()] = self
@@ -203,14 +209,21 @@ def record_exception(
         escaped: bool = False,
     ) -> None:
         timestamp = timestamp or self.ns_timestamp_generator()
-        record_exception(self.span, exception, attributes=attributes, timestamp=timestamp, escaped=escaped)
+        record_exception(
+            self.span,
+            exception,
+            attributes=attributes,
+            timestamp=timestamp,
+            escaped=escaped,
+            callback=self.exception_callback,
+        )
 
     def increment_metric(self, name: str, attributes: Mapping[str, otel_types.AttributeValue], value: float) -> None:
         if not (self.is_recording() and (self.record_metrics or name == 'operation.cost')):
             return
 
         self.metrics[name].increment(attributes, value)
-        if self.parent and (parent := OPEN_SPANS.get(_open_spans_key(self.parent))):
+        if parent := get_parent_span(self):
             parent.increment_metric(name, attributes, value)
 
     def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: Any) -> None:
@@ -225,6 +238,10 @@ def __getattr__(self, name: str) -> Any:
             return getattr(self.span, name)
 
 
+def get_parent_span(span: ReadableSpan) -> _LogfireWrappedSpan | None:
+    return span.parent and OPEN_SPANS.get(_open_spans_key(span.parent))
+
+
 def _open_spans_key(ctx: SpanContext) -> tuple[int, int]:
     return ctx.trace_id, ctx.span_id
 
@@ -257,10 +274,11 @@ def start_span(
         start_time: int | None = None,
         record_exception: bool = True,
         set_status_on_exception: bool = True,
-    ) -> Span:
+    ) -> _LogfireWrappedSpan:
         config = self.provider.config
         ns_timestamp_generator = config.advanced.ns_timestamp_generator
         record_metrics: bool = not isinstance(config.metrics, (bool, type(None))) and config.metrics.collect_in_spans
+        exception_callback = config.advanced.exception_callback
 
         start_time = start_time or ns_timestamp_generator()
 
@@ -289,6 +307,7 @@ def start_span(
             span,
             ns_timestamp_generator=ns_timestamp_generator,
             record_metrics=record_metrics,
+            exception_callback=exception_callback,
         )
 
     # This means that `with start_as_current_span(...):`
@@ -399,10 +418,23 @@ def record_exception(
     attributes: otel_types.Attributes = None,
     timestamp: int | None = None,
     escaped: bool = False,
+    callback: ExceptionCallback | None = None,
 ) -> None:
     """Similar to the OTEL SDK Span.record_exception method, with our own additions."""
-    if is_starlette_http_exception_400(exception):
-        span.set_attributes(log_level_attributes('warn'))
+    from ..types import ExceptionCallbackHelper
+
+    if is_starlette_http_exception(exception):
+        if 400 <= exception.status_code < 500:
+            # Don't mark 4xx HTTP exceptions as errors, they are expected to happen in normal operation.
+            # But do record them as warnings.
+            span.set_attributes(log_level_attributes('warn'))
+        elif exception.status_code >= 500:
+            # Set this as an error now for ExceptionCallbackHelper.create_issue to see,
+            # particularly so that if this is raised in a FastAPI pseudo_span and the event is marked with
+            # the recorded_by_logfire_fastapi it will still create an issue in this case.
+            # FastAPI will 'handle' this exception meaning it won't get recorded again by OTel.
+            set_exception_status(span, exception)
+            span.set_attributes(log_level_attributes('error'))
 
     # From https://opentelemetry.io/docs/specs/semconv/attributes-registry/exception/
     # `escaped=True` means that the exception is escaping the scope of the span.
@@ -412,7 +444,20 @@ def record_exception(
         set_exception_status(span, exception)
         span.set_attributes(log_level_attributes('error'))
 
-    attributes = {**(attributes or {})}
+    helper = ExceptionCallbackHelper(
+        span=cast(SDKSpan, span),
+        exception=exception,
+        event_attributes={**(attributes or {})},
+    )
+
+    if callback is not None:
+        with handle_internal_errors:
+            callback(helper)
+
+    if not helper._record_exception:  # type: ignore
+        return
+
+    attributes = helper.event_attributes
     if ValidationError is not None and isinstance(exception, ValidationError):
         # insert a more detailed breakdown of pydantic errors
         try:
@@ -430,7 +475,9 @@ def record_exception(
         stacktrace = ''.join(traceback.format_exception(type(exception), exception, exception.__traceback__))
         attributes['exception.stacktrace'] = stacktrace
 
-    span.set_attribute(ATTRIBUTES_EXCEPTION_FINGERPRINT_KEY, sha256_string(canonicalize_exception_traceback(exception)))
+    if helper.create_issue:
+        span.set_attribute(ATTRIBUTES_EXCEPTION_FINGERPRINT_KEY, sha256_string(helper.issue_fingerprint_source))
+
     span.record_exception(exception, attributes=attributes, timestamp=timestamp, escaped=escaped)
 
 
@@ -443,10 +490,10 @@ def set_exception_status(span: trace_api.Span, exception: BaseException):
     )
 
 
-def is_starlette_http_exception_400(exception: BaseException) -> bool:
+def is_starlette_http_exception(exception: BaseException) -> TypeIs[HTTPException]:
     if 'starlette.exceptions' not in sys.modules:  # pragma: no cover
         return False
 
     from starlette.exceptions import HTTPException
 
-    return isinstance(exception, HTTPException) and 400 <= exception.status_code < 500
+    return isinstance(exception, HTTPException)

logfire/sampling/_tail_sampling.py

@@ -11,58 +11,11 @@
 from typing_extensions import Self
 
 from logfire._internal.constants import (
-    ATTRIBUTES_LOG_LEVEL_NUM_KEY,
-    LEVEL_NUMBERS,
-    NUMBER_TO_LEVEL,
     ONE_SECOND_IN_NANOSECONDS,
     LevelName,
 )
 from logfire._internal.exporters.wrapper import WrapperSpanProcessor
-
-
-@dataclass
-class SpanLevel:
-    """A convenience class for comparing span/log levels.
-
-    Can be compared to log level names (strings) such as 'info' or 'error' using
-    `<`, `>`, `<=`, or `>=`, so e.g. `level >= 'error'` is valid.
-
-    Will raise an exception if compared to a non-string or an invalid level name.
-    """
-
-    number: int
-    """
-    The raw numeric value of the level. Higher values are more severe.
-    """
-
-    @property
-    def name(self) -> LevelName | None:
-        """The human-readable name of the level, or `None` if the number is invalid."""
-        return NUMBER_TO_LEVEL.get(self.number)
-
-    def __eq__(self, other: object):
-        if isinstance(other, int):
-            return self.number == other
-        if isinstance(other, str):
-            return self.name == other
-        if isinstance(other, SpanLevel):
-            return self.number == other.number
-        return NotImplemented
-
-    def __hash__(self):
-        return hash(self.number)
-
-    def __lt__(self, other: LevelName):
-        return self.number < LEVEL_NUMBERS[other]
-
-    def __gt__(self, other: LevelName):
-        return self.number > LEVEL_NUMBERS[other]
-
-    def __ge__(self, other: LevelName):
-        return self.number >= LEVEL_NUMBERS[other]
-
-    def __le__(self, other: LevelName):
-        return self.number <= LEVEL_NUMBERS[other]
+from logfire.types import SpanLevel
 
 
 @dataclass
@@ -111,11 +64,7 @@ class TailSamplingSpanInfo:
     @property
     def level(self) -> SpanLevel:
         """The log level of the span."""
-        attributes = self.span.attributes or {}
-        level = attributes.get(ATTRIBUTES_LOG_LEVEL_NUM_KEY)
-        if not isinstance(level, int):
-            level = LEVEL_NUMBERS['info']
-        return SpanLevel(level)
+        return SpanLevel.from_span(self.span)
 
     @property
     def duration(self) -> float: