• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 use crate::{OtelData, PreSampledTracer};
2 use once_cell::unsync;
3 use opentelemetry::{
4     trace::{self as otel, noop, OrderMap, TraceContextExt},
5     Context as OtelContext, Key, KeyValue, StringValue, Value,
6 };
7 use std::any::TypeId;
8 use std::fmt;
9 use std::marker;
10 use std::thread;
11 use std::time::{Instant, SystemTime};
12 use tracing_core::span::{self, Attributes, Id, Record};
13 use tracing_core::{field, Event, Subscriber};
14 #[cfg(feature = "tracing-log")]
15 use tracing_log::NormalizeEvent;
16 use tracing_subscriber::layer::Context;
17 use tracing_subscriber::registry::LookupSpan;
18 use tracing_subscriber::Layer;
19 
20 const SPAN_NAME_FIELD: &str = "otel.name";
21 const SPAN_KIND_FIELD: &str = "otel.kind";
22 const SPAN_STATUS_CODE_FIELD: &str = "otel.status_code";
23 const SPAN_STATUS_MESSAGE_FIELD: &str = "otel.status_message";
24 
25 const FIELD_EXCEPTION_MESSAGE: &str = "exception.message";
26 const FIELD_EXCEPTION_STACKTRACE: &str = "exception.stacktrace";
27 
28 /// An [OpenTelemetry] propagation layer for use in a project that uses
29 /// [tracing].
30 ///
31 /// [OpenTelemetry]: https://opentelemetry.io
32 /// [tracing]: https://github.com/tokio-rs/tracing
33 pub struct OpenTelemetryLayer<S, T> {
34     tracer: T,
35     location: bool,
36     tracked_inactivity: bool,
37     with_threads: bool,
38     exception_config: ExceptionFieldConfig,
39     get_context: WithContext,
40     _registry: marker::PhantomData<S>,
41 }
42 
43 impl<S> Default for OpenTelemetryLayer<S, noop::NoopTracer>
44 where
45     S: Subscriber + for<'span> LookupSpan<'span>,
46 {
default() -> Self47     fn default() -> Self {
48         OpenTelemetryLayer::new(noop::NoopTracer::new())
49     }
50 }
51 
52 /// Construct a layer to track spans via [OpenTelemetry].
53 ///
54 /// [OpenTelemetry]: https://opentelemetry.io
55 ///
56 /// # Examples
57 ///
58 /// ```rust,no_run
59 /// use tracing_subscriber::layer::SubscriberExt;
60 /// use tracing_subscriber::Registry;
61 ///
62 /// // Use the tracing subscriber `Registry`, or any other subscriber
63 /// // that impls `LookupSpan`
64 /// let subscriber = Registry::default().with(tracing_opentelemetry::layer());
65 /// # drop(subscriber);
66 /// ```
layer<S>() -> OpenTelemetryLayer<S, noop::NoopTracer> where S: Subscriber + for<'span> LookupSpan<'span>,67 pub fn layer<S>() -> OpenTelemetryLayer<S, noop::NoopTracer>
68 where
69     S: Subscriber + for<'span> LookupSpan<'span>,
70 {
71     OpenTelemetryLayer::default()
72 }
73 
74 // this function "remembers" the types of the subscriber so that we
75 // can downcast to something aware of them without knowing those
76 // types at the callsite.
77 //
78 // See https://github.com/tokio-rs/tracing/blob/4dad420ee1d4607bad79270c1520673fa6266a3d/tracing-error/src/layer.rs
79 pub(crate) struct WithContext(
80     fn(&tracing::Dispatch, &span::Id, f: &mut dyn FnMut(&mut OtelData, &dyn PreSampledTracer)),
81 );
82 
83 impl WithContext {
84     // This function allows a function to be called in the context of the
85     // "remembered" subscriber.
with_context<'a>( &self, dispatch: &'a tracing::Dispatch, id: &span::Id, mut f: impl FnMut(&mut OtelData, &dyn PreSampledTracer), )86     pub(crate) fn with_context<'a>(
87         &self,
88         dispatch: &'a tracing::Dispatch,
89         id: &span::Id,
90         mut f: impl FnMut(&mut OtelData, &dyn PreSampledTracer),
91     ) {
92         (self.0)(dispatch, id, &mut f)
93     }
94 }
95 
str_to_span_kind(s: &str) -> Option<otel::SpanKind>96 fn str_to_span_kind(s: &str) -> Option<otel::SpanKind> {
97     match s {
98         s if s.eq_ignore_ascii_case("server") => Some(otel::SpanKind::Server),
99         s if s.eq_ignore_ascii_case("client") => Some(otel::SpanKind::Client),
100         s if s.eq_ignore_ascii_case("producer") => Some(otel::SpanKind::Producer),
101         s if s.eq_ignore_ascii_case("consumer") => Some(otel::SpanKind::Consumer),
102         s if s.eq_ignore_ascii_case("internal") => Some(otel::SpanKind::Internal),
103         _ => None,
104     }
105 }
106 
str_to_status(s: &str) -> otel::Status107 fn str_to_status(s: &str) -> otel::Status {
108     match s {
109         s if s.eq_ignore_ascii_case("ok") => otel::Status::Ok,
110         s if s.eq_ignore_ascii_case("error") => otel::Status::error(""),
111         _ => otel::Status::Unset,
112     }
113 }
114 
115 struct SpanEventVisitor<'a, 'b> {
116     event_builder: &'a mut otel::Event,
117     span_builder: Option<&'b mut otel::SpanBuilder>,
118     exception_config: ExceptionFieldConfig,
119 }
120 
121 impl<'a, 'b> field::Visit for SpanEventVisitor<'a, 'b> {
122     /// Record events on the underlying OpenTelemetry [`Span`] from `bool` values.
123     ///
124     /// [`Span`]: opentelemetry::trace::Span
record_bool(&mut self, field: &field::Field, value: bool)125     fn record_bool(&mut self, field: &field::Field, value: bool) {
126         match field.name() {
127             "message" => self.event_builder.name = value.to_string().into(),
128             // Skip fields that are actually log metadata that have already been handled
129             #[cfg(feature = "tracing-log")]
130             name if name.starts_with("log.") => (),
131             name => {
132                 self.event_builder
133                     .attributes
134                     .push(KeyValue::new(name, value));
135             }
136         }
137     }
138 
139     /// Record events on the underlying OpenTelemetry [`Span`] from `f64` values.
140     ///
141     /// [`Span`]: opentelemetry::trace::Span
record_f64(&mut self, field: &field::Field, value: f64)142     fn record_f64(&mut self, field: &field::Field, value: f64) {
143         match field.name() {
144             "message" => self.event_builder.name = value.to_string().into(),
145             // Skip fields that are actually log metadata that have already been handled
146             #[cfg(feature = "tracing-log")]
147             name if name.starts_with("log.") => (),
148             name => {
149                 self.event_builder
150                     .attributes
151                     .push(KeyValue::new(name, value));
152             }
153         }
154     }
155 
156     /// Record events on the underlying OpenTelemetry [`Span`] from `i64` values.
157     ///
158     /// [`Span`]: opentelemetry::trace::Span
record_i64(&mut self, field: &field::Field, value: i64)159     fn record_i64(&mut self, field: &field::Field, value: i64) {
160         match field.name() {
161             "message" => self.event_builder.name = value.to_string().into(),
162             // Skip fields that are actually log metadata that have already been handled
163             #[cfg(feature = "tracing-log")]
164             name if name.starts_with("log.") => (),
165             name => {
166                 self.event_builder
167                     .attributes
168                     .push(KeyValue::new(name, value));
169             }
170         }
171     }
172 
173     /// Record events on the underlying OpenTelemetry [`Span`] from `&str` values.
174     ///
175     /// [`Span`]: opentelemetry::trace::Span
record_str(&mut self, field: &field::Field, value: &str)176     fn record_str(&mut self, field: &field::Field, value: &str) {
177         match field.name() {
178             "message" => self.event_builder.name = value.to_string().into(),
179             // Skip fields that are actually log metadata that have already been handled
180             #[cfg(feature = "tracing-log")]
181             name if name.starts_with("log.") => (),
182             name => {
183                 self.event_builder
184                     .attributes
185                     .push(KeyValue::new(name, value.to_string()));
186             }
187         }
188     }
189 
190     /// Record events on the underlying OpenTelemetry [`Span`] from values that
191     /// implement Debug.
192     ///
193     /// [`Span`]: opentelemetry::trace::Span
record_debug(&mut self, field: &field::Field, value: &dyn fmt::Debug)194     fn record_debug(&mut self, field: &field::Field, value: &dyn fmt::Debug) {
195         match field.name() {
196             "message" => self.event_builder.name = format!("{:?}", value).into(),
197             // Skip fields that are actually log metadata that have already been handled
198             #[cfg(feature = "tracing-log")]
199             name if name.starts_with("log.") => (),
200             name => {
201                 self.event_builder
202                     .attributes
203                     .push(KeyValue::new(name, format!("{:?}", value)));
204             }
205         }
206     }
207 
208     /// Set attributes on the underlying OpenTelemetry [`Span`] using a [`std::error::Error`]'s
209     /// [`std::fmt::Display`] implementation. Also adds the `source` chain as an extra field
210     ///
211     /// [`Span`]: opentelemetry::trace::Span
record_error( &mut self, field: &tracing_core::Field, value: &(dyn std::error::Error + 'static), )212     fn record_error(
213         &mut self,
214         field: &tracing_core::Field,
215         value: &(dyn std::error::Error + 'static),
216     ) {
217         let mut chain = Vec::new();
218         let mut next_err = value.source();
219 
220         while let Some(err) = next_err {
221             chain.push(StringValue::from(err.to_string()));
222             next_err = err.source();
223         }
224 
225         let error_msg = value.to_string();
226 
227         if self.exception_config.record {
228             self.event_builder
229                 .attributes
230                 .push(Key::new(FIELD_EXCEPTION_MESSAGE).string(error_msg.clone()));
231 
232             // NOTE: This is actually not the stacktrace of the exception. This is
233             // the "source chain". It represents the heirarchy of errors from the
234             // app level to the lowest level such as IO. It does not represent all
235             // of the callsites in the code that led to the error happening.
236             // `std::error::Error::backtrace` is a nightly-only API and cannot be
237             // used here until the feature is stabilized.
238             self.event_builder
239                 .attributes
240                 .push(Key::new(FIELD_EXCEPTION_STACKTRACE).array(chain.clone()));
241         }
242 
243         if self.exception_config.propagate {
244             if let Some(span) = &mut self.span_builder {
245                 if let Some(attrs) = span.attributes.as_mut() {
246                     attrs.insert(Key::new(FIELD_EXCEPTION_MESSAGE), error_msg.clone().into());
247 
248                     // NOTE: This is actually not the stacktrace of the exception. This is
249                     // the "source chain". It represents the heirarchy of errors from the
250                     // app level to the lowest level such as IO. It does not represent all
251                     // of the callsites in the code that led to the error happening.
252                     // `std::error::Error::backtrace` is a nightly-only API and cannot be
253                     // used here until the feature is stabilized.
254                     attrs.insert(
255                         Key::new(FIELD_EXCEPTION_STACKTRACE),
256                         Value::Array(chain.clone().into()),
257                     );
258                 }
259             }
260         }
261 
262         self.event_builder
263             .attributes
264             .push(Key::new(field.name()).string(error_msg));
265         self.event_builder
266             .attributes
267             .push(Key::new(format!("{}.chain", field.name())).array(chain));
268     }
269 }
270 
271 /// Control over opentelemetry conventional exception fields
272 #[derive(Clone, Copy)]
273 struct ExceptionFieldConfig {
274     /// If an error value is recorded on an event/span, should the otel fields
275     /// be added
276     record: bool,
277 
278     /// If an error value is recorded on an event, should the otel fields be
279     /// added to the corresponding span
280     propagate: bool,
281 }
282 
283 struct SpanAttributeVisitor<'a> {
284     span_builder: &'a mut otel::SpanBuilder,
285     exception_config: ExceptionFieldConfig,
286 }
287 
288 impl<'a> SpanAttributeVisitor<'a> {
record(&mut self, attribute: KeyValue)289     fn record(&mut self, attribute: KeyValue) {
290         debug_assert!(self.span_builder.attributes.is_some());
291         if let Some(v) = self.span_builder.attributes.as_mut() {
292             v.insert(attribute.key, attribute.value);
293         }
294     }
295 }
296 
297 impl<'a> field::Visit for SpanAttributeVisitor<'a> {
298     /// Set attributes on the underlying OpenTelemetry [`Span`] from `bool` values.
299     ///
300     /// [`Span`]: opentelemetry::trace::Span
record_bool(&mut self, field: &field::Field, value: bool)301     fn record_bool(&mut self, field: &field::Field, value: bool) {
302         self.record(KeyValue::new(field.name(), value));
303     }
304 
305     /// Set attributes on the underlying OpenTelemetry [`Span`] from `f64` values.
306     ///
307     /// [`Span`]: opentelemetry::trace::Span
record_f64(&mut self, field: &field::Field, value: f64)308     fn record_f64(&mut self, field: &field::Field, value: f64) {
309         self.record(KeyValue::new(field.name(), value));
310     }
311 
312     /// Set attributes on the underlying OpenTelemetry [`Span`] from `i64` values.
313     ///
314     /// [`Span`]: opentelemetry::trace::Span
record_i64(&mut self, field: &field::Field, value: i64)315     fn record_i64(&mut self, field: &field::Field, value: i64) {
316         self.record(KeyValue::new(field.name(), value));
317     }
318 
319     /// Set attributes on the underlying OpenTelemetry [`Span`] from `&str` values.
320     ///
321     /// [`Span`]: opentelemetry::trace::Span
record_str(&mut self, field: &field::Field, value: &str)322     fn record_str(&mut self, field: &field::Field, value: &str) {
323         match field.name() {
324             SPAN_NAME_FIELD => self.span_builder.name = value.to_string().into(),
325             SPAN_KIND_FIELD => self.span_builder.span_kind = str_to_span_kind(value),
326             SPAN_STATUS_CODE_FIELD => self.span_builder.status = str_to_status(value),
327             SPAN_STATUS_MESSAGE_FIELD => {
328                 self.span_builder.status = otel::Status::error(value.to_string())
329             }
330             _ => self.record(KeyValue::new(field.name(), value.to_string())),
331         }
332     }
333 
334     /// Set attributes on the underlying OpenTelemetry [`Span`] from values that
335     /// implement Debug.
336     ///
337     /// [`Span`]: opentelemetry::trace::Span
record_debug(&mut self, field: &field::Field, value: &dyn fmt::Debug)338     fn record_debug(&mut self, field: &field::Field, value: &dyn fmt::Debug) {
339         match field.name() {
340             SPAN_NAME_FIELD => self.span_builder.name = format!("{:?}", value).into(),
341             SPAN_KIND_FIELD => {
342                 self.span_builder.span_kind = str_to_span_kind(&format!("{:?}", value))
343             }
344             SPAN_STATUS_CODE_FIELD => {
345                 self.span_builder.status = str_to_status(&format!("{:?}", value))
346             }
347             SPAN_STATUS_MESSAGE_FIELD => {
348                 self.span_builder.status = otel::Status::error(format!("{:?}", value))
349             }
350             _ => self.record(Key::new(field.name()).string(format!("{:?}", value))),
351         }
352     }
353 
354     /// Set attributes on the underlying OpenTelemetry [`Span`] using a [`std::error::Error`]'s
355     /// [`std::fmt::Display`] implementation. Also adds the `source` chain as an extra field
356     ///
357     /// [`Span`]: opentelemetry::trace::Span
record_error( &mut self, field: &tracing_core::Field, value: &(dyn std::error::Error + 'static), )358     fn record_error(
359         &mut self,
360         field: &tracing_core::Field,
361         value: &(dyn std::error::Error + 'static),
362     ) {
363         let mut chain = Vec::new();
364         let mut next_err = value.source();
365 
366         while let Some(err) = next_err {
367             chain.push(StringValue::from(err.to_string()));
368             next_err = err.source();
369         }
370 
371         let error_msg = value.to_string();
372 
373         if self.exception_config.record {
374             self.record(Key::new(FIELD_EXCEPTION_MESSAGE).string(error_msg.clone()));
375 
376             // NOTE: This is actually not the stacktrace of the exception. This is
377             // the "source chain". It represents the heirarchy of errors from the
378             // app level to the lowest level such as IO. It does not represent all
379             // of the callsites in the code that led to the error happening.
380             // `std::error::Error::backtrace` is a nightly-only API and cannot be
381             // used here until the feature is stabilized.
382             self.record(Key::new(FIELD_EXCEPTION_STACKTRACE).array(chain.clone()));
383         }
384 
385         self.record(Key::new(field.name()).string(error_msg));
386         self.record(Key::new(format!("{}.chain", field.name())).array(chain));
387     }
388 }
389 
390 impl<S, T> OpenTelemetryLayer<S, T>
391 where
392     S: Subscriber + for<'span> LookupSpan<'span>,
393     T: otel::Tracer + PreSampledTracer + 'static,
394 {
395     /// Set the [`Tracer`] that this layer will use to produce and track
396     /// OpenTelemetry [`Span`]s.
397     ///
398     /// [`Tracer`]: opentelemetry::trace::Tracer
399     /// [`Span`]: opentelemetry::trace::Span
400     ///
401     /// # Examples
402     ///
403     /// ```no_run
404     /// use tracing_opentelemetry::OpenTelemetryLayer;
405     /// use tracing_subscriber::layer::SubscriberExt;
406     /// use tracing_subscriber::Registry;
407     ///
408     /// // Create a jaeger exporter pipeline for a `trace_demo` service.
409     /// let tracer = opentelemetry_jaeger::new_agent_pipeline()
410     ///     .with_service_name("trace_demo")
411     ///     .install_simple()
412     ///     .expect("Error initializing Jaeger exporter");
413     ///
414     /// // Create a layer with the configured tracer
415     /// let otel_layer = OpenTelemetryLayer::new(tracer);
416     ///
417     /// // Use the tracing subscriber `Registry`, or any other subscriber
418     /// // that impls `LookupSpan`
419     /// let subscriber = Registry::default().with(otel_layer);
420     /// # drop(subscriber);
421     /// ```
new(tracer: T) -> Self422     pub fn new(tracer: T) -> Self {
423         OpenTelemetryLayer {
424             tracer,
425             location: true,
426             tracked_inactivity: true,
427             with_threads: true,
428             exception_config: ExceptionFieldConfig {
429                 record: false,
430                 propagate: false,
431             },
432             get_context: WithContext(Self::get_context),
433             _registry: marker::PhantomData,
434         }
435     }
436 
437     /// Set the [`Tracer`] that this layer will use to produce and track
438     /// OpenTelemetry [`Span`]s.
439     ///
440     /// [`Tracer`]: opentelemetry::trace::Tracer
441     /// [`Span`]: opentelemetry::trace::Span
442     ///
443     /// # Examples
444     ///
445     /// ```no_run
446     /// use tracing_subscriber::layer::SubscriberExt;
447     /// use tracing_subscriber::Registry;
448     ///
449     /// // Create a jaeger exporter pipeline for a `trace_demo` service.
450     /// let tracer = opentelemetry_jaeger::new_agent_pipeline()
451     ///     .with_service_name("trace_demo")
452     ///     .install_simple()
453     ///     .expect("Error initializing Jaeger exporter");
454     ///
455     /// // Create a layer with the configured tracer
456     /// let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer);
457     ///
458     /// // Use the tracing subscriber `Registry`, or any other subscriber
459     /// // that impls `LookupSpan`
460     /// let subscriber = Registry::default().with(otel_layer);
461     /// # drop(subscriber);
462     /// ```
with_tracer<Tracer>(self, tracer: Tracer) -> OpenTelemetryLayer<S, Tracer> where Tracer: otel::Tracer + PreSampledTracer + 'static,463     pub fn with_tracer<Tracer>(self, tracer: Tracer) -> OpenTelemetryLayer<S, Tracer>
464     where
465         Tracer: otel::Tracer + PreSampledTracer + 'static,
466     {
467         OpenTelemetryLayer {
468             tracer,
469             location: self.location,
470             tracked_inactivity: self.tracked_inactivity,
471             with_threads: self.with_threads,
472             exception_config: self.exception_config,
473             get_context: WithContext(OpenTelemetryLayer::<S, Tracer>::get_context),
474             _registry: self._registry,
475         }
476     }
477 
478     /// Sets whether or not span and event metadata should include OpenTelemetry
479     /// exception fields such as `exception.message` and `exception.backtrace`
480     /// when an `Error` value is recorded. If multiple error values are recorded
481     /// on the same span/event, only the most recently recorded error value will
482     /// show up under these fields.
483     ///
484     /// These attributes follow the [OpenTelemetry semantic conventions for
485     /// exceptions][conv].
486     ///
487     /// By default, these attributes are not recorded.
488     ///
489     /// [conv]: https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/exceptions/
with_exception_fields(self, exception_fields: bool) -> Self490     pub fn with_exception_fields(self, exception_fields: bool) -> Self {
491         Self {
492             exception_config: ExceptionFieldConfig {
493                 record: exception_fields,
494                 ..self.exception_config
495             },
496             ..self
497         }
498     }
499 
500     /// Sets whether or not reporting an `Error` value on an event will
501     /// propagate the OpenTelemetry exception fields such as `exception.message`
502     /// and `exception.backtrace` to the corresponding span. You do not need to
503     /// enable `with_exception_fields` in order to enable this. If multiple
504     /// error values are recorded on the same span/event, only the most recently
505     /// recorded error value will show up under these fields.
506     ///
507     /// These attributes follow the [OpenTelemetry semantic conventions for
508     /// exceptions][conv].
509     ///
510     /// By default, these attributes are not propagated to the span.
511     ///
512     /// [conv]: https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/exceptions/
with_exception_field_propagation(self, exception_field_propagation: bool) -> Self513     pub fn with_exception_field_propagation(self, exception_field_propagation: bool) -> Self {
514         Self {
515             exception_config: ExceptionFieldConfig {
516                 propagate: exception_field_propagation,
517                 ..self.exception_config
518             },
519             ..self
520         }
521     }
522 
523     /// Sets whether or not span and event metadata should include OpenTelemetry
524     /// attributes with location information, such as the file, module and line number.
525     ///
526     /// These attributes follow the [OpenTelemetry semantic conventions for
527     /// source locations][conv].
528     ///
529     /// By default, locations are enabled.
530     ///
531     /// [conv]: https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/#source-code-attributes
with_location(self, location: bool) -> Self532     pub fn with_location(self, location: bool) -> Self {
533         Self { location, ..self }
534     }
535 
536     /// Sets whether or not span and event metadata should include OpenTelemetry
537     /// attributes with location information, such as the file, module and line number.
538     ///
539     /// These attributes follow the [OpenTelemetry semantic conventions for
540     /// source locations][conv].
541     ///
542     /// By default, locations are enabled.
543     ///
544     /// [conv]: https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/#source-code-attributes
545     #[deprecated(
546         since = "0.17.3",
547         note = "renamed to `OpenTelemetrySubscriber::with_location`"
548     )]
with_event_location(self, event_location: bool) -> Self549     pub fn with_event_location(self, event_location: bool) -> Self {
550         Self {
551             location: event_location,
552             ..self
553         }
554     }
555 
556     /// Sets whether or not spans metadata should include the _busy time_
557     /// (total time for which it was entered), and _idle time_ (total time
558     /// the span existed but was not entered).
with_tracked_inactivity(self, tracked_inactivity: bool) -> Self559     pub fn with_tracked_inactivity(self, tracked_inactivity: bool) -> Self {
560         Self {
561             tracked_inactivity,
562             ..self
563         }
564     }
565 
566     /// Sets whether or not spans record additional attributes for the thread
567     /// name and thread ID of the thread they were created on, following the
568     /// [OpenTelemetry semantic conventions for threads][conv].
569     ///
570     /// By default, thread attributes are enabled.
571     ///
572     /// [conv]: https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/#general-thread-attributes
with_threads(self, threads: bool) -> Self573     pub fn with_threads(self, threads: bool) -> Self {
574         Self {
575             with_threads: threads,
576             ..self
577         }
578     }
579 
580     /// Retrieve the parent OpenTelemetry [`Context`] from the current tracing
581     /// [`span`] through the [`Registry`]. This [`Context`] links spans to their
582     /// parent for proper hierarchical visualization.
583     ///
584     /// [`Context`]: opentelemetry::Context
585     /// [`span`]: tracing::Span
586     /// [`Registry`]: tracing_subscriber::Registry
parent_context(&self, attrs: &Attributes<'_>, ctx: &Context<'_, S>) -> OtelContext587     fn parent_context(&self, attrs: &Attributes<'_>, ctx: &Context<'_, S>) -> OtelContext {
588         // If a span is specified, it _should_ exist in the underlying `Registry`.
589         if let Some(parent) = attrs.parent() {
590             let span = ctx.span(parent).expect("Span not found, this is a bug");
591             let mut extensions = span.extensions_mut();
592             extensions
593                 .get_mut::<OtelData>()
594                 .map(|builder| self.tracer.sampled_context(builder))
595                 .unwrap_or_default()
596         // Else if the span is inferred from context, look up any available current span.
597         } else if attrs.is_contextual() {
598             ctx.lookup_current()
599                 .and_then(|span| {
600                     let mut extensions = span.extensions_mut();
601                     extensions
602                         .get_mut::<OtelData>()
603                         .map(|builder| self.tracer.sampled_context(builder))
604                 })
605                 .unwrap_or_else(OtelContext::current)
606         // Explicit root spans should have no parent context.
607         } else {
608             OtelContext::new()
609         }
610     }
611 
get_context( dispatch: &tracing::Dispatch, id: &span::Id, f: &mut dyn FnMut(&mut OtelData, &dyn PreSampledTracer), )612     fn get_context(
613         dispatch: &tracing::Dispatch,
614         id: &span::Id,
615         f: &mut dyn FnMut(&mut OtelData, &dyn PreSampledTracer),
616     ) {
617         let subscriber = dispatch
618             .downcast_ref::<S>()
619             .expect("subscriber should downcast to expected type; this is a bug!");
620         let span = subscriber
621             .span(id)
622             .expect("registry should have a span for the current ID");
623         let layer = dispatch
624             .downcast_ref::<OpenTelemetryLayer<S, T>>()
625             .expect("layer should downcast to expected type; this is a bug!");
626 
627         let mut extensions = span.extensions_mut();
628         if let Some(builder) = extensions.get_mut::<OtelData>() {
629             f(builder, &layer.tracer);
630         }
631     }
632 
extra_span_attrs(&self) -> usize633     fn extra_span_attrs(&self) -> usize {
634         let mut extra_attrs = 0;
635         if self.location {
636             extra_attrs += 3;
637         }
638         if self.with_threads {
639             extra_attrs += 2;
640         }
641         extra_attrs
642     }
643 }
644 
645 thread_local! {
646     static THREAD_ID: unsync::Lazy<u64> = unsync::Lazy::new(|| {
647         // OpenTelemetry's semantic conventions require the thread ID to be
648         // recorded as an integer, but `std::thread::ThreadId` does not expose
649         // the integer value on stable, so we have to convert it to a `usize` by
650         // parsing it. Since this requires allocating a `String`, store it in a
651         // thread local so we only have to do this once.
652         // TODO(eliza): once `std::thread::ThreadId::as_u64` is stabilized
653         // (https://github.com/rust-lang/rust/issues/67939), just use that.
654         thread_id_integer(thread::current().id())
655     });
656 }
657 
658 impl<S, T> Layer<S> for OpenTelemetryLayer<S, T>
659 where
660     S: Subscriber + for<'span> LookupSpan<'span>,
661     T: otel::Tracer + PreSampledTracer + 'static,
662 {
663     /// Creates an [OpenTelemetry `Span`] for the corresponding [tracing `Span`].
664     ///
665     /// [OpenTelemetry `Span`]: opentelemetry::trace::Span
666     /// [tracing `Span`]: tracing::Span
on_new_span(&self, attrs: &Attributes<'_>, id: &span::Id, ctx: Context<'_, S>)667     fn on_new_span(&self, attrs: &Attributes<'_>, id: &span::Id, ctx: Context<'_, S>) {
668         let span = ctx.span(id).expect("Span not found, this is a bug");
669         let mut extensions = span.extensions_mut();
670 
671         if self.tracked_inactivity && extensions.get_mut::<Timings>().is_none() {
672             extensions.insert(Timings::new());
673         }
674 
675         let parent_cx = self.parent_context(attrs, &ctx);
676         let mut builder = self
677             .tracer
678             .span_builder(attrs.metadata().name())
679             .with_start_time(SystemTime::now())
680             // Eagerly assign span id so children have stable parent id
681             .with_span_id(self.tracer.new_span_id());
682 
683         // Record new trace id if there is no active parent span
684         if !parent_cx.has_active_span() {
685             builder.trace_id = Some(self.tracer.new_trace_id());
686         }
687 
688         let builder_attrs = builder.attributes.get_or_insert(OrderMap::with_capacity(
689             attrs.fields().len() + self.extra_span_attrs(),
690         ));
691 
692         if self.location {
693             let meta = attrs.metadata();
694 
695             if let Some(filename) = meta.file() {
696                 builder_attrs.insert("code.filepath".into(), filename.into());
697             }
698 
699             if let Some(module) = meta.module_path() {
700                 builder_attrs.insert("code.namespace".into(), module.into());
701             }
702 
703             if let Some(line) = meta.line() {
704                 builder_attrs.insert("code.lineno".into(), (line as i64).into());
705             }
706         }
707 
708         if self.with_threads {
709             THREAD_ID.with(|id| builder_attrs.insert("thread.id".into(), (**id as i64).into()));
710             if let Some(name) = std::thread::current().name() {
711                 // TODO(eliza): it's a bummer that we have to allocate here, but
712                 // we can't easily get the string as a `static`. it would be
713                 // nice if `opentelemetry` could also take `Arc<str>`s as
714                 // `String` values...
715                 builder_attrs.insert("thread.name".into(), name.to_owned().into());
716             }
717         }
718 
719         attrs.record(&mut SpanAttributeVisitor {
720             span_builder: &mut builder,
721             exception_config: self.exception_config,
722         });
723         extensions.insert(OtelData { builder, parent_cx });
724     }
725 
on_enter(&self, id: &span::Id, ctx: Context<'_, S>)726     fn on_enter(&self, id: &span::Id, ctx: Context<'_, S>) {
727         if !self.tracked_inactivity {
728             return;
729         }
730 
731         let span = ctx.span(id).expect("Span not found, this is a bug");
732         let mut extensions = span.extensions_mut();
733 
734         if let Some(timings) = extensions.get_mut::<Timings>() {
735             let now = Instant::now();
736             timings.idle += (now - timings.last).as_nanos() as i64;
737             timings.last = now;
738         }
739     }
740 
on_exit(&self, id: &span::Id, ctx: Context<'_, S>)741     fn on_exit(&self, id: &span::Id, ctx: Context<'_, S>) {
742         if !self.tracked_inactivity {
743             return;
744         }
745 
746         let span = ctx.span(id).expect("Span not found, this is a bug");
747         let mut extensions = span.extensions_mut();
748 
749         if let Some(timings) = extensions.get_mut::<Timings>() {
750             let now = Instant::now();
751             timings.busy += (now - timings.last).as_nanos() as i64;
752             timings.last = now;
753         }
754     }
755 
756     /// Record OpenTelemetry [`attributes`] for the given values.
757     ///
758     /// [`attributes`]: opentelemetry::trace::SpanBuilder::attributes
on_record(&self, id: &Id, values: &Record<'_>, ctx: Context<'_, S>)759     fn on_record(&self, id: &Id, values: &Record<'_>, ctx: Context<'_, S>) {
760         let span = ctx.span(id).expect("Span not found, this is a bug");
761         let mut extensions = span.extensions_mut();
762         if let Some(data) = extensions.get_mut::<OtelData>() {
763             values.record(&mut SpanAttributeVisitor {
764                 span_builder: &mut data.builder,
765                 exception_config: self.exception_config,
766             });
767         }
768     }
769 
on_follows_from(&self, id: &Id, follows: &Id, ctx: Context<S>)770     fn on_follows_from(&self, id: &Id, follows: &Id, ctx: Context<S>) {
771         let span = ctx.span(id).expect("Span not found, this is a bug");
772         let mut extensions = span.extensions_mut();
773         let data = extensions
774             .get_mut::<OtelData>()
775             .expect("Missing otel data span extensions");
776 
777         let follows_span = ctx
778             .span(follows)
779             .expect("Span to follow not found, this is a bug");
780         let mut follows_extensions = follows_span.extensions_mut();
781         let follows_data = follows_extensions
782             .get_mut::<OtelData>()
783             .expect("Missing otel data span extensions");
784 
785         let follows_context = self
786             .tracer
787             .sampled_context(follows_data)
788             .span()
789             .span_context()
790             .clone();
791         let follows_link = otel::Link::new(follows_context, Vec::new());
792         if let Some(ref mut links) = data.builder.links {
793             links.push(follows_link);
794         } else {
795             data.builder.links = Some(vec![follows_link]);
796         }
797     }
798 
799     /// Records OpenTelemetry [`Event`] data on event.
800     ///
801     /// Note: an [`ERROR`]-level event will also set the OpenTelemetry span status code to
802     /// [`Error`], signaling that an error has occurred.
803     ///
804     /// [`Event`]: opentelemetry::trace::Event
805     /// [`ERROR`]: tracing::Level::ERROR
806     /// [`Error`]: opentelemetry::trace::StatusCode::Error
on_event(&self, event: &Event<'_>, ctx: Context<'_, S>)807     fn on_event(&self, event: &Event<'_>, ctx: Context<'_, S>) {
808         // Ignore events that have no explicit parent set *and* are not in the context of a span
809         if let Some(span) = ctx.event_span(event) {
810             // Performing read operations before getting a write lock to avoid a deadlock
811             // See https://github.com/tokio-rs/tracing/issues/763
812             #[cfg(feature = "tracing-log")]
813             let normalized_meta = event.normalized_metadata();
814             #[cfg(feature = "tracing-log")]
815             let meta = normalized_meta.as_ref().unwrap_or_else(|| event.metadata());
816             #[cfg(not(feature = "tracing-log"))]
817             let meta = event.metadata();
818 
819             let target = Key::new("target");
820 
821             #[cfg(feature = "tracing-log")]
822             let target = if normalized_meta.is_some() {
823                 target.string(meta.target().to_owned())
824             } else {
825                 target.string(event.metadata().target())
826             };
827 
828             #[cfg(not(feature = "tracing-log"))]
829             let target = target.string(meta.target());
830 
831             let mut extensions = span.extensions_mut();
832             let span_builder = extensions
833                 .get_mut::<OtelData>()
834                 .map(|data| &mut data.builder);
835 
836             let mut otel_event = otel::Event::new(
837                 String::new(),
838                 SystemTime::now(),
839                 vec![Key::new("level").string(meta.level().as_str()), target],
840                 0,
841             );
842             event.record(&mut SpanEventVisitor {
843                 event_builder: &mut otel_event,
844                 span_builder,
845                 exception_config: self.exception_config,
846             });
847 
848             if let Some(OtelData { builder, .. }) = extensions.get_mut::<OtelData>() {
849                 if builder.status == otel::Status::Unset
850                     && *meta.level() == tracing_core::Level::ERROR
851                 {
852                     builder.status = otel::Status::error("")
853                 }
854 
855                 if self.location {
856                     #[cfg(not(feature = "tracing-log"))]
857                     let normalized_meta: Option<tracing_core::Metadata<'_>> = None;
858                     let (file, module) = match &normalized_meta {
859                         Some(meta) => (
860                             meta.file().map(|s| Value::from(s.to_owned())),
861                             meta.module_path().map(|s| Value::from(s.to_owned())),
862                         ),
863                         None => (
864                             event.metadata().file().map(Value::from),
865                             event.metadata().module_path().map(Value::from),
866                         ),
867                     };
868 
869                     if let Some(file) = file {
870                         otel_event
871                             .attributes
872                             .push(KeyValue::new("code.filepath", file));
873                     }
874                     if let Some(module) = module {
875                         otel_event
876                             .attributes
877                             .push(KeyValue::new("code.namespace", module));
878                     }
879                     if let Some(line) = meta.line() {
880                         otel_event
881                             .attributes
882                             .push(KeyValue::new("code.lineno", line as i64));
883                     }
884                 }
885 
886                 if let Some(ref mut events) = builder.events {
887                     events.push(otel_event);
888                 } else {
889                     builder.events = Some(vec![otel_event]);
890                 }
891             }
892         };
893     }
894 
895     /// Exports an OpenTelemetry [`Span`] on close.
896     ///
897     /// [`Span`]: opentelemetry::trace::Span
on_close(&self, id: span::Id, ctx: Context<'_, S>)898     fn on_close(&self, id: span::Id, ctx: Context<'_, S>) {
899         let span = ctx.span(&id).expect("Span not found, this is a bug");
900         let mut extensions = span.extensions_mut();
901 
902         if let Some(OtelData {
903             mut builder,
904             parent_cx,
905         }) = extensions.remove::<OtelData>()
906         {
907             if self.tracked_inactivity {
908                 // Append busy/idle timings when enabled.
909                 if let Some(timings) = extensions.get_mut::<Timings>() {
910                     let busy_ns = Key::new("busy_ns");
911                     let idle_ns = Key::new("idle_ns");
912 
913                     let attributes = builder
914                         .attributes
915                         .get_or_insert_with(|| OrderMap::with_capacity(2));
916                     attributes.insert(busy_ns, timings.busy.into());
917                     attributes.insert(idle_ns, timings.idle.into());
918                 }
919             }
920 
921             // Assign end time, build and start span, drop span to export
922             builder
923                 .with_end_time(SystemTime::now())
924                 .start_with_context(&self.tracer, &parent_cx);
925         }
926     }
927 
928     // SAFETY: this is safe because the `WithContext` function pointer is valid
929     // for the lifetime of `&self`.
downcast_raw(&self, id: TypeId) -> Option<*const ()>930     unsafe fn downcast_raw(&self, id: TypeId) -> Option<*const ()> {
931         match id {
932             id if id == TypeId::of::<Self>() => Some(self as *const _ as *const ()),
933             id if id == TypeId::of::<WithContext>() => {
934                 Some(&self.get_context as *const _ as *const ())
935             }
936             _ => None,
937         }
938     }
939 }
940 
941 struct Timings {
942     idle: i64,
943     busy: i64,
944     last: Instant,
945 }
946 
947 impl Timings {
new() -> Self948     fn new() -> Self {
949         Self {
950             idle: 0,
951             busy: 0,
952             last: Instant::now(),
953         }
954     }
955 }
956 
thread_id_integer(id: thread::ThreadId) -> u64957 fn thread_id_integer(id: thread::ThreadId) -> u64 {
958     let thread_id = format!("{:?}", id);
959     thread_id
960         .trim_start_matches("ThreadId(")
961         .trim_end_matches(')')
962         .parse::<u64>()
963         .expect("thread ID should parse as an integer")
964 }
965 
966 #[cfg(test)]
967 mod tests {
968     use super::*;
969     use crate::OtelData;
970     use opentelemetry::{
971         trace::{noop, TraceFlags},
972         StringValue,
973     };
974     use std::{
975         borrow::Cow,
976         collections::HashMap,
977         error::Error,
978         fmt::Display,
979         sync::{Arc, Mutex},
980         thread,
981         time::SystemTime,
982     };
983     use tracing_subscriber::prelude::*;
984 
985     #[derive(Debug, Clone)]
986     struct TestTracer(Arc<Mutex<Option<OtelData>>>);
987     impl otel::Tracer for TestTracer {
988         type Span = noop::NoopSpan;
start_with_context<T>(&self, _name: T, _context: &OtelContext) -> Self::Span where T: Into<Cow<'static, str>>,989         fn start_with_context<T>(&self, _name: T, _context: &OtelContext) -> Self::Span
990         where
991             T: Into<Cow<'static, str>>,
992         {
993             noop::NoopSpan::new()
994         }
span_builder<T>(&self, name: T) -> otel::SpanBuilder where T: Into<Cow<'static, str>>,995         fn span_builder<T>(&self, name: T) -> otel::SpanBuilder
996         where
997             T: Into<Cow<'static, str>>,
998         {
999             otel::SpanBuilder::from_name(name)
1000         }
build_with_context( &self, builder: otel::SpanBuilder, parent_cx: &OtelContext, ) -> Self::Span1001         fn build_with_context(
1002             &self,
1003             builder: otel::SpanBuilder,
1004             parent_cx: &OtelContext,
1005         ) -> Self::Span {
1006             *self.0.lock().unwrap() = Some(OtelData {
1007                 builder,
1008                 parent_cx: parent_cx.clone(),
1009             });
1010             noop::NoopSpan::new()
1011         }
1012     }
1013 
1014     impl PreSampledTracer for TestTracer {
sampled_context(&self, _builder: &mut crate::OtelData) -> OtelContext1015         fn sampled_context(&self, _builder: &mut crate::OtelData) -> OtelContext {
1016             OtelContext::new()
1017         }
new_trace_id(&self) -> otel::TraceId1018         fn new_trace_id(&self) -> otel::TraceId {
1019             otel::TraceId::INVALID
1020         }
new_span_id(&self) -> otel::SpanId1021         fn new_span_id(&self) -> otel::SpanId {
1022             otel::SpanId::INVALID
1023         }
1024     }
1025 
1026     impl TestTracer {
with_data<T>(&self, f: impl FnOnce(&OtelData) -> T) -> T1027         fn with_data<T>(&self, f: impl FnOnce(&OtelData) -> T) -> T {
1028             let lock = self.0.lock().unwrap();
1029             let data = lock.as_ref().expect("no span data has been recorded yet");
1030             f(data)
1031         }
1032     }
1033 
1034     #[derive(Debug, Clone)]
1035     struct TestSpan(otel::SpanContext);
1036     impl otel::Span for TestSpan {
add_event_with_timestamp<T: Into<Cow<'static, str>>>( &mut self, _: T, _: SystemTime, _: Vec<KeyValue>, )1037         fn add_event_with_timestamp<T: Into<Cow<'static, str>>>(
1038             &mut self,
1039             _: T,
1040             _: SystemTime,
1041             _: Vec<KeyValue>,
1042         ) {
1043         }
span_context(&self) -> &otel::SpanContext1044         fn span_context(&self) -> &otel::SpanContext {
1045             &self.0
1046         }
is_recording(&self) -> bool1047         fn is_recording(&self) -> bool {
1048             false
1049         }
set_attribute(&mut self, _attribute: KeyValue)1050         fn set_attribute(&mut self, _attribute: KeyValue) {}
set_status(&mut self, _status: otel::Status)1051         fn set_status(&mut self, _status: otel::Status) {}
update_name<T: Into<Cow<'static, str>>>(&mut self, _new_name: T)1052         fn update_name<T: Into<Cow<'static, str>>>(&mut self, _new_name: T) {}
end_with_timestamp(&mut self, _timestamp: SystemTime)1053         fn end_with_timestamp(&mut self, _timestamp: SystemTime) {}
1054     }
1055 
1056     #[derive(Debug)]
1057     struct TestDynError {
1058         msg: &'static str,
1059         source: Option<Box<TestDynError>>,
1060     }
1061     impl Display for TestDynError {
fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result1062         fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1063             write!(f, "{}", self.msg)
1064         }
1065     }
1066     impl Error for TestDynError {
source(&self) -> Option<&(dyn Error + 'static)>1067         fn source(&self) -> Option<&(dyn Error + 'static)> {
1068             match &self.source {
1069                 Some(source) => Some(source),
1070                 None => None,
1071             }
1072         }
1073     }
1074     impl TestDynError {
new(msg: &'static str) -> Self1075         fn new(msg: &'static str) -> Self {
1076             Self { msg, source: None }
1077         }
with_parent(self, parent_msg: &'static str) -> Self1078         fn with_parent(self, parent_msg: &'static str) -> Self {
1079             Self {
1080                 msg: parent_msg,
1081                 source: Some(Box::new(self)),
1082             }
1083         }
1084     }
1085 
1086     #[test]
dynamic_span_names()1087     fn dynamic_span_names() {
1088         let dynamic_name = "GET http://example.com".to_string();
1089         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1090         let subscriber = tracing_subscriber::registry().with(layer().with_tracer(tracer.clone()));
1091 
1092         tracing::subscriber::with_default(subscriber, || {
1093             tracing::debug_span!("static_name", otel.name = dynamic_name.as_str());
1094         });
1095 
1096         let recorded_name = tracer
1097             .0
1098             .lock()
1099             .unwrap()
1100             .as_ref()
1101             .map(|b| b.builder.name.clone());
1102         assert_eq!(recorded_name, Some(dynamic_name.into()))
1103     }
1104 
1105     #[test]
span_kind()1106     fn span_kind() {
1107         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1108         let subscriber = tracing_subscriber::registry().with(layer().with_tracer(tracer.clone()));
1109 
1110         tracing::subscriber::with_default(subscriber, || {
1111             tracing::debug_span!("request", otel.kind = "server");
1112         });
1113 
1114         let recorded_kind = tracer.with_data(|data| data.builder.span_kind.clone());
1115         assert_eq!(recorded_kind, Some(otel::SpanKind::Server))
1116     }
1117 
1118     #[test]
span_status_code()1119     fn span_status_code() {
1120         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1121         let subscriber = tracing_subscriber::registry().with(layer().with_tracer(tracer.clone()));
1122 
1123         tracing::subscriber::with_default(subscriber, || {
1124             tracing::debug_span!("request", otel.status_code = ?otel::Status::Ok);
1125         });
1126         let recorded_status = tracer
1127             .0
1128             .lock()
1129             .unwrap()
1130             .as_ref()
1131             .unwrap()
1132             .builder
1133             .status
1134             .clone();
1135 
1136         assert_eq!(recorded_status, otel::Status::Ok)
1137     }
1138 
1139     #[test]
span_status_message()1140     fn span_status_message() {
1141         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1142         let subscriber = tracing_subscriber::registry().with(layer().with_tracer(tracer.clone()));
1143 
1144         let message = "message";
1145 
1146         tracing::subscriber::with_default(subscriber, || {
1147             tracing::debug_span!("request", otel.status_message = message);
1148         });
1149 
1150         let recorded_status_message = tracer
1151             .0
1152             .lock()
1153             .unwrap()
1154             .as_ref()
1155             .unwrap()
1156             .builder
1157             .status
1158             .clone();
1159 
1160         assert_eq!(recorded_status_message, otel::Status::error(message))
1161     }
1162 
1163     #[test]
trace_id_from_existing_context()1164     fn trace_id_from_existing_context() {
1165         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1166         let subscriber = tracing_subscriber::registry().with(layer().with_tracer(tracer.clone()));
1167         let trace_id = otel::TraceId::from(42u128.to_be_bytes());
1168         let existing_cx = OtelContext::current_with_span(TestSpan(otel::SpanContext::new(
1169             trace_id,
1170             otel::SpanId::from(1u64.to_be_bytes()),
1171             TraceFlags::default(),
1172             false,
1173             Default::default(),
1174         )));
1175         let _g = existing_cx.attach();
1176 
1177         tracing::subscriber::with_default(subscriber, || {
1178             tracing::debug_span!("request", otel.kind = "server");
1179         });
1180 
1181         let recorded_trace_id =
1182             tracer.with_data(|data| data.parent_cx.span().span_context().trace_id());
1183         assert_eq!(recorded_trace_id, trace_id)
1184     }
1185 
1186     #[test]
includes_timings()1187     fn includes_timings() {
1188         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1189         let subscriber = tracing_subscriber::registry().with(
1190             layer()
1191                 .with_tracer(tracer.clone())
1192                 .with_tracked_inactivity(true),
1193         );
1194 
1195         tracing::subscriber::with_default(subscriber, || {
1196             tracing::debug_span!("request");
1197         });
1198 
1199         let attributes = tracer.with_data(|data| data.builder.attributes.as_ref().unwrap().clone());
1200         let keys = attributes
1201             .iter()
1202             .map(|(key, _)| key.as_str())
1203             .collect::<Vec<&str>>();
1204         assert!(keys.contains(&"idle_ns"));
1205         assert!(keys.contains(&"busy_ns"));
1206     }
1207 
1208     #[test]
records_error_fields()1209     fn records_error_fields() {
1210         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1211         let subscriber = tracing_subscriber::registry().with(
1212             layer()
1213                 .with_tracer(tracer.clone())
1214                 .with_exception_fields(true),
1215         );
1216 
1217         let err = TestDynError::new("base error")
1218             .with_parent("intermediate error")
1219             .with_parent("user error");
1220 
1221         tracing::subscriber::with_default(subscriber, || {
1222             tracing::debug_span!(
1223                 "request",
1224                 error = &err as &(dyn std::error::Error + 'static)
1225             );
1226         });
1227 
1228         let attributes = tracer
1229             .0
1230             .lock()
1231             .unwrap()
1232             .as_ref()
1233             .unwrap()
1234             .builder
1235             .attributes
1236             .as_ref()
1237             .unwrap()
1238             .clone();
1239 
1240         let key_values = attributes
1241             .into_iter()
1242             .map(|(key, value)| (key.as_str().to_owned(), value))
1243             .collect::<HashMap<_, _>>();
1244 
1245         assert_eq!(key_values["error"].as_str(), "user error");
1246         assert_eq!(
1247             key_values["error.chain"],
1248             Value::Array(
1249                 vec![
1250                     StringValue::from("intermediate error"),
1251                     StringValue::from("base error")
1252                 ]
1253                 .into()
1254             )
1255         );
1256 
1257         assert_eq!(key_values[FIELD_EXCEPTION_MESSAGE].as_str(), "user error");
1258         assert_eq!(
1259             key_values[FIELD_EXCEPTION_STACKTRACE],
1260             Value::Array(
1261                 vec![
1262                     StringValue::from("intermediate error"),
1263                     StringValue::from("base error")
1264                 ]
1265                 .into()
1266             )
1267         );
1268     }
1269 
1270     #[test]
includes_span_location()1271     fn includes_span_location() {
1272         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1273         let subscriber = tracing_subscriber::registry()
1274             .with(layer().with_tracer(tracer.clone()).with_location(true));
1275 
1276         tracing::subscriber::with_default(subscriber, || {
1277             tracing::debug_span!("request");
1278         });
1279 
1280         let attributes = tracer.with_data(|data| data.builder.attributes.as_ref().unwrap().clone());
1281         let keys = attributes
1282             .iter()
1283             .map(|(key, _)| key.as_str())
1284             .collect::<Vec<&str>>();
1285         assert!(keys.contains(&"code.filepath"));
1286         assert!(keys.contains(&"code.namespace"));
1287         assert!(keys.contains(&"code.lineno"));
1288     }
1289 
1290     #[test]
excludes_span_location()1291     fn excludes_span_location() {
1292         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1293         let subscriber = tracing_subscriber::registry()
1294             .with(layer().with_tracer(tracer.clone()).with_location(false));
1295 
1296         tracing::subscriber::with_default(subscriber, || {
1297             tracing::debug_span!("request");
1298         });
1299 
1300         let attributes = tracer.with_data(|data| data.builder.attributes.as_ref().unwrap().clone());
1301         let keys = attributes
1302             .iter()
1303             .map(|(key, _)| key.as_str())
1304             .collect::<Vec<&str>>();
1305         assert!(!keys.contains(&"code.filepath"));
1306         assert!(!keys.contains(&"code.namespace"));
1307         assert!(!keys.contains(&"code.lineno"));
1308     }
1309 
1310     #[test]
includes_thread()1311     fn includes_thread() {
1312         let thread = thread::current();
1313         let expected_name = thread
1314             .name()
1315             .map(|name| Value::String(name.to_owned().into()));
1316         let expected_id = Value::I64(thread_id_integer(thread.id()) as i64);
1317 
1318         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1319         let subscriber = tracing_subscriber::registry()
1320             .with(layer().with_tracer(tracer.clone()).with_threads(true));
1321 
1322         tracing::subscriber::with_default(subscriber, || {
1323             tracing::debug_span!("request");
1324         });
1325 
1326         let attributes = tracer
1327             .with_data(|data| data.builder.attributes.as_ref().unwrap().clone())
1328             .drain(..)
1329             .map(|(key, value)| (key.as_str().to_string(), value))
1330             .collect::<HashMap<_, _>>();
1331         assert_eq!(attributes.get("thread.name"), expected_name.as_ref());
1332         assert_eq!(attributes.get("thread.id"), Some(&expected_id));
1333     }
1334 
1335     #[test]
excludes_thread()1336     fn excludes_thread() {
1337         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1338         let subscriber = tracing_subscriber::registry()
1339             .with(layer().with_tracer(tracer.clone()).with_threads(false));
1340 
1341         tracing::subscriber::with_default(subscriber, || {
1342             tracing::debug_span!("request");
1343         });
1344 
1345         let attributes = tracer.with_data(|data| data.builder.attributes.as_ref().unwrap().clone());
1346         let keys = attributes
1347             .iter()
1348             .map(|(key, _)| key.as_str())
1349             .collect::<Vec<&str>>();
1350         assert!(!keys.contains(&"thread.name"));
1351         assert!(!keys.contains(&"thread.id"));
1352     }
1353 
1354     #[test]
propagates_error_fields_from_event_to_span()1355     fn propagates_error_fields_from_event_to_span() {
1356         let tracer = TestTracer(Arc::new(Mutex::new(None)));
1357         let subscriber = tracing_subscriber::registry().with(
1358             layer()
1359                 .with_tracer(tracer.clone())
1360                 .with_exception_field_propagation(true),
1361         );
1362 
1363         let err = TestDynError::new("base error")
1364             .with_parent("intermediate error")
1365             .with_parent("user error");
1366 
1367         tracing::subscriber::with_default(subscriber, || {
1368             let _guard = tracing::debug_span!("request",).entered();
1369 
1370             tracing::error!(
1371                 error = &err as &(dyn std::error::Error + 'static),
1372                 "request error!"
1373             )
1374         });
1375 
1376         let attributes = tracer
1377             .0
1378             .lock()
1379             .unwrap()
1380             .as_ref()
1381             .unwrap()
1382             .builder
1383             .attributes
1384             .as_ref()
1385             .unwrap()
1386             .clone();
1387 
1388         let key_values = attributes
1389             .into_iter()
1390             .map(|(key, value)| (key.as_str().to_owned(), value))
1391             .collect::<HashMap<_, _>>();
1392 
1393         assert_eq!(key_values[FIELD_EXCEPTION_MESSAGE].as_str(), "user error");
1394         assert_eq!(
1395             key_values[FIELD_EXCEPTION_STACKTRACE],
1396             Value::Array(
1397                 vec![
1398                     StringValue::from("intermediate error"),
1399                     StringValue::from("base error")
1400                 ]
1401                 .into()
1402             )
1403         );
1404     }
1405 }
1406