Without tracing:
  Request fails in Service D — you check Service D logs
  Error says "dependency failed" — which dependency? Which call?
  You grep logs across 5 services by timestamp — painful, imprecise

With tracing:
  One trace ID follows the request through all services
  Waterfall view shows exactly where time was spent
  Each span has attributes: SQL query, HTTP URL, queue name, error details

Trace:   the end-to-end journey of one request across all services
Span:    one unit of work within a trace (HTTP call, DB query, queue publish)
TraceId: 128-bit ID shared by all spans in the same trace
SpanId:  64-bit ID unique to each span
Parent:  each span knows its parent span — forms a tree

W3C TraceContext (traceparent header):
  00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
  ──  ────────────────────────────────  ────────────────  ──
  ver  TraceId (32 hex chars)            SpanId            flags

<!-- Api.csproj -->
<PackageReference Include="OpenTelemetry.Extensions.Hosting"   Version="1.*" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.*" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.*" />
<PackageReference Include="OpenTelemetry.Instrumentation.SqlClient" Version="1.*" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.*" />
<!-- For Jaeger: -->
<PackageReference Include="OpenTelemetry.Exporter.Jaeger" Version="1.*" />
<!-- For Azure Monitor: -->
<PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.*" />

// Program.cs
builder.Services.AddOpenTelemetry()
    .ConfigureResource(resource => resource
        .AddService(
            serviceName:    "OrderService",
            serviceVersion: "1.0.0"))
    .WithTracing(tracing => tracing
        // Auto-instrument ASP.NET Core (incoming HTTP requests)
        .AddAspNetCoreInstrumentation(opts =>
        {
            // Enrich spans with request/response details
            opts.EnrichWithHttpRequest  = (activity, req)  => activity.SetTag("http.request_id", req.Headers.RequestId.FirstOrDefault());
            opts.EnrichWithHttpResponse = (activity, resp) => activity.SetTag("http.response_size", resp.ContentLength);
            opts.Filter = ctx => ctx.Request.Path != "/health";   // skip health checks
        })
        // Auto-instrument outgoing HttpClient calls
        .AddHttpClientInstrumentation()
        // Auto-instrument Entity Framework / SQL
        .AddSqlClientInstrumentation(opts => opts.SetDbStatementForText = true)
        // Your own ActivitySource for custom spans
        .AddSource("OrderService.*")
        // Export to Jaeger (dev) or OTLP (prod)
        .AddOtlpExporter(opts =>
        {
            opts.Endpoint = new Uri(builder.Configuration["Otel:Endpoint"]!);
        }))
    .WithMetrics(metrics => metrics
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddOtlpExporter());

// Define ActivitySource once — share across the service
public static class Telemetry
{
    public static readonly ActivitySource Orders = new("OrderService.Orders", "1.0");
}

// Use in a handler
public class CreateOrderHandler(IOrderRepository repo, IEventBus bus)
    : IRequestHandler<CreateOrderCommand, int>
{
    public async Task<int> Handle(CreateOrderCommand cmd, CancellationToken ct)
    {
        // Start a custom span — child of the incoming HTTP span
        using var activity = Telemetry.Orders.StartActivity("CreateOrder");
        activity?.SetTag("order.customer_id", cmd.CustomerId);
        activity?.SetTag("order.item_count", cmd.Items.Count);

        try
        {
            var order = Order.Create(cmd.CustomerId, cmd.Items);
            await repo.AddAsync(order, ct);

            // Nested span for event publishing
            using var publishActivity = Telemetry.Orders.StartActivity("PublishOrderCreated");
            await bus.PublishAsync(new OrderCreatedEvent(order.Id), ct);
            publishActivity?.SetTag("order.id", order.Id);

            activity?.SetTag("order.id", order.Id);
            return order.Id;
        }
        catch (Exception ex)
        {
            activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
            activity?.RecordException(ex);
            throw;
        }
    }
}

// When publishing a message: attach trace context
public class RabbitMqEventBus(IModel channel) : IEventBus
{
    public async Task PublishAsync<T>(T @event, CancellationToken ct = default)
    {
        var props = channel.CreateBasicProperties();
        props.Headers = new Dictionary<string, object?>();

        // Inject W3C TraceContext into message headers
        var propagator = Propagators.DefaultTextMapPropagator;
        propagator.Inject(
            new PropagationContext(Activity.Current?.Context ?? default, Baggage.Current),
            props.Headers,
            (headers, key, value) => headers[key] = value);

        var body = JsonSerializer.SerializeToUtf8Bytes(@event);
        channel.BasicPublish("", typeof(T).Name, props, body);
    }
}

// When consuming a message: extract trace context → continue the trace
public class OrderCreatedConsumer(IMediator mediator)
{
    public async Task ConsumeAsync(BasicDeliverEventArgs args, CancellationToken ct)
    {
        // Extract trace context from message headers
        var parentContext = Propagators.DefaultTextMapPropagator.Extract(
            default,
            args.BasicProperties.Headers,
            (headers, key) => headers.TryGetValue(key, out var val)
                ? [Encoding.UTF8.GetString((byte[])val!)]
                : []);

        // Start span as a child of the publisher's span
        using var activity = Telemetry.Orders.StartActivity(
            "ProcessOrderCreated",
            ActivityKind.Consumer,
            parentContext.ActivityContext);

        var @event = JsonSerializer.Deserialize<OrderCreatedEvent>(args.Body.Span)!;
        await mediator.Publish(@event, ct);
    }
}

// Middleware: ensure every request has a correlation ID in logs
public class CorrelationIdMiddleware(RequestDelegate next)
{
    public async Task InvokeAsync(HttpContext ctx)
    {
        // Use the OpenTelemetry TraceId as the correlation ID (already propagated)
        var traceId = Activity.Current?.TraceId.ToString()
            ?? ctx.Request.Headers["X-Correlation-Id"].FirstOrDefault()
            ?? Guid.NewGuid().ToString("N");

        ctx.Response.Headers["X-Correlation-Id"] = traceId;

        using (LogContext.PushProperty("CorrelationId", traceId))
        using (LogContext.PushProperty("TraceId",       Activity.Current?.TraceId.ToString()))
        using (LogContext.PushProperty("SpanId",        Activity.Current?.SpanId.ToString()))
        {
            await next(ctx);
        }
    }
}

// Register before other middleware
app.UseMiddleware<CorrelationIdMiddleware>();

// Serilog with OpenTelemetry enrichment — all logs include TraceId automatically
Log.Logger = new LoggerConfiguration()
    .Enrich.WithProperty("Service", "OrderService")
    .Enrich.FromLogContext()     // picks up CorrelationId from middleware
    .WriteTo.Console(new JsonFormatter())
    .CreateLogger();

// Log output includes:
// { "TraceId": "4bf92f3577b34da6a3ce929d0e0e4736", "SpanId": "00f067aa0ba902b7",
//   "CorrelationId": "4bf92f3577b34da6a3ce929d0e0e4736", "message": "Order created" }

Local development — Jaeger:
  docker run -p 16686:16686 -p 4317:4317 jaegertracing/all-in-one

  Configure OTLP endpoint: http://localhost:4317
  View traces:            http://localhost:16686

Production:
  Azure Monitor / Application Insights:
    builder.Services.AddOpenTelemetry()
      .UseAzureMonitor(opts => opts.ConnectionString = "InstrumentationKey=...");

  Grafana + Tempo:
    .AddOtlpExporter(opts => opts.Endpoint = new Uri("http://tempo:4317"))

// Auto-trace every MediatR command/query without adding Activity to every handler
public class TracingBehaviour<TRequest, TResponse>
    : IPipelineBehavior<TRequest, TResponse>
    where TRequest : notnull
{
    public async Task<TResponse> Handle(
        TRequest request,
        RequestHandlerDelegate<TResponse> next,
        CancellationToken ct)
    {
        var operationName = typeof(TRequest).Name;
        using var activity = Telemetry.Orders.StartActivity(operationName);
        activity?.SetTag("mediator.request", operationName);

        try
        {
            var response = await next();
            return response;
        }
        catch (Exception ex)
        {
            activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
            activity?.RecordException(ex);
            throw;
        }
    }
}

// Register in DI — runs for every handler
builder.Services.AddTransient(typeof(IPipelineBehavior<,>), typeof(TracingBehaviour<,>));