Distributed Tracing and Correlation IDs in .NET

A single user action may span multiple services:
  API Gateway → PatientService → LabService → NotificationService

Without correlation:
  Log A: "Prescription created" (PatientService)
  Log B: "Lab order submitted" (LabService)
  Log C: "Notification sent" (NotificationService)
  Which logs belong to the same user request? Impossible to tell.

With correlation (same CorrelationId / TraceId on all three):
  Query: CorrelationId = "abc-123" → shows all three logs together
  Root cause analysis: follow the exact path of one request across services

// .NET automatically propagates W3C trace context via Activity API
// ASP.NET Core reads the traceparent header and creates an Activity

// No code needed for basic propagation:
// HTTP: traceparent header
// gRPC: grpc-trace-bin header
// Azure Service Bus: Diagnostic-Id property

// Access the current trace in code:
var traceId    = Activity.Current?.TraceId.ToString();    // W3C TraceId (128-bit hex)
var spanId     = Activity.Current?.SpanId.ToString();     // W3C SpanId
var requestId  = httpContext.TraceIdentifier;              // ASP.NET Core TraceIdentifier

// Log with the trace ID to correlate with Application Insights / OpenTelemetry
_logger.LogInformation(
    "Processing prescription {PrescriptionId} [Trace: {TraceId}]",
    prescriptionId, Activity.Current?.TraceId);

// For systems that don't use W3C trace context — custom X-Correlation-Id header
public sealed class CorrelationIdMiddleware : IMiddleware
{
    private const string CorrelationIdHeader = "X-Correlation-Id";
    private readonly ILogger<CorrelationIdMiddleware> _logger;

    public CorrelationIdMiddleware(ILogger<CorrelationIdMiddleware> logger)
        => _logger = logger;

    public async Task InvokeAsync(HttpContext context, RequestDelegate next)
    {
        // Accept incoming correlation ID or generate a new one
        var correlationId = context.Request.Headers[CorrelationIdHeader].FirstOrDefault()
            ?? Guid.NewGuid().ToString("N");

        // Echo it in the response
        context.Response.Headers[CorrelationIdHeader] = correlationId;

        // Push to log context — all log entries in this request carry CorrelationId
        using (LogContext.PushProperty("CorrelationId", correlationId))
        {
            _logger.LogDebug("Request started with CorrelationId {CorrelationId}", correlationId);
            await next(context);
        }
    }
}

app.UseMiddleware<CorrelationIdMiddleware>();

// When calling a downstream HTTP service, forward the correlation ID
public sealed class CorrelationIdDelegatingHandler : DelegatingHandler
{
    private readonly IHttpContextAccessor _httpContextAccessor;
    private const string CorrelationIdHeader = "X-Correlation-Id";

    public CorrelationIdDelegatingHandler(IHttpContextAccessor accessor)
        => _httpContextAccessor = accessor;

    protected override Task<HttpResponseMessage> SendAsync(
        HttpRequestMessage request, CancellationToken ct)
    {
        var correlationId = _httpContextAccessor.HttpContext?.Response.Headers[CorrelationIdHeader]
            .FirstOrDefault();

        if (!string.IsNullOrEmpty(correlationId))
            request.Headers.TryAddWithoutValidation(CorrelationIdHeader, correlationId);

        return base.SendAsync(request, ct);
    }
}

// Register with HttpClient
builder.Services
    .AddHttpContextAccessor()
    .AddTransient<CorrelationIdDelegatingHandler>()
    .AddHttpClient<ILabServiceClient, LabServiceClient>(client =>
        client.BaseAddress = new Uri("https://lab-api.internal"))
    .AddHttpMessageHandler<CorrelationIdDelegatingHandler>();

// NuGet: OpenTelemetry.Extensions.Hosting
//         OpenTelemetry.Instrumentation.AspNetCore
//         OpenTelemetry.Instrumentation.Http
//         OpenTelemetry.Exporter.OpenTelemetryProtocol (for Jaeger/Zipkin)

builder.Services.AddOpenTelemetry()
    .ConfigureResource(resource =>
        resource.AddService("SystemForge-Clinical-Api"))
    .WithTracing(tracing =>
    {
        tracing
            .AddAspNetCoreInstrumentation()   // HTTP requests
            .AddHttpClientInstrumentation()   // outgoing HTTP calls
            .AddSqlClientInstrumentation()    // SQL Server queries
            .AddSource("SystemForge.*")       // custom ActivitySource
            .AddOtlpExporter(options =>
                options.Endpoint = new Uri("http://otel-collector:4317"));
    })
    .WithMetrics(metrics =>
    {
        metrics
            .AddAspNetCoreInstrumentation()
            .AddRuntimeInstrumentation()
            .AddOtlpExporter();
    });

// Create spans for business operations — visible in Jaeger, Zipkin, Application Insights
private static readonly ActivitySource ActivitySource =
    new ActivitySource("SystemForge.Clinical.Prescriptions");

public async Task<Result> CreatePrescriptionAsync(
    CreatePrescriptionCommand command, CancellationToken ct)
{
    using var activity = ActivitySource.StartActivity("CreatePrescription");
    activity?.SetTag("patient.id",    command.PatientId.ToString());
    activity?.SetTag("medication",    command.MedicationName);
    activity?.SetTag("prescriber.id", command.PrescriberId.ToString());

    try
    {
        var result = await _repo.InsertAsync(prescription, ct);
        activity?.SetStatus(ActivityStatusCode.Ok);
        return Result.Success();
    }
    catch (Exception ex)
    {
        activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
        activity?.RecordException(ex);
        throw;
    }
}