Add Distributed Tracing with OpenTelemetry

# otel-collector-config.yaml — Collector pipeline configuration.
# Receives OTLP from all services, processes in batches, exports to
# Jaeger (traces), Prometheus (metrics), and Loki (logs).

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317      # gRPC — primary protocol, used by SDKs
      http:
        endpoint: 0.0.0.0:4318      # HTTP — fallback for environments that block gRPC

processors:
  # Batch telemetry to reduce export overhead
  batch:
    timeout: 5s                      # Flush every 5 seconds
    send_batch_size: 1024            # Or when 1024 items accumulate

  # Prevent OOM if a service floods telemetry data
  memory_limiter:
    check_interval: 1s
    limit_mib: 512                   # Hard limit — start dropping data
    spike_limit_mib: 128             # Soft limit — start throttling

  # Add deployment metadata to all telemetry
  resource:
    attributes:
      - key: deployment.environment
        value: production
        action: upsert
      - key: service.namespace
        value: checkout-platform
        action: upsert

  # Tail-based sampling: keep 100% of error traces, 10% of successful ones
  tail_sampling:
    decision_wait: 10s               # Wait 10s for all spans of a trace to arrive
    policies:
      - name: errors-always
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow-requests
        type: latency
        latency: { threshold_ms: 2000 }  # Keep all traces slower than 2s
      - name: sample-rest
        type: probabilistic
        probabilistic: { sampling_percentage: 10 }

exporters:
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true                 # Internal network — TLS handled by service mesh

  prometheus:
    endpoint: 0.0.0.0:8889          # Prometheus scrapes this endpoint
    resource_to_telemetry_conversion:
      enabled: true                  # service.name becomes a Prometheus label

  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, tail_sampling, batch, resource]
      exporters: [otlp/jaeger]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch, resource]
      exporters: [loki]

// src/telemetry/setup.ts — OpenTelemetry initialization.
// This file MUST be imported before any other module — the SDK needs
// to patch HTTP, Express, etc. before they're loaded.

import { NodeSDK } from "@opentelemetry/sdk-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-grpc";
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-grpc";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { Resource } from "@opentelemetry/resources";
import {
  ATTR_SERVICE_NAME,
  ATTR_SERVICE_VERSION,
} from "@opentelemetry/semantic-conventions";
import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
import { BatchLogRecordProcessor } from "@opentelemetry/sdk-logs";

const collectorUrl = process.env.OTEL_COLLECTOR_URL || "http://otel-collector:4317";

const sdk = new NodeSDK({
  resource: new Resource({
    [ATTR_SERVICE_NAME]: process.env.SERVICE_NAME || "unknown-service",
    [ATTR_SERVICE_VERSION]: process.env.SERVICE_VERSION || "0.0.0",
    "deployment.environment": process.env.NODE_ENV || "development",
  }),

  traceExporter: new OTLPTraceExporter({ url: collectorUrl }),

  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({ url: collectorUrl }),
    exportIntervalMillis: 15000,  // Export metrics every 15 seconds
  }),

  logRecordProcessor: new BatchLogRecordProcessor(
    new OTLPLogExporter({ url: collectorUrl })
  ),

  instrumentations: [
    getNodeAutoInstrumentations({
      // Customize which libraries to instrument
      "@opentelemetry/instrumentation-http": {
        // Don't trace health checks — they create noise
        ignoreIncomingRequestHook: (req) =>
          req.url === "/health" || req.url === "/ready",
      },
      "@opentelemetry/instrumentation-express": { enabled: true },
      "@opentelemetry/instrumentation-pg": {
        // Include SQL query text in spans (careful: may contain PII)
        enhancedDatabaseReporting: true,
      },
      "@opentelemetry/instrumentation-redis": { enabled: true },
      // Disable noisy instrumentations
      "@opentelemetry/instrumentation-fs": { enabled: false },
      "@opentelemetry/instrumentation-dns": { enabled: false },
    }),
  ],
});

sdk.start();

// Graceful shutdown — flush pending telemetry before process exits
process.on("SIGTERM", () => {
  sdk.shutdown()
    .then(() => console.log("Telemetry SDK shut down"))
    .catch((err) => console.error("Error shutting down SDK", err))
    .finally(() => process.exit(0));
});

export { sdk };

// src/routes/checkout.ts — Checkout handler with custom spans.
// Auto-instrumentation captures the HTTP request and database queries.
// Custom spans add business context: payment provider, cart value, item count.

import { trace, SpanStatusCode, context } from "@opentelemetry/api";
import type { Request, Response } from "express";

const tracer = trace.getTracer("checkout-service", "1.0.0");

export async function handleCheckout(req: Request, res: Response) {
  // Custom span wraps the entire checkout business logic
  return tracer.startActiveSpan("checkout.process", async (span) => {
    try {
      const { cartId, paymentMethod } = req.body;

      // Add business attributes — these show up in Jaeger/Grafana
      span.setAttribute("checkout.cart_id", cartId);
      span.setAttribute("checkout.payment_method", paymentMethod);

      // Step 1: Validate cart (calls inventory service via HTTP)
      const cart = await tracer.startActiveSpan("checkout.validate_cart", async (cartSpan) => {
        const result = await inventoryClient.validateCart(cartId);
        cartSpan.setAttribute("cart.item_count", result.items.length);
        cartSpan.setAttribute("cart.total_cents", result.totalCents);
        cartSpan.end();
        return result;
      });

      span.setAttribute("checkout.total_cents", cart.totalCents);
      span.setAttribute("checkout.item_count", cart.items.length);

      // Step 2: Process payment (calls payment service)
      const payment = await tracer.startActiveSpan("checkout.process_payment", async (paySpan) => {
        paySpan.setAttribute("payment.provider", paymentMethod);
        paySpan.setAttribute("payment.amount_cents", cart.totalCents);

        try {
          const result = await paymentClient.charge(cart.totalCents, paymentMethod);
          paySpan.setAttribute("payment.transaction_id", result.transactionId);
          paySpan.setStatus({ code: SpanStatusCode.OK });
          paySpan.end();
          return result;
        } catch (err) {
          paySpan.setStatus({
            code: SpanStatusCode.ERROR,
            message: err instanceof Error ? err.message : "Payment failed",
          });
          paySpan.recordException(err as Error);
          paySpan.end();
          throw err;
        }
      });

      // Step 3: Reserve inventory (calls inventory service)
      await inventoryClient.reserve(cartId, payment.transactionId);

      // Step 4: Send confirmation (calls notification service)
      // Fire-and-forget — don't block the response
      notificationClient.sendConfirmation(req.user.email, payment.transactionId)
        .catch((err) => span.addEvent("notification_failed", { error: err.message }));

      span.setStatus({ code: SpanStatusCode.OK });
      res.json({ orderId: payment.transactionId, status: "confirmed" });
    } catch (err) {
      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: err instanceof Error ? err.message : "Checkout failed",
      });
      span.recordException(err as Error);
      res.status(500).json({ error: "Checkout failed" });
    } finally {
      span.end();
    }
  });
}

// src/lib/logger.ts — Structured logger with trace correlation.
// Every log message includes the active trace ID and span ID.
// In Grafana, clicking a trace ID jumps directly to related logs.

import pino from "pino";
import { context, trace } from "@opentelemetry/api";

function getTraceContext() {
  const activeSpan = trace.getSpan(context.active());
  if (!activeSpan) return {};

  const spanContext = activeSpan.spanContext();
  return {
    trace_id: spanContext.traceId,
    span_id: spanContext.spanId,
    trace_flags: spanContext.traceFlags,
  };
}

export const logger = pino({
  level: process.env.LOG_LEVEL || "info",
  mixin() {
    // Automatically inject trace context into every log line
    return getTraceContext();
  },
  formatters: {
    level(label) {
      // Grafana Loki expects "level" not "lvl"
      return { level: label };
    },
  },
});

// Usage in any handler:
// logger.info({ cartId, itemCount: 5 }, "Cart validated successfully");
// Output: {"level":"info","trace_id":"abc123...","span_id":"def456...","cartId":"cart_789","itemCount":5,"msg":"Cart validated successfully"}

# docker-compose.observability.yml — Full observability stack.
# Jaeger for traces, Prometheus for metrics, Loki for logs, Grafana for dashboards.

services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.96.0
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "4317:4317"     # OTLP gRPC
      - "4318:4318"     # OTLP HTTP
      - "8889:8889"     # Prometheus metrics endpoint

  jaeger:
    image: jaegertracing/all-in-one:1.54
    ports:
      - "16686:16686"   # Jaeger UI
    environment:
      - COLLECTOR_OTLP_ENABLED=true

  prometheus:
    image: prom/prometheus:v2.50.0
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"

  loki:
    image: grafana/loki:2.9.4
    ports:
      - "3100:3100"

  grafana:
    image: grafana/grafana:10.3.0
    ports:
      - "3000:3000"
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning

# prometheus.yml — Scrape the OTel Collector's metrics endpoint.

global:
  scrape_interval: 15s

scrape_configs:
  - job_name: "otel-collector"
    static_configs:
      - targets: ["otel-collector:8889"]

// src/telemetry/custom-metrics.ts — Business metrics alongside technical ones.
// These metrics appear in Grafana dashboards next to latency and error rates.

import { metrics } from "@opentelemetry/api";

const meter = metrics.getMeter("checkout-service", "1.0.0");

// Business metrics
export const checkoutCounter = meter.createCounter("checkout.completed", {
  description: "Number of completed checkouts",
  unit: "1",
});

export const checkoutValueHistogram = meter.createHistogram("checkout.value", {
  description: "Distribution of checkout values",
  unit: "cents",
  advice: {
    explicitBucketBoundaries: [
      500, 1000, 2500, 5000, 10000, 25000, 50000, 100000,
    ],  // $5, $10, $25, $50, $100, $250, $500, $1000
  },
});

export const cartAbandonmentCounter = meter.createCounter("cart.abandoned", {
  description: "Carts that started checkout but failed to complete",
  unit: "1",
});

// Usage in checkout handler:
// checkoutCounter.add(1, { payment_method: "stripe", status: "success" });
// checkoutValueHistogram.record(cart.totalCents, { currency: "usd" });