Observability & Monitoring Complete Guide 2025: Logging, Metrics, Tracing, Alerting Strategy

// Node.js OpenTelemetry setup
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { Resource } from '@opentelemetry/resources';
import {
  ATTR_SERVICE_NAME,
  ATTR_SERVICE_VERSION,
} from '@opentelemetry/semantic-conventions';

const sdk = new NodeSDK({
  resource: new Resource({
    [ATTR_SERVICE_NAME]: 'order-service',
    [ATTR_SERVICE_VERSION]: '1.2.0',
    environment: process.env.NODE_ENV || 'production',
  }),
  traceExporter: new OTLPTraceExporter({
    url: 'http://otel-collector:4318/v1/traces',
  }),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({
      url: 'http://otel-collector:4318/v1/metrics',
    }),
    exportIntervalMillis: 30000,
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      // Auto-instrument HTTP, Express, pg, Redis, etc.
      '@opentelemetry/instrumentation-http': {
        ignoreIncomingRequestHook: (req) => {
          // Exclude health check paths
          return req.url === '/health';
        },
      },
      '@opentelemetry/instrumentation-express': {
        enabled: true,
      },
    }),
  ],
});

sdk.start();

// Flush telemetry on graceful shutdown
process.on('SIGTERM', () => {
  sdk.shutdown().then(() => process.exit(0));
});

import { trace, SpanStatusCode, context } from '@opentelemetry/api';

const tracer = trace.getTracer('order-service');

async function processOrder(orderData) {
  // Create a manual span
  return tracer.startActiveSpan('processOrder', async (span) => {
    try {
      // Add attributes to span
      span.setAttribute('order.id', orderData.id);
      span.setAttribute('order.total', orderData.total);
      span.setAttribute('order.items_count', orderData.items.length);

      // Child span: inventory check
      const inventory = await tracer.startActiveSpan(
        'checkInventory',
        async (childSpan) => {
          childSpan.setAttribute('inventory.warehouse', 'us-east-1');
          const result = await inventoryService.check(orderData.items);
          childSpan.end();
          return result;
        }
      );

      if (!inventory.available) {
        span.setStatus({
          code: SpanStatusCode.ERROR,
          message: 'Insufficient inventory',
        });
        throw new Error('Insufficient inventory');
      }

      // Child span: payment processing
      const payment = await tracer.startActiveSpan(
        'processPayment',
        async (childSpan) => {
          childSpan.setAttribute('payment.method', orderData.paymentMethod);
          childSpan.setAttribute('payment.amount', orderData.total);
          const result = await paymentService.charge(orderData);
          childSpan.end();
          return result;
        }
      );

      // Add span event
      span.addEvent('order_completed', {
        'order.id': orderData.id,
        'payment.transaction_id': payment.transactionId,
      });

      span.setStatus({ code: SpanStatusCode.OK });
      return { orderId: orderData.id, transactionId: payment.transactionId };
    } catch (error) {
      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: error.message,
      });
      span.recordException(error);
      throw error;
    } finally {
      span.end();
    }
  });
}

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    send_batch_size: 1024
    timeout: 5s

  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  attributes:
    actions:
      - key: environment
        value: production
        action: upsert

  # Tail sampling (prioritize error traces)
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: error-policy
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: slow-policy
        type: latency
        latency:
          threshold_ms: 1000
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

exporters:
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true

  prometheus:
    endpoint: 0.0.0.0:8889

  loki:
    endpoint: http://loki:3100/loki/api/v1/push

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, tail_sampling]
      exporters: [otlp/jaeger]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [loki]

// Structured logging with pino
import pino from 'pino';

const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  // JSON format (default)
  formatters: {
    level(label) {
      return { level: label };
    },
    bindings(bindings) {
      return {
        service: 'order-service',
        version: '1.2.0',
        host: bindings.hostname,
        pid: bindings.pid,
      };
    },
  },
  // Timestamp format
  timestamp: pino.stdTimeFunctions.isoTime,
  // Redact sensitive information
  redact: ['req.headers.authorization', 'req.headers.cookie', '*.password'],
});

// Correlation ID middleware
function correlationMiddleware(req, res, next) {
  const correlationId = req.headers['x-correlation-id'] || generateId();
  req.correlationId = correlationId;
  res.setHeader('x-correlation-id', correlationId);

  // Per-request child logger
  req.log = logger.child({
    correlationId,
    requestId: generateId(),
    method: req.method,
    path: req.path,
    userAgent: req.headers['user-agent'],
  });

  next();
}

// Usage example
app.post('/api/orders', correlationMiddleware, async (req, res) => {
  req.log.info({ body: req.body }, 'Order creation started');

  try {
    const order = await createOrder(req.body);
    req.log.info(
      { orderId: order.id, duration: Date.now() - req.startTime },
      'Order created successfully'
    );
    res.json(order);
  } catch (error) {
    req.log.error(
      { err: error, body: req.body },
      'Order creation failed'
    );
    res.status(500).json({ error: 'Internal server error' });
  }
});

// Log levels by environment
const logLevels = {
  production: 'info',
  staging: 'debug',
  development: 'trace',
};

# docker-compose.yml - ELK Stack
version: '3.8'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ports:
      - "9200:9200"
    volumes:
      - es-data:/usr/share/elasticsearch/data

  logstash:
    image: docker.elastic.co/logstash/logstash:8.12.0
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    depends_on:
      - elasticsearch

  kibana:
    image: docker.elastic.co/kibana/kibana:8.12.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    depends_on:
      - elasticsearch

volumes:
  es-data:

# logstash.conf
input {
  tcp {
    port => 5044
    codec => json
  }
}

filter {
  # Timestamp parsing
  date {
    match => [ "timestamp", "ISO8601" ]
    target => "@timestamp"
  }

  # Parse error stack traces
  if [level] == "error" {
    grok {
      match => {
        "stack" => "%{GREEDYDATA:error_class}: %{GREEDYDATA:error_message}"
      }
    }
  }

  # Add geolocation data
  if [client_ip] {
    geoip {
      source => "client_ip"
    }
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "logs-%{+YYYY.MM.dd}"
  }
}

# Loki configuration
auth_enabled: false

server:
  http_listen_port: 3100

common:
  ring:
    kvstore:
      store: inmemory
  replication_factor: 1

schema_config:
  configs:
    - from: 2024-01-01
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

storage_config:
  filesystem:
    directory: /loki/chunks

limits_config:
  retention_period: 30d

// Send logs directly to Loki (winston-loki)
import winston from 'winston';
import LokiTransport from 'winston-loki';

const logger = winston.createLogger({
  transports: [
    new LokiTransport({
      host: 'http://loki:3100',
      labels: {
        service: 'order-service',
        environment: 'production',
      },
      json: true,
      batching: true,
      interval: 5,
    }),
  ],
});

// Node.js Prometheus client
import { Registry, Counter, Gauge, Histogram, Summary } from 'prom-client';

const register = new Registry();

// Collect default metrics (CPU, memory, event loop, etc.)
import { collectDefaultMetrics } from 'prom-client';
collectDefaultMetrics({ register });

// Counter: HTTP request count
const httpRequestsTotal = new Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests',
  labelNames: ['method', 'path', 'status'],
  registers: [register],
});

// Gauge: Current active connections
const activeConnections = new Gauge({
  name: 'active_connections',
  help: 'Number of active connections',
  registers: [register],
});

// Histogram: Request response time
const httpRequestDuration = new Histogram({
  name: 'http_request_duration_seconds',
  help: 'HTTP request duration in seconds',
  labelNames: ['method', 'path', 'status'],
  buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
  registers: [register],
});

// Summary: DB query time
const dbQueryDuration = new Summary({
  name: 'db_query_duration_seconds',
  help: 'Database query duration',
  labelNames: ['operation', 'table'],
  percentiles: [0.5, 0.9, 0.95, 0.99],
  registers: [register],
});

// Express middleware
function metricsMiddleware(req, res, next) {
  const start = process.hrtime.bigint();
  activeConnections.inc();

  res.on('finish', () => {
    const duration = Number(process.hrtime.bigint() - start) / 1e9;
    const labels = {
      method: req.method,
      path: req.route?.path || req.path,
      status: res.statusCode.toString(),
    };

    httpRequestsTotal.inc(labels);
    httpRequestDuration.observe(labels, duration);
    activeConnections.dec();
  });

  next();
}

// Metrics endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', register.contentType);
  res.end(await register.metrics());
});

# Requests per second (RPS)
rate(http_requests_total[5m])

# Error rate by service
sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
/
sum(rate(http_requests_total[5m])) by (service)

# p99 response time
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))

# p95 response time (by path)
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, path)
)

# Memory usage growth rate
deriv(process_resident_memory_bytes[1h])

# Uptime
time() - process_start_time_seconds

# Apdex score (target response time 0.5s)
(
  sum(rate(http_request_duration_seconds_bucket{le="0.5"}[5m]))
  +
  sum(rate(http_request_duration_seconds_bucket{le="2.0"}[5m]))
) / 2
/
sum(rate(http_request_duration_seconds_count[5m]))

# prometheus-rules.yml
groups:
  - name: request_rates
    interval: 30s
    rules:
      # Pre-computed request rate
      - record: service:http_requests:rate5m
        expr: sum(rate(http_requests_total[5m])) by (service)

      # Pre-computed error rate
      - record: service:http_errors:rate5m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)

      # Pre-computed latency percentiles
      - record: service:http_latency:p99
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          )

{
  "panels": [
    {
      "title": "Request Rate (RPS)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "sum(rate(http_requests_total[5m])) by (service)",
          "legendFormat": "service"
        }
      ]
    },
    {
      "title": "Error Rate (%)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "100 * sum(rate(http_requests_total{status=~\"5..\"}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)",
          "legendFormat": "service"
        }
      ]
    },
    {
      "title": "Response Time (p99)",
      "type": "timeseries",
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))",
          "legendFormat": "service"
        }
      ]
    }
  ]
}

# CPU Utilization
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# Memory Utilization
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# Disk I/O Saturation
rate(node_disk_io_time_weighted_seconds_total[5m])

# Network Errors
rate(node_network_receive_errs_total[5m])
+ rate(node_network_transmit_errs_total[5m])

// W3C Trace Context propagation
// Request headers:
// traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
// tracestate: congo=t61rcWkgMzE

// Extract/inject context in Express middleware
import { propagation, context, trace } from '@opentelemetry/api';

// Inject context in HTTP client
async function makeRequest(url, data) {
  const headers = {};

  // Inject trace info from current context into headers
  propagation.inject(context.active(), headers);

  return fetch(url, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      ...headers, // Includes traceparent, tracestate
    },
    body: JSON.stringify(data),
  });
}

// gRPC context propagation
import { GrpcInstrumentation } from '@opentelemetry/instrumentation-grpc';

// Auto-instrumentation injects/extracts context in gRPC metadata

# docker-compose.yml - Jaeger
services:
  jaeger:
    image: jaegertracing/all-in-one:1.53
    ports:
      - "16686:16686"  # UI
      - "4317:4317"    # OTLP gRPC
      - "4318:4318"    # OTLP HTTP
    environment:
      - COLLECTOR_OTLP_ENABLED=true
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch:9200

// Sampling configuration
import { ParentBasedSampler, TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-base';

// Ratio-based sampling (collect only 10%)
const sampler = new TraceIdRatioBasedSampler(0.1);

// Parent-based sampling (if parent is sampled, child is also sampled)
const parentBasedSampler = new ParentBasedSampler({
  root: new TraceIdRatioBasedSampler(0.1),
});

# Prometheus alerting rules
groups:
  - name: service_alerts
    rules:
      # Critical: Immediate response required
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)
          > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate on service"
          description: "Error rate is above 5% for 5 minutes"
          runbook_url: "https://wiki.example.com/runbooks/high-error-rate"

      # Warning: Attention needed
      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          ) > 2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High latency on service"
          description: "p99 latency is above 2 seconds for 10 minutes"

      # Info: Informational
      - alert: PodRestart
        expr: |
          increase(kube_pod_container_status_restarts_total[1h]) > 3
        labels:
          severity: info
        annotations:
          summary: "Pod restarting frequently"

# alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/xxx'

route:
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default-slack'

  routes:
    # Critical -> PagerDuty + Slack
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 1h
      continue: true

    - match:
        severity: critical
      receiver: 'slack-critical'

    # Warning -> Slack
    - match:
        severity: warning
      receiver: 'slack-warning'
      repeat_interval: 4h

    # Info -> Slack (business hours only)
    - match:
        severity: info
      receiver: 'slack-info'
      active_time_intervals:
        - business-hours

receivers:
  - name: 'default-slack'
    slack_configs:
      - channel: '#alerts-general'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_KEY'
        severity: critical

  - name: 'slack-critical'
    slack_configs:
      - channel: '#alerts-critical'
        color: 'danger'

  - name: 'slack-warning'
    slack_configs:
      - channel: '#alerts-warning'
        color: 'warning'

  - name: 'slack-info'
    slack_configs:
      - channel: '#alerts-info'

time_intervals:
  - name: business-hours
    time_intervals:
      - weekdays: ['monday:friday']
        times:
          - start_time: '09:00'
            end_time: '18:00'

inhibit_rules:
  # When Critical fires, suppress Warning for the same service
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['service']

# Runbook: High Error Rate

## Alert Condition
- Error rate (5xx) above 5% persisting for more than 5 minutes

## Immediate Checks
1. Identify affected endpoints
2. Check recent deployment history
3. Verify dependent service health

## Response Procedure
1. Check error patterns in Grafana dashboard
2. Review error details in logs
3. Identify failure point using traces
4. If caused by recent deployment, rollback
5. If dependent service issue, check circuit breaker

## Escalation
- 15 min unresolved: Report to team lead
- 30 min unresolved: Page senior engineer

SLO: 99.9% availability
= Allowed downtime per month (30 days): 43.2 minutes
= Error Budget: 0.1%

Error Budget consumption rate:
- 50% consumed: Caution
- 75% consumed: Feature release freeze
- 100% consumed: Focus exclusively on reliability work

# Burn Rate based alerting
groups:
  - name: slo_alerts
    rules:
      # Fast burn (2% consumed in 1 hour) - Immediate response
      - alert: SLOBurnRateCritical
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[1h]))
            /
            sum(rate(http_requests_total[1h]))
          ) > (14.4 * 0.001)
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "SLO burn rate critical - error budget exhausting fast"

      # Slow burn (5% consumed in 6 hours) - Attention
      - alert: SLOBurnRateWarning
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[6h]))
            /
            sum(rate(http_requests_total[6h]))
          ) > (6 * 0.001)
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "SLO burn rate elevated"

+-----------------+     +------------------+
| Application     |---->| OTel Collector   |
| (OTel SDK)      |     | (collect/process)|
+-----------------+     +--------+---------+
                                 |
                    +------------+------------+
                    |            |            |
              +-----v-----+ +---v---+ +-----v-----+
              | Prometheus | | Tempo | | Loki      |
              | (Metrics)  | |(Trace)| | (Logs)    |
              +-----+------+ +---+---+ +-----+-----+
                    |            |            |
                    +------+-----+------+-----+
                           |            |
                      +----v----+ +----v----+
                      | Grafana | | Grafana |
                      | (visual)| | (unified)|
                      +---------+ +---------+

# Log retention policy
retention_policy:
  hot_tier: 7d        # Recent 7 days: Fast search
  warm_tier: 30d       # Recent 30 days: Slow search
  cold_tier: 90d       # Recent 90 days: Archive
  delete_after: 365d   # Delete after 1 year

# Index Lifecycle Management (Elasticsearch ILM)
index_lifecycle:
  phases:
    hot:
      actions:
        rollover:
          max_size: 50gb
          max_age: 1d
    warm:
      min_age: 7d
      actions:
        shrink:
          number_of_shards: 1
        forcemerge:
          max_num_segments: 1
    cold:
      min_age: 30d
      actions:
        searchable_snapshot:
          snapshot_repository: s3_repo
    delete:
      min_age: 365d

// Adaptive sampling
class AdaptiveSampler {
  constructor(targetRate = 100) {
    this.targetRate = targetRate; // Target collections per second
    this.currentRate = 0;
    this.samplingProbability = 1.0;

    // Adjust sampling probability every 10 seconds
    setInterval(() => this.adjust(), 10000);
  }

  shouldSample() {
    this.currentRate++;
    return Math.random() < this.samplingProbability;
  }

  adjust() {
    if (this.currentRate > this.targetRate * 10) {
      // If traffic is 10x above target, reduce sampling probability
      this.samplingProbability = this.targetRate / this.currentRate;
    } else {
      this.samplingProbability = Math.min(1.0, this.samplingProbability * 1.1);
    }
    this.currentRate = 0;
  }
}

# Aggregate raw data with Prometheus Recording Rules
groups:
  - name: aggregation
    interval: 1m
    rules:
      # Service-level aggregation (reduce cardinality by removing instance label)
      - record: service:requests:rate5m
        expr: sum(rate(http_requests_total[5m])) by (service, method, status)

      # Reduce time resolution (5min -> 1hour)
      - record: service:requests:rate1h
        expr: sum(rate(http_requests_total[1h])) by (service)