2.6 Observability и мониторинг #

# prometheus.yml
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'my-app'
    static_configs:
      - targets: ['localhost:8080']
    metrics_path: /metrics
    scrape_interval: 5s

# Вместо 15s
scrape_interval: 15s

# Используйте 5s или даже 1s для критических метрик
scrape_interval: 5s

{
  "timestamp": "2024-01-15T12:06:30Z",
  "level": "WARN", 
  "message": "High CPU spike detected",
  "cpu_percent": 97.5,
  "duration_seconds": 30
}

# APM инструменты для детального анализа
- Jaeger (трейсинг запросов)
- New Relic (мониторинг производительности приложений)  
- DataDog (мониторинг пользователей)

# Single node версия
docker run -it --rm -v victoria-metrics-data:/victoria-metrics-data \
  -p 8428:8428 \
  victoriametrics/victoria-metrics:latest

# docker-compose.yml
version: '3.8'
services:
  victoria-metrics:
    image: victoriametrics/victoria-metrics:latest
    ports:
      - "8428:8428"
    volumes:
      - vmdata:/victoria-metrics-data
    command:
      - '--storageDataPath=/victoria-metrics-data'
      - '--httpListenAddr=:8428'
      - '--retentionPeriod=12'  # 12 месяцев хранения

  vmagent:
    image: victoriametrics/vmagent:latest
    ports:
      - "8429:8429"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    command:
      - '--promscrape.config=/etc/prometheus/prometheus.yml'
      - '--remoteWrite.url=http://victoria-metrics:8428/api/v1/write'

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin

volumes:
  vmdata:

{
  "name": "VictoriaMetrics",
  "type": "prometheus",
  "url": "http://victoria-metrics:8428",
  "access": "proxy",
  "isDefault": true
}

# VictoriaMetrics Cluster
version: '3.8'
services:
  vmstorage:
    image: victoriametrics/vmstorage:cluster
    ports:
      - "8482:8482"
    volumes:
      - vmstorage:/storage
    command:
      - '--storageDataPath=/storage'

  vminsert:
    image: victoriametrics/vminsert:cluster
    ports:
      - "8480:8480"
    depends_on:
      - vmstorage
    command:
      - '--storageNode=vmstorage:8400'

  vmselect:
    image: victoriametrics/vmselect:cluster
    ports:
      - "8481:8481"
    depends_on:
      - vmstorage
    command:
      - '--storageNode=vmstorage:8401'

volumes:
  vmstorage:

{
  "timestamp": "2024-01-15T10:30:00Z",
  "level": "ERROR",
  "service": "user-service",
  "message": "Failed to authenticate user",
  "user_id": "12345",
  "request_id": "abc-def-123"
}

# docker-compose для Loki
version: '3.8'
services:
  loki:
    image: grafana/loki:latest
    ports:
      - "3100:3100"
    volumes:
      - ./loki-config.yml:/etc/loki/loki-config.yml
    command: -config.file=/etc/loki/local-config.yaml

  promtail:
    image: grafana/promtail:latest
    volumes:
      - /var/log:/var/log:ro
      - ./promtail-config.yml:/etc/promtail/config.yml
    command: -config.file=/etc/promtail/config.yml

✅ **Эффективное хранение**: индексирует только метки, не контент
✅ **Grafana интеграция**: native поддержка в Grafana
✅ **LogQL**: мощный язык запросов похожий на PromQL
✅ **Horizontal scaling**: поддержка кластерного развертывания
✅ **Cost effective**: дешевле ELK stack для больших объемов

# Пример LogQL запроса
{job="nginx"} |= "error" | json | status_code >= 400

# Mimir для long-term storage метрик Prometheus
version: '3.8'
services:
  mimir:
    image: grafana/mimir:latest
    ports:
      - "9009:9009"
    volumes:
      - ./mimir.yml:/etc/mimir/mimir.yml
    command:
      - --config.file=/etc/mimir/mimir.yml
      - --target=all

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --enable-feature=remote-write-receiver

# prometheus.yml
global:
  scrape_interval: 15s

remote_write:
  - url: http://mimir:9009/api/v1/push
    queue_config:
      capacity: 10000
      max_shards: 1000
      min_shards: 1
      max_samples_per_send: 2000

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

✅ **Horizontal scaling**: автоматическое масштабирование
✅ **Multi-tenancy**: изоляция данных разных команд
✅ **Long-term storage**: эффективное хранение исторических данных
✅ **Prometheus compatibility**: 100% совместимость с Prometheus API
✅ **Cloud native**: оптимизирован для Kubernetes и облачных хранилищ

# docker-compose.yml для ELK
version: '3'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.0
    environment:
      - discovery.type=single-node
    ports:
      - "9200:9200"

  logstash:
    image: docker.elastic.co/logstash/logstash:7.15.0
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    ports:
      - "5000:5000"

  kibana:
    image: docker.elastic.co/kibana/kibana:7.15.0
    ports:
      - "5601:5601"

# alert.rules.yml
groups:
- name: example
  rules:
  - alert: HighErrorRate
    expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High error rate on {{ $labels.instance }}"
      description: "Error rate is {{ $value }} requests per second"

  - alert: DiskSpaceLow
    expr: disk_free_percent < 10
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "Disk space is running low"

version: '3.8'
services:
  app:
    build: .
    ports:
      - "5000:5000"
    
  prometheus:
    image: prom/prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./alert.rules.yml:/etc/prometheus/alert.rules.yml
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--web.enable-lifecycle'
  
  grafana:
    image: grafana/grafana
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana-storage:/var/lib/grafana
      
  alertmanager:
    image: prom/alertmanager
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml

volumes:
  grafana-storage:

# monitoring.tf (Terraform)
resource "aws_cloudwatch_dashboard" "main" {
  dashboard_name = "MyApp-Dashboard"
  
  dashboard_body = jsonencode({
    widgets = [
      {
        type = "metric"
        properties = {
          metrics = [
            ["AWS/ApplicationELB", "RequestCount"],
            ["AWS/ApplicationELB", "TargetResponseTime"]
          ]
          period = 300
          stat = "Sum"
          region = "us-west-2"
          title = "Application Metrics"
        }
      }
    ]
  })
}

# docker-compose для Zabbix
version: '3.8'
services:
  zabbix-server:
    image: zabbix/zabbix-server-mysql:latest
    environment:
      - DB_SERVER_HOST=mysql
      - MYSQL_DATABASE=zabbix
      - MYSQL_USER=zabbix
      - MYSQL_PASSWORD=zabbix_password
    ports:
      - "10051:10051"
    depends_on:
      - mysql

  zabbix-web:
    image: zabbix/zabbix-web-nginx-mysql:latest
    environment:
      - DB_SERVER_HOST=mysql
      - MYSQL_DATABASE=zabbix
      - MYSQL_USER=zabbix
      - MYSQL_PASSWORD=zabbix_password
      - ZBX_SERVER_HOST=zabbix-server
    ports:
      - "8080:8080"

  mysql:
    image: mysql:8.0
    environment:
      - MYSQL_DATABASE=zabbix
      - MYSQL_USER=zabbix
      - MYSQL_PASSWORD=zabbix_password
      - MYSQL_ROOT_PASSWORD=root_password

✅ **SNMP мониторинг**: отличная поддержка сетевого оборудования
✅ **Агенты**: мощные агенты для глубокого мониторинга ОС
✅ **Templates**: готовые шаблоны для популярных систем
✅ **Network discovery**: автоматическое обнаружение устройств
✅ **Notifications**: гибкая система уведомлений

# Установка SNMP tools
sudo apt install snmp snmp-mibs-downloader

# Проверка SNMP доступности
snmpwalk -v2c -c public 192.168.1.1 1.3.6.1.2.1.1.1.0

# Мониторинг интерфейсов
snmpwalk -v2c -c public 192.168.1.1 1.3.6.1.2.1.2.2.1.2

# CPU загрузка на Cisco
snmpget -v2c -c public 192.168.1.1 1.3.6.1.4.1.9.2.1.56.0

# Память на Cisco
snmpget -v2c -c public 192.168.1.1 1.3.6.1.4.1.9.2.1.8.0

# snmp.yml конфигурация для SNMP exporter
modules:
  cisco_router:
    walk:
      - 1.3.6.1.2.1.2.2.1.2    # Interface names
      - 1.3.6.1.2.1.2.2.1.10   # Interface inbound octets
      - 1.3.6.1.2.1.2.2.1.16   # Interface outbound octets
      - 1.3.6.1.4.1.9.2.1.56   # CPU utilization
    metrics:
      - name: snmp_cisco_cpu_utilization
        oid: 1.3.6.1.4.1.9.2.1.56
        type: gauge
      - name: snmp_interface_in_octets
        oid: 1.3.6.1.2.1.2.2.1.10
        type: counter

# prometheus.yml для SNMP мониторинга
scrape_configs:
  - job_name: 'snmp'
    static_configs:
      - targets:
        - 192.168.1.1  # Router IP
        - 192.168.1.2  # Switch IP
    metrics_path: /snmp
    params:
      module: [cisco_router]
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: snmp-exporter:9116

# Node Exporter для серверного железа
version: '3.8'
services:
  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    restart: unless-stopped
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
      - '--collector.textfile.directory=/var/lib/node_exporter/textfile_collector'
      - '--collector.systemd'
      - '--collector.processes'
    ports:
      - "9100:9100"

# CPU загрузка
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# Использование памяти
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# Использование диска
100 - ((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes)

# Network I/O
rate(node_network_receive_bytes_total[5m])
rate(node_network_transmit_bytes_total[5m])

# Disk I/O
rate(node_disk_read_bytes_total[5m])
rate(node_disk_written_bytes_total[5m])

# Load Average
node_load1
node_load5
node_load15