Docker Swarm + Loki + Protmail + Grafana

• grafana/loki:3.5.8
• grafana/promtail:3.5.8
• grafana/grafana:10.2.2

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096
  log_level: info
  # Таймауты для стабильности
  http_server_read_timeout: 300s
  http_server_write_timeout: 300s

common:
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    instance_addr: 0.0.0.0
    kvstore:
      store: inmemory
 
# Критически важные настройки для query-frontend
query_scheduler:
  max_outstanding_requests_per_tenant: 2048

frontend:
  # Параметр max_outstanding_per_tenant удален из этой секции
  log_queries_longer_than: 30s
  compress_responses: true

query_range:
  parallelise_shardable_queries: true
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100
  max_retries: 5

limits_config:
  # Таймаут запросов теперь настраивается здесь
  query_timeout: 5m
  
  ingestion_rate_mb: 1000
  ingestion_burst_size_mb: 2000
  max_line_size: 256000
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  max_entries_limit_per_query: 50000
  retention_period: 8760h
  max_query_length: 721h
  max_query_parallelism: 32

schema_config:
  configs:
    - from: 2024-01-01
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

ingester:
  wal:
    enabled: true
    dir: /loki/wal
    flush_on_shutdown: true
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
      heartbeat_timeout: 1m
  chunk_idle_period: 1h
  max_chunk_age: 2h
  chunk_target_size: 1572864
  chunk_retain_period: 30s

storage_config:
  tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/tsdb-shipper-cache
  filesystem:
    directory: /loki/chunks

compactor:
  working_directory: /loki/boltdb-shipper-compactor
  retention_enabled: true
  compaction_interval: 30m
  # ИСПРАВЛЕНИЕ: Добавляем конфигурацию для delete request store
  delete_request_store: filesystem
  delete_request_cancel_period: 24h
 

querier:
  max_concurrent: 256
  tail_max_duration: 1h

analytics:
  reporting_enabled: false

server:
  http_listen_port: 9080
  http_listen_address: 0.0.0.0

positions:
  filename: /tmp/positions.yaml
  # Запись позиций каждые 15 секунд
  sync_period: 15s

clients:
  - url: http://loki:3100/loki/api/v1/push
    backoff_config:
      min_period: 10s
      max_period: 5m
      max_retries: 50
    batchwait: 30s
    batchsize: 2097152
    timeout: 60s
    external_labels:
      cluster: docker-swarm
      host: "${HOSTNAME}"

scrape_configs:

  - job_name: docker_containers
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 60s
        filters:
          - name: status
            values: ["running"]
    relabel_configs:
      - target_label: job
        replacement: docker_containers
      - source_labels: [__meta_docker_container_name]
        regex: '/(.*)'
        target_label: container
        action: replace
      - source_labels: [__meta_docker_container_label_com_docker_swarm_service_name]
        target_label: service
        regex: ".+"
        action: keep
      - source_labels: [__meta_docker_stack_namespace]
        target_label: stack
        action: replace
      - source_labels: [__meta_docker_swarm_node_id]
        target_label: node_id
        action: replace
      - source_labels: [__meta_docker_container_label_com_docker_swarm_node_hostname]
        target_label: node
        action: replace
      - source_labels: [__meta_docker_container_log_stream]
        target_label: log_stream
        action: replace
      - source_labels: [__meta_docker_container_id]
        target_label: container_id
        action: replace
      - source_labels: [__meta_docker_container_image]
        target_label: image
        action: replace
      # Правило дропа для тестирования
      # - source_labels: [__meta_docker_container_name]
      #   regex: '(promtail|loki|grafana)'
      #   action: drop
    pipeline_stages:
      - docker: {}
      - timestamp:
          source: current_time
          format: RFC3339

  - job_name: journal_logs
    journal:
      max_age: 24h
      path: /var/log/journal
      labels:
        job: systemd_journal
    relabel_configs:
      - source_labels: [__journal__hostname]
        target_label: node
      - source_labels: [__journal__systemd_unit]
        target_label: unit
      - source_labels: [__journal__syslog_identifier]
        target_label: component
    pipeline_stages:
      - timestamp:
          source: current_time
          format: RFC3339

monitoring

version: "3.8"

services:

  loki:
    image: grafana/loki:3.5.8
    ports:
      - "3100:3100"
    configs:
      - source: loki_config
        target: /etc/loki/local-config.yaml
    volumes:
      - loki_data:/loki
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
    command: 
      - -config.file=/etc/loki/local-config.yaml
      - -config.expand-env=true
      - -target=all
    networks:
      - monitoring
    deploy:
      placement:
        constraints:
          - node.role == manager
      resources:
        limits:
          memory: 2G  # Уменьшаем память
          cpus: '1.0'
        reservations:
          memory: 1G
          cpus: '0.5'
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 3

  promtail:
    image: grafana/promtail:3.5.8
    configs:
      - source: promtail_config
        target: /etc/promtail/config.yaml
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /var/log:/var/log:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/log/journal:/var/log/journal:ro
      - /run/systemd/journal:/run/systemd/journal:ro
      - promtail_positions:/tmp
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
    command: 
      - -config.file=/etc/promtail/config.yaml
      - -client.external-labels=host=${HOSTNAME}
      - -config.expand-env=true
      #- -log.level=info
    environment:
      - HOSTNAME={{.Node.Hostname}}
    networks:
      - monitoring
    deploy:
      mode: global
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
          cpus: '0.25'
      restart_policy:
        condition: any
        delay: 30s
        max_attempts: 10

  grafana:
    image: grafana/grafana:10.2.2
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_SECURITY_ADMIN_USER=admin
      - GF_USERS_ALLOW_SIGN_UP=false
    volumes:
      - grafana_data:/var/lib/grafana
      - /etc/localtime:/etc/localtime:ro
      - /etc/timezone:/etc/timezone:ro
    networks:
      - monitoring
    deploy:
      placement:
        constraints:
          - node.role == manager
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
          cpus: '0.25'

configs:
  loki_config:
    external: true
  promtail_config:
    external: true

networks:
  monitoring:
    driver: overlay
    attachable: true
 

volumes:
  promtail_positions:
    driver: local
  loki_data:
    driver: local
  grafana_data:
    driver: local