Обработка логов syslog + Docker

fluent-bit.conf

[SERVICE] flush 1 log_level info daemon off storage.path /var/log/flb-storage/ storage.sync normal storage.checksum off storage.max_chunks_up 128 storage.backlog.mem_limit 10M parsers_file parsers.conf http_server on http_listen 0.0.0.0 http_port 2020 coro_stack_size 24576 plugins_path /fluent-bit/bin/ # ==================== ASUS ROUTER SYSLOG INPUT ==================== [INPUT] Name syslog Listen 0.0.0.0 Port 5140 Parser syslog-rfc5424 Tag router.syslog Buffer_Chunk_Size 4MB Buffer_Max_Size 16MB Mode tcp # ==================== DOCKER LOGS INPUT ==================== [INPUT] Name tail Path /var/lib/docker/containers/*/*.log Parser docker Refresh_Interval 5 Ignore_Older 1h Docker_Mode On Tag docker.<file_name> Tag_Regex (?<file_name>[a-f0-9]*)-json.log Mem_Buf_Limit 50MB Skip_Long_Lines On DB /var/log/flb-storage/flb_db.db DB.sync normal Storage.Type filesystem Read_from_Head false # Только для отладки # [INPUT] # Name tail # Path /var/lib/docker/containers/*/*.log # Parser docker # Refresh_Interval 10 # Docker_Mode On # Tag docker.<file_name> # Tag_Regex (?<file_name>[a-f0-9]*)-json.log # Mem_Buf_Limit 50MB # Skip_Long_Lines On # DB /var/log/flb-storage/flb_db.db # DB.sync normal # Storage.Type filesystem # Read_from_Head true # ==================== DOCKER FILTERS ==================== # Фильтруем пустые логи [FILTER] name grep match docker.* Exclude log ^$ Exclude log ^\s*$ Exclude log ^==>.+<==$ # Извлекаем сырой лог [FILTER] name modify match docker.* copy log raw_log copy exception.backtrace exception_backtrace copy exception.class exception_class copy exception.message exception_message # Очищаем docker.* - оставляем ТОЛЬКО нужные поля [FILTER] name record_modifier match docker.* whitelist_key date whitelist_key log whitelist_key raw_log whitelist_key exception_backtrace whitelist_key exception_class whitelist_key exception_message whitelist_key method whitelist_key path whitelist_key action whitelist_key status whitelist_key remote_ip whitelist_key controller whitelist_key line_id [FILTER] name parser match docker.* key_name raw_log parser nginx_access reserve_data true [FILTER] name parser match docker.* key_name raw_log parser gitlab_json reserve_data true [FILTER] name parser match docker.* key_name raw_log parser gitlab_registry reserve_data true [FILTER] name parser match docker.* key_name raw_log parser sidekiq_json reserve_data true [FILTER] name parser match docker.* key_name raw_log parser mysql_error reserve_data true [FILTER] name parser match docker.* key_name raw_log parser mysql_slow reserve_data true [FILTER] name parser match docker.* key_name raw_log parser postgresql reserve_data true [FILTER] name parser match docker.* key_name raw_log parser postgresql_detailed reserve_data true [FILTER] name parser match docker.* key_name raw_log parser nextcloud_access reserve_data true Preserve_Key true [FILTER] name parser match docker.* key_name raw_log parser apache_access reserve_data true [FILTER] name parser match docker.* key_name raw_log parser php_fpm reserve_data true [FILTER] name parser match docker.* key_name raw_log parser grafana_regex reserve_data true # Структуризация распарсенных полей В raw_log [FILTER] name nest match docker.* operation nest wildcard remote_* wildcard method wildcard path wildcard status wildcard body_bytes wildcard user_agent wildcard referrer wildcard logger wildcard endpoint wildcard pluginId wildcard dsName wildcard dsUID wildcard uname wildcard level wildcard msg wildcard statusCode wildcard resourcePath wildcard exception nest_under parsed_data # Метаданные Docker [FILTER] name lua match docker.* script /fluent-bit/bin/docker-metadata.lua call enrich_with_docker_metadata # Копируем метаданные [FILTER] name modify match docker.* copy docker.hostname hostname copy docker.container_started started copy docker.container_name container_name copy docker.container_name service_name copy docker.container_id container_id copy docker.state state copy docker.stream stream copy docker.line_id line_id copy log _raw copy parsed_data _parsed # copy exception_class _class # copy exception_message _message # copy exception_backtrace _backtrace copy docker.label_project project copy docker.label_service service copy docker.label_logging logging copy docker.label_logging_jobname logging_jobname # Структурируем через nest [FILTER] name nest match docker.* operation nest wildcard _* nest_under log remove_prefix _ # Добавляем host metadata [FILTER] name modify match docker.* set node_id ${NODE_ID} set node_name ${NODE_NAME} set host_name ${NODE_NAME} # Перетагиваем только логи с enabled logging [FILTER] name rewrite_tag match docker.* rule $logging ^enabled$ data.$container_id true # ==================== ROUTER FILTERS ==================== [FILTER] Name parser Match router.* Key_Name message Parser router_logs Reserve_Data true [FILTER] Name record_modifier Match router.* Record hostname ${HOSTNAME} Record device_type router Record source asus_merlin Record cluster docker_swarm [FILTER] Name modify Match router.* Rename host source_host Rename ident facility Set log_type syslog Set environment production # Отфильтровываем служебные контейнеры [FILTER] Name grep Match docker.* Exclude container_name ^/loki.* Exclude container_name ^/fluent-bit.* Exclude container_name ^/grafana.* Exclude container_name ^/traefik.* [FILTER] name record_modifier match data.* whitelist_key date whitelist_key log whitelist_key exception_class whitelist_key exception_message whitelist_key exception_backtrace whitelist_key method whitelist_key path whitelist_key action whitelist_key status whitelist_key remote_ip whitelist_key controller whitelist_key line_id whitelist_key node_id whitelist_key node_name whitelist_key host_name whitelist_key hostname whitelist_key started whitelist_key container_name whitelist_key service_name whitelist_key container_id whitelist_key stream whitelist_key project whitelist_key service whitelist_key logging_jobname # Логи docker в Loki [OUTPUT] name loki match data.* host loki port 3100 labels job=$logging_jobname, node_name=$node_name, container_id=$container_id, container_name=$container_name, service_name=$service_name, project=$project, service=$service, level=$stream line_format json auto_kubernetes_labels off # Логи роутера в Loki [OUTPUT] Name loki Match router.* Host loki Port 3100 Labels job=asus_router, device_type=router, source=syslog, cluster=docker_swarm Line_Format key_value Auto_Kubernetes_Labels off # Только для отладки # [OUTPUT] # name stdout # match data.* # format json

loki_config

auth_enabled: false server: http_listen_port: 3100 common: instance_addr: 127.0.0.1 path_prefix: /loki storage: filesystem: chunks_directory: /loki/chunks rules_directory: /loki/rules replication_factor: 1 ring: kvstore: store: inmemory schema_config: configs: - from: 2020-10-24 store: tsdb object_store: filesystem schema: v13 index: prefix: index_ period: 24h ruler: alertmanager_url: http://localhost:9093 limits_config: retention_period: 720h reject_old_samples: true reject_old_samples_max_age: 720h allow_structured_metadata: true max_query_length: 721h ingester: lifecycler: ring: kvstore: store: inmemory replication_factor: 1 final_sleep: 0s chunk_idle_period: 1h max_chunk_age: 1h chunk_target_size: 1048576 chunk_retain_period: 30s table_manager: retention_deletes_enabled: true retention_period: 720h

Развертывание

version: "3.8" x-logging: &default-logging driver: "json-file" options: max-size: "10m" max-file: "3" tag: "{{.Name}}/{{.ImageName}}" x-labels: &default-labels logging: "enabled" logging_jobname: "docker_swarm" monitoring: "true" services: loki: image: grafana/loki:3.5.8 ports: - "3100:3100" configs: - source: loki_config target: /etc/loki/local-config.yaml volumes: - loki_data:/loki command: -config.file=/etc/loki/local-config.yaml networks: - monitoring deploy: placement: constraints: - node.role == manager logging: *default-logging labels: <<: *default-labels service: "loki" component: "logging" fluent-bit: image: fluent/fluent-bit:4.2.0 configs: - source: fluent_bit_config target: /fluent-bit/etc/fluent-bit.conf - source: fluent_bit_parser target: /fluent-bit/etc/parsers.conf - source: docker_metadata target: /fluent-bit/bin/docker-metadata.lua environment: - NODE_ID={{.Node.ID}} - NODE_NAME={{.Node.Hostname}} ports: - "5140:5140/tcp" # Для приема syslog от роутера - "2020:2020" # HTTP мониторинг Fluent Bit networks: - monitoring volumes: - flb_storage:/var/log/flb-storage/ - /var/lib/docker/containers:/var/lib/docker/containers:ro deploy: mode: global grafana: image: grafana/grafana:12.1.4 ports: - "3000:3000" environment: - GF_SECURITY_ADMIN_PASSWORD=admin - GF_SECURITY_ADMIN_USER=admin volumes: - grafana_data:/var/lib/grafana networks: - monitoring deploy: placement: constraints: - node.role == manager logging: *default-logging labels: <<: *default-labels service: "grafana" component: "monitoring" configs: loki_config: external: true fluent_bit_config: external: true fluent_bit_parser: external: true docker_metadata: external: true networks: monitoring: driver: overlay volumes: flb_storage: driver: local loki_data: driver: local grafana_data: driver: local

Проверка работы

# Проверяем конфигурацию $ docker exec -it <fluentbit_container> /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf --dry-run # Смотрим логи $ docker service logs logging_fluent-bit # Проверяем метрики $ curl http://localhost:2020/api/v1/metrics | jq # Тестируем парсеры $ echo '2025-11-21T02:13:34.366Z {"method":"PUT","path":"/projects","status":500}' | \ $ docker exec -i <fluentbit_container> /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf -i stdin -o stdout

Запросы в Grafana

{job="fluent-bit"} |= "gitlab" {container_name="gitlab"} {job="fluent-bit"} |~ "(?i)error|exception|fail" {node_name="node-1"} # Все логи GitLab {container_name=~".*gitlab.*"} # Логи по компонентам {container_name=~".*gitlab.*"} | json | component="gitaly.UnaryServerInterceptor" # Ошибки {container_name=~".*gitlab.*"} | json | level="error" # Запросы с определенным correlation_id {container_name=~".*gitlab.*"} | json | correlation_id="01KAJ30DCE4BW6JSAT7KHGZ9PX" # Логи Sidekiq {container_name=~".*gitlab.*"} | json | severity="INFO" # Все логи GitLab с parsed_data {container_name=~".*gitlab.*"} | json # Логи с ошибками {environment="production"} | json | level="error" # Медленные PostgreSQL запросы {service_name=~".*postgres.*"} | json | duration > 1000 # Nginx 5xx ошибки {container_name=~".*nginx.*"} | json | status >= 500 # Sidekiq логи {container_name=~".*sidekiq.*"} | json | severity="INFO"