Обработка многострочных логов

fluent-bit.conf

[SERVICE] Daemon Off Flush 1 Log_Level info Parsers_File /fluent-bit/etc/parsers.conf HTTP_Server On HTTP_Listen 0.0.0.0 HTTP_Port 2020 # Вход для Docker JSON логов [INPUT] Name tail Path /var/lib/docker/containers/*/*.log Tag docker.* Parser docker Docker_Mode On Docker_Mode_Flush 4 Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 DB /var/log/flb-storage/tail.db DB.sync normal # Парсинг JSON логов GitLab [FILTER] Name parser Match docker.* Key_Name log Parser json_auto Reserve_Data On Preserve_Key On # Добавление Docker метаданных [FILTER] Name modify Match docker.* Rename docker.container_id container_id Rename docker.container_name container_name Add log_source docker Add node_id ${NODE_ID} Add node_name ${NODE_NAME} Add cluster docker_swarm # Обработка многострочных логов (стектрейсы GitLab) [FILTER] Name multiline Match docker.* multiline.key_content log multiline.parser java, go, python, ruby # Выход в Loki [OUTPUT] Name grafana-loki Match * Url http://loki:3100/loki/api/v1/push TenantID "" Labels {job="fluent-bit"} LabelKeys container_name,node_name,log_source RemoveKeys docker_id,container_id BatchWait 1 BatchSize 1MB LineFormat json DropSingleKey Off

parsers_multiline.conf

[PARSER] Name docker Format json Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%LZ Time_Keep On [PARSER] Name json_auto Format json Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%LZ Time_Keep On # Парсер для многострочных логов GitLab [PARSER] Name gitlab_multiline_firstline Format regex Regex ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z [MULTILINE_PARSER] Name gitlab_multiline Type regex Flush_Timeout 1000 Rule "start_state" "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z" "cont" Rule "cont" "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z).*" "cont"

docker_metadata

function docker_metadata(tag, timestamp, record) local docker = require('docker_metadata') -- Получаем container_id из пути файла local container_id = record["docker.container_id"] if container_id then -- Добавляем дополнительные метаданные record["container_id_short"] = string.sub(container_id, 1, 12) record["log_type"] = "docker_json" end return 1, timestamp, record end

Установка

# Создаем конфиги $ docker config create loki_config ./loki-config.yaml $ docker config create fluent_bit_config ./fluent-bit.conf $ docker config create fluent_bit_parser ./parsers.conf $ docker config create docker_metadata ./docker-metadata.lua

loki_config

auth_enabled: false server: http_listen_port: 3100 grpc_listen_port: 9096 common: path: /loki storage: filesystem: chunks_directory: /loki/chunks rules_directory: /loki/rules replication_factor: 1 ring: instance_addr: 127.0.0.1 kvstore: store: inmemory query_range: results_cache: cache: embedded_cache: enabled: true max_size_mb: 100 schema_config: configs: - from: 2020-10-24 store: boltdb-shipper object_store: filesystem schema: v11 index: prefix: index_ period: 24h ruler: alertmanager_url: http://localhost:9093 analytics: reporting_enabled: false

Развертывание

# Запускаем стек $ docker stack deploy -c docker-compose.yml logging

Проверка работы

# Проверяем сервисы docker service ls # Смотрим логи Fluent Bit docker service logs logging_fluent-bit # Проверяем Loki curl http://localhost:3100/ready # Проверяем метрики Fluent Bit curl http://localhost:2020/api/v1/metrics

Запросы в Grafana

{job="fluent-bit"} |= "gitlab"

{container_name="gitlab"}

{job="fluent-bit"} |~ "(?i)error|exception|fail"

{node_name="node-1"}