Обработка многострочных логов

[SERVICE]
    Daemon Off
    Flush 1
    Log_Level info
    Parsers_File /fluent-bit/etc/parsers.conf
    HTTP_Server On
    HTTP_Listen 0.0.0.0
    HTTP_Port 2020
 
# Вход для Docker JSON логов
[INPUT]
    Name tail
    Path /var/lib/docker/containers/*/*.log
    Tag docker.*
    Parser docker
    Docker_Mode On
    Docker_Mode_Flush 4
    Mem_Buf_Limit 50MB
    Skip_Long_Lines On
    Refresh_Interval 10
    DB /var/log/flb-storage/tail.db
    DB.sync normal
 
# Парсинг JSON логов GitLab
[FILTER]
    Name parser
    Match docker.*
    Key_Name log
    Parser json_auto
    Reserve_Data On
    Preserve_Key On
 
# Добавление Docker метаданных
[FILTER]
    Name modify
    Match docker.*
    Rename docker.container_id container_id
    Rename docker.container_name container_name
    Add log_source docker
    Add node_id ${NODE_ID}
    Add node_name ${NODE_NAME}
    Add cluster docker_swarm
 
# Обработка многострочных логов (стектрейсы GitLab)
[FILTER]
    Name multiline
    Match docker.*
    multiline.key_content log
    multiline.parser java, go, python, ruby
 
# Выход в Loki
[OUTPUT]
    Name grafana-loki
    Match *
    Url http://loki:3100/loki/api/v1/push
    TenantID ""
    Labels {job="fluent-bit"}
    LabelKeys container_name,node_name,log_source
    RemoveKeys docker_id,container_id
    BatchWait 1
    BatchSize 1MB
    LineFormat json
    DropSingleKey Off

[PARSER]
    Name docker
    Format json
    Time_Key time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep On
 
[PARSER]
    Name json_auto
    Format json
    Time_Key time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep On
 
# Парсер для многострочных логов GitLab
[PARSER]
    Name gitlab_multiline_firstline
    Format regex
    Regex ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z
 
[MULTILINE_PARSER]
    Name gitlab_multiline
    Type regex
    Flush_Timeout 1000
    Rule "start_state" "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z" "cont"
    Rule "cont" "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z).*" "cont"

function docker_metadata(tag, timestamp, record)
    local docker = require('docker_metadata')
 
    -- Получаем container_id из пути файла
    local container_id = record["docker.container_id"]
 
    if container_id then
        -- Добавляем дополнительные метаданные
        record["container_id_short"] = string.sub(container_id, 1, 12)
        record["log_type"] = "docker_json"
    end
 
    return 1, timestamp, record
end

# Создаем конфиги
$ docker config create loki_config ./loki-config.yaml
$ docker config create fluent_bit_config ./fluent-bit.conf
$ docker config create fluent_bit_parser ./parsers.conf
$ docker config create docker_metadata ./docker-metadata.lua

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  path: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

analytics:
  reporting_enabled: false

# Запускаем стек
$ docker stack deploy -c docker-compose.yml logging

# Проверяем сервисы
docker service ls
 
# Смотрим логи Fluent Bit
docker service logs logging_fluent-bit
 
# Проверяем Loki
curl http://localhost:3100/ready
 
# Проверяем метрики Fluent Bit
curl http://localhost:2020/api/v1/metrics

{job="fluent-bit"} |= "gitlab"

{container_name="gitlab"} 

{job="fluent-bit"} |~ "(?i)error|exception|fail"

{node_name="node-1"}

# Все логи GitLab
{container_name=~".*gitlab.*"} 

# Логи по компонентам
{container_name=~".*gitlab.*"} | json | component="gitaly.UnaryServerInterceptor"

# Ошибки
{container_name=~".*gitlab.*"} | json | level="error"

# Запросы с определенным correlation_id
{container_name=~".*gitlab.*"} | json | correlation_id="01KAJ30DCE4BW6JSAT7KHGZ9PX"

# Логи Sidekiq
{container_name=~".*gitlab.*"} | json | severity="INFO"