Библиотека знаний - software:fluent-bit-loki-grafana:fluent-bit

Обработка многострочных логов

Anonymous (anonymous@undisclosed.example.com) — 2025-11-26T00:26:35+00:00

Обработка многострочных логов

fluent-bit.conf

[SERVICE]
    Daemon Off
    Flush 1
    Log_Level info
    Parsers_File /fluent-bit/etc/parsers.conf
    HTTP_Server On
    HTTP_Listen 0.0.0.0
    HTTP_Port 2020
 
# Вход для Docker JSON логов
[INPUT]
    Name tail
    Path /var/lib/docker/containers/*/*.log
    Tag docker.*
    Parser docker
    Docker_Mode On
    Docker_Mode_Flush 4
    Mem_Buf_Limit 50MB
    Skip_Long_Lines On
    Refresh_Interval 10
    DB /var/log/flb-storage/tail.db
    DB.sync normal
 
# Парсинг JSON логов GitLab
[FILTER]
    Name parser
    Match docker.*
    Key_Name log
    Parser json_auto
    Reserve_Data On
    Preserve_Key On
 
# Добавление Docker метаданных
[FILTER]
    Name modify
    Match docker.*
    Rename docker.container_id container_id
    Rename docker.container_name container_name
    Add log_source docker
    Add node_id ${NODE_ID}
    Add node_name ${NODE_NAME}
    Add cluster docker_swarm
 
# Обработка многострочных логов (стектрейсы GitLab)
[FILTER]
    Name multiline
    Match docker.*
    multiline.key_content log
    multiline.parser java, go, python, ruby
 
# Выход в Loki
[OUTPUT]
    Name grafana-loki
    Match *
    Url http://loki:3100/loki/api/v1/push
    TenantID ""
    Labels {job="fluent-bit"}
    LabelKeys container_name,node_name,log_source
    RemoveKeys docker_id,container_id
    BatchWait 1
    BatchSize 1MB
    LineFormat json
    DropSingleKey Off

parsers_multiline.conf

[PARSER]
    Name docker
    Format json
    Time_Key time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep On
 
[PARSER]
    Name json_auto
    Format json
    Time_Key time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep On
 
# Парсер для многострочных логов GitLab
[PARSER]
    Name gitlab_multiline_firstline
    Format regex
    Regex ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z
 
[MULTILINE_PARSER]
    Name gitlab_multiline
    Type regex
    Flush_Timeout 1000
    Rule "start_state" "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z" "cont"
    Rule "cont" "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z|^{\"time\":\"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z).*" "cont"

docker_metadata

function docker_metadata(tag, timestamp, record)
    local docker = require('docker_metadata')
 
    -- Получаем container_id из пути файла
    local container_id = record["docker.container_id"]
 
    if container_id then
        -- Добавляем дополнительные метаданные
        record["container_id_short"] = string.sub(container_id, 1, 12)
        record["log_type"] = "docker_json"
    end
 
    return 1, timestamp, record
end

Установка

# Создаем конфиги
$ docker config create loki_config ./loki-config.yaml
$ docker config create fluent_bit_config ./fluent-bit.conf
$ docker config create fluent_bit_parser ./parsers.conf
$ docker config create docker_metadata ./docker-metadata.lua

loki_config

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  path: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

analytics:
  reporting_enabled: false

Развертывание

# Запускаем стек
$ docker stack deploy -c docker-compose.yml logging

Проверка работы

# Проверяем сервисы
docker service ls
 
# Смотрим логи Fluent Bit
docker service logs logging_fluent-bit
 
# Проверяем Loki
curl http://localhost:3100/ready
 
# Проверяем метрики Fluent Bit
curl http://localhost:2020/api/v1/metrics

Запросы в Grafana

{job="fluent-bit"} |= "gitlab"

{container_name="gitlab"}

{job="fluent-bit"} |~ "(?i)error|exception|fail"

{node_name="node-1"}

# Все логи GitLab
{container_name=~".*gitlab.*"}

# Логи по компонентам
{container_name=~".*gitlab.*"} | json | component="gitaly.UnaryServerInterceptor"

# Ошибки
{container_name=~".*gitlab.*"} | json | level="error"

# Запросы с определенным correlation_id
{container_name=~".*gitlab.*"} | json | correlation_id="01KAJ30DCE4BW6JSAT7KHGZ9PX"

# Логи Sidekiq
{container_name=~".*gitlab.*"} | json | severity="INFO"

Обработка логов Docker

Anonymous (anonymous@undisclosed.example.com) — 2025-12-06T03:35:25+00:00

Обработка логов Docker

fluent-bit.conf

[SERVICE]
    flush                     1
    log_level                 info
    daemon                    off
    storage.path              /var/log/flb-storage/
    storage.sync              normal
    storage.checksum          off
    storage.max_chunks_up     128
    storage.backlog.mem_limit 10M
    parsers_file              parsers.conf
    http_server               on
    http_listen               0.0.0.0
    http_port                 2020
 
[INPUT]
    Name              tail
    Path              /var/lib/docker/containers/*/*.log
    Parser            docker
    Refresh_Interval  10
    Ignore_Older      1h
    Docker_Mode       On
    Tag               docker.<file_name>
    Tag_Regex         (?<file_name>[a-f0-9]*)-json.log
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    DB                /var/log/flb-storage/flb_db.db
    DB.sync           normal
    Storage.Type      filesystem
    Read_from_Head    false
 
# Только для отладки
# [INPUT]
#     Name              tail
#     Path              /var/lib/docker/containers/*/*.log
#     Parser            docker
#     Refresh_Interval  10
#     Docker_Mode       On
#     Tag               docker.<file_name>
#     Tag_Regex         (?<file_name>[a-f0-9]*)-json.log
#     Mem_Buf_Limit     50MB
#     Skip_Long_Lines   On
#     DB                /var/log/flb-storage/flb_db.db
#     DB.sync           normal
#     Storage.Type      filesystem
#     Read_from_Head    true
 
# Фильтруем пустые логи
[FILTER]
    name                grep
    match               docker.*
    Exclude             log ^$
    Exclude             log ^\s*$
    Exclude             log ^==>.+<==$
 
# Извлекаем сырой лог
[FILTER]
    name                modify
    match               docker.*
    copy                log raw_log
    copy                exception.backtrace exception_backtrace
    copy                exception.class exception_class
    copy                exception.message exception_message
 
# Очищаем docker.* - оставляем ТОЛЬКО нужные поля
[FILTER]
    name                record_modifier
    match               docker.*
 
    whitelist_key       date
    whitelist_key       log
    whitelist_key       raw_log
    whitelist_key       exception_backtrace
    whitelist_key       exception_class
    whitelist_key       exception_message
 
    whitelist_key       method
    whitelist_key       path
    whitelist_key       action
    whitelist_key       status
    whitelist_key       remote_ip
    whitelist_key       controller
    whitelist_key       line_id
 
#
# FILTERS
#
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              nginx_access
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              gitlab_json
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              gitlab_registry
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              sidekiq_json
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              mysql_error
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              mysql_slow
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              postgresql
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              postgresql_detailed
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              nextcloud_access
    reserve_data        true
    Preserve_Key        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              apache_access
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              php_fpm
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              grafana_regex
    reserve_data        true
 
# Структуризация распарсенных полей В raw_log
[FILTER]
    name                nest
    match               docker.*
    operation           nest
    wildcard            remote_*
    wildcard            method
    wildcard            path
    wildcard            status
    wildcard            body_bytes
    wildcard            user_agent
    wildcard            referrer
    wildcard            logger
    wildcard            endpoint
    wildcard            pluginId
    wildcard            dsName
    wildcard            dsUID
    wildcard            uname
    wildcard            level
    wildcard            msg
    wildcard            statusCode
    wildcard            resourcePath
    wildcard            exception
    nest_under          parsed_data
 
#
# FILTERS
#
 
# Метаданные Docker
[FILTER]
    name                lua
    match               docker.*
    script              /fluent-bit/bin/docker-metadata.lua
    call                enrich_with_docker_metadata
 
# Копируем метаданные
[FILTER]
    name                modify
    match               docker.*
    copy                docker.hostname hostname
    copy                docker.container_started started
    copy                docker.container_name container_name
    copy                docker.container_name service_name
    copy                docker.container_id container_id
    copy                docker.state state
    copy                docker.stream stream
    copy                docker.line_id line_id
 
    copy                log _raw
    copy                parsed_data _parsed
    # copy                exception_class _class
    # copy                exception_message _message
    # copy                exception_backtrace _backtrace
 
    copy                docker.label_project project
    copy                docker.label_service service
    copy                docker.label_logging logging
    copy                docker.label_logging_jobname logging_jobname
 
# Структурируем через nest
[FILTER]
    name                nest
    match               docker.*
    operation           nest
    wildcard            _*
    nest_under          log
    remove_prefix       _
 
# Добавляем host metadata
[FILTER]
    name                modify
    match               docker.*
    set                 node_id ${NODE_ID}
    set                 node_name ${NODE_NAME}
    set                 host_name ${NODE_NAME}
 
# Перетагиваем только логи с enabled logging
[FILTER]
    name                rewrite_tag
    match               docker.*
    rule                $logging ^enabled$ data.$container_id true
 
[FILTER]
    name                record_modifier
    match               data.*
 
    whitelist_key       date
    whitelist_key       log
    whitelist_key       exception_class
    whitelist_key       exception_message
    whitelist_key       exception_backtrace
 
    whitelist_key       method
    whitelist_key       path
    whitelist_key       action
    whitelist_key       status
    whitelist_key       remote_ip
    whitelist_key       controller
    whitelist_key       line_id
 
    whitelist_key       node_id
    whitelist_key       node_name
    whitelist_key       host_name
    whitelist_key       hostname
    whitelist_key       started
    whitelist_key       container_name
    whitelist_key       service_name
    whitelist_key       container_id
    whitelist_key       stream
    whitelist_key       project
    whitelist_key       service
    whitelist_key       logging_jobname
 
[OUTPUT]
    name                loki
    match               data.*
    host                loki
    port                3100
    labels              job=$logging_jobname, node_name=$node_name, container_id=$container_id, container_name=$container_name, service_name=$service_name, project=$project, service=$service, level=$stream
    label_keys          $node_name,$container_id,$container_name,$service_name,$project,$service
    line_format         json
    auto_kubernetes_labels off
 
# Только для отладки
# [OUTPUT]
#     name                stdout
#     match               data.*
#     format              json

parsers.conf

# Базовый парсер для Docker JSON логов
[PARSER]
    Name        docker
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On
 
# Nginx access логи
[PARSER]
    Name        nginx_access
    Format      regex
    Regex       ^(?<remote_ip>[^ ]*) - - \[(?<timestamp>[^\]]*)\] "(?<method>\w+) (?<path>[^ ]*) HTTP/[0-9.]+" (?<status>\d+) (?<body_bytes>\d+) "(?<referrer>[^"]*)" "(?<user_agent>[^"]*)"
    Time_Key    timestamp
    Time_Format %d/%b/%Y:%H:%M:%S %z
 
# Gitlab JSON логи (application logs)
[PARSER]
    Name        gitlab_json
    Format      json
    # Time_Key    time    # Используем время из Docker
    Time_Keep   On
 
# GitLab Registry логи
[PARSER]
    Name        gitlab_registry
    Format      regex
    Regex       ^\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}\.\d+ time="(?<time>[^"]*)" level=(?<level>\w+) msg="(?<msg>[^"]*)"(?<rest>.*)
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On
 
# Sidekiq JSON логи
[PARSER]
    Name        sidekiq_json
    Format      json
    # Time_Key    time    # Используем время из Docker
    Time_Keep   On
 
# Nextcloud Apache-style логи
[PARSER]
    Name        nextcloud_access
    Format      regex
    Regex       ^(?<remote_ip>[^ ]*) (?<user_ident>[^ ]*) (?<user_id>[^ ]*) \[(?<timestamp>[^\]]*)\] "(?<method>\w+) (?<path>[^ ]*) HTTP/[0-9.]+" (?<status>\d+) (?<body_bytes>\d+) "(?<referrer>[^"]*)" "(?<user_agent>[^"]*)"
    Time_Key    timestamp
    Time_Format %d/%b/%Y:%H:%M:%S %z  # ← Apache/Nginx формат!
    Time_Keep   On
 
# Apache access логи
[PARSER]
    Name        apache_access
    Format      regex
    Regex       ^(?<remote_ip>[^ ]*) - - \[(?<timestamp>[^\]]*)\] "(?<method>\w+) (?<path>[^ ]*) HTTP/[0-9.]+" (?<status>\d+) (?<body_bytes>\d+) "(?<referrer>[^"]*)" "(?<user_agent>[^"]*)"
    Time_Key    timestamp
    Time_Format %d/%b/%Y:%H:%M:%S %z
 
# MySQL error логи
[PARSER]
    Name        mysql_error
    Format      regex
    Regex       ^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+Z) (?<level>\w+) (?<message>.*)
    Time_Key    timestamp
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On
 
# MySQL slow query логи
[PARSER]
    Name        mysql_slow
    Format      regex
    Regex       ^# Time: (?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+Z).*# User@Host: (?<user>[^\[]*)\[(?<database>[^\]]*)\] @ (?<host>[\w\.]*)\s*\[(?<ip>[\d\.]*)\].*# Query_time: (?<query_time>[\d\.]*) Lock_time: (?<lock_time>[\d\.]*) Rows_sent: (?<rows_sent>\d*) Rows_examined: (?<rows_examined>\d*).*use (?<used_database>\w*);.*SET timestamp=(?<timestamp_unix>\d*);(?<query>.*)
    Time_Key    timestamp
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On
 
# PostgreSQL логи
[PARSER]
    Name        postgresql
    Format      regex
    Regex       ^(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ [A-Z]{3}) \[(?<pid>\d+)\] (?<level>\w+):\s*(?<message>.*)
    Time_Key    timestamp
    Time_Format %Y-%m-%d %H:%M:%S.%L %Z
    Time_Keep   On
 
# PostgreSQL extended логи (с деталями запросов)
[PARSER]
    Name        postgresql_detailed
    Format      regex
    Regex       ^(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ [A-Z]{3}) \[(?<pid>\d+)\] (?<level>\w+):\s*duration: (?<duration>[\d\.]*) ms\s*(?<message>.*)
    Time_Key    timestamp
    Time_Format %Y-%m-%d %H:%M:%S.%L %Z
    Time_Keep   On
 
# PHP-FPM логи
[PARSER]
    Name        php_fpm
    Format      regex
    Regex       ^\[(?<timestamp>[^\]]+)\] (?<level>\w+): (?<message>.*)
    Time_Key    timestamp
    Time_Format %d-%b-%Y %H:%M:%S
    Time_Keep   On
 
# Стандартный syslog
[PARSER]
    Name        syslog_rfc3164
    Format      regex
    Regex       ^\<(?<pri>[0-9]+)\>(?<timestamp>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<message>.*)$
    Time_Key    timestamp
    Time_Format %b %d %H:%M:%S
    Time_Keep   On
 
# Grafana лог формата key=value
[PARSER]
    Name        grafana_logfmt
    Format      logfmt
    Time_Key    t
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On
 
[PARSER]
    Name        grafana_regex
    Format      regex
    Regex       logger=(?<logger>[^ ]*) endpoint=(?<endpoint>[^ ]*) pluginId=(?<pluginId>[^ ]*) dsName=(?<dsName>[^ ]*) dsUID=(?<dsUID>[^ ]*) uname=(?<uname>[^ ]*) t=(?<t>[^ ]*) level=(?<level>[^ ]*) msg="(?<msg>[^"]*)" error=(?<error>[^ ]*) statusCode=(?<statusCode>[^ ]*) resourcePath="(?<resourcePath>[^"]*)"
    Time_Key    t
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On

Установка

loki_config

auth_enabled: false

server:
  http_listen_port: 3100

common:
  instance_addr: 127.0.0.1
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2020-10-24
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

limits_config:
  retention_period: 720h
  reject_old_samples: true
  reject_old_samples_max_age: 720h
  allow_structured_metadata: true
  max_query_length: 721h
  
ingester:
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 1h
  max_chunk_age: 1h
  chunk_target_size: 1048576
  chunk_retain_period: 30s

table_manager:
  retention_deletes_enabled: true
  retention_period: 720h

Развертывание

Проверка работы

# Проверяем конфигурацию
$ docker exec -it <fluentbit_container> /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf --dry-run
 
# Смотрим логи
$ docker service logs logging_fluent-bit
 
# Проверяем метрики
$ curl http://localhost:2020/api/v1/metrics | jq
 
# Тестируем парсеры
$ echo '2025-11-21T02:13:34.366Z {"method":"PUT","path":"/projects","status":500}' | \
$ docker exec -i <fluentbit_container> /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf -i stdin -o stdout

Запросы в Grafana

{job="fluent-bit"} |= "gitlab"
 
{container_name="gitlab"} 
 
{job="fluent-bit"} |~ "(?i)error|exception|fail"
 
{node_name="node-1"}
 
# Все логи GitLab
{container_name=~".*gitlab.*"} 
 
# Логи по компонентам
{container_name=~".*gitlab.*"} | json | component="gitaly.UnaryServerInterceptor"
 
# Ошибки
{container_name=~".*gitlab.*"} | json | level="error"
 
# Запросы с определенным correlation_id
{container_name=~".*gitlab.*"} | json | correlation_id="01KAJ30DCE4BW6JSAT7KHGZ9PX"
 
# Логи Sidekiq
{container_name=~".*gitlab.*"} | json | severity="INFO"
 
# Все логи GitLab с parsed_data
{container_name=~".*gitlab.*"} | json
 
# Логи с ошибками
{environment="production"} | json | level="error" 
 
# Медленные PostgreSQL запросы
{service_name=~".*postgres.*"} | json | duration > 1000
 
# Nginx 5xx ошибки
{container_name=~".*nginx.*"} | json | status >= 500
 
# Sidekiq логи
{container_name=~".*sidekiq.*"} | json | severity="INFO"

Обработка логов syslog + Docker

Anonymous (anonymous@undisclosed.example.com) — 2025-12-07T02:11:51+00:00

Обработка логов syslog + Docker

fluent-bit.conf

[SERVICE]
    flush                     1
    log_level                 info
    daemon                    off
    storage.path              /var/log/flb-storage/
    storage.sync              normal
    storage.checksum          off
    storage.max_chunks_up     128
    storage.backlog.mem_limit 10M
    parsers_file              parsers.conf
    http_server               on
    http_listen               0.0.0.0
    http_port                 2020
    coro_stack_size           24576
    plugins_path              /fluent-bit/bin/
 
# ==================== ASUS ROUTER SYSLOG INPUT ====================
# [INPUT]
#     name              syslog
#     listen            0.0.0.0
#     port              5140
#     parser            syslog-rfc5424
#     tag               router.syslog
#     buffer_chunk_size 4MB
#     buffer_max_size   16MB
#     mode              tcp
 
# ==================== DOCKER LOGS INPUT ====================
[INPUT]
    name              tail
    path              /var/lib/docker/containers/*/*.log
    parser            docker
    refresh_interval  5
    ignore_older      1h
    docker_mode       on
    tag               docker.<file_name>
    tag_regex         (?<file_name>[a-f0-9]*)-json.log
    mem_buf_limit     50MB
    skip_long_lines   on
    db                /var/log/flb-storage/flb_db.db
    db.sync           normal
    storage.type      filesystem
    read_from_head    false
 
# Только для отладки
# [INPUT]
#     Name              tail
#     Path              /var/lib/docker/containers/*/*.log
#     Parser            docker
#     Refresh_Interval  10
#     Docker_Mode       On
#     Tag               docker.<file_name>
#     Tag_Regex         (?<file_name>[a-f0-9]*)-json.log
#     Mem_Buf_Limit     50MB
#     Skip_Long_Lines   On
#     DB                /var/log/flb-storage/flb_db.db
#     DB.sync           normal
#     Storage.Type      filesystem
#     Read_from_Head    true
 
# ==================== DOCKER FILTERS ====================
 
# Фильтруем пустые логи
[FILTER]
    name                grep
    match               docker.*
    exclude             log ^$
    exclude             log ^\s*$
    exclude             log ^==>.+<==$
 
# Извлекаем сырой лог
[FILTER]
    name                modify
    match               docker.*
    copy                log raw_log
    copy                exception.backtrace exception_backtrace
    copy                exception.class exception_class
    copy                exception.message exception_message
 
# Очищаем docker.* - оставляем ТОЛЬКО нужные поля
[FILTER]
    name                record_modifier
    match               docker.*
 
    whitelist_key       date
    whitelist_key       log
    whitelist_key       raw_log
    whitelist_key       exception_backtrace
    whitelist_key       exception_class
    whitelist_key       exception_message
 
    whitelist_key       method
    whitelist_key       path
    whitelist_key       action
    whitelist_key       status
    whitelist_key       remote_ip
    whitelist_key       controller
    whitelist_key       line_id
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              nginx_access
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              gitlab_json
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              gitlab_registry
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              sidekiq_json
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              mysql_error
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              mysql_slow
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              postgresql
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              postgresql_detailed
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              nextcloud_access
    reserve_data        true
    preserve_key        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              apache_access
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              php_fpm
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              grafana_regex
    reserve_data        true
 
# Структуризация распарсенных полей В raw_log
[FILTER]
    name                nest
    match               docker.*
    operation           nest
    wildcard            remote_*
    wildcard            method
    wildcard            path
    wildcard            status
    wildcard            body_bytes
    wildcard            user_agent
    wildcard            referrer
    wildcard            logger
    wildcard            endpoint
    wildcard            pluginId
    wildcard            dsName
    wildcard            dsUID
    wildcard            uname
    wildcard            level
    wildcard            msg
    wildcard            statusCode
    wildcard            resourcePath
    wildcard            exception
    nest_under          parsed_data
 
# Метаданные Docker
[FILTER]
    name                lua
    match               docker.*
    script              /fluent-bit/bin/docker-metadata.lua
    call                enrich_with_docker_metadata
 
# Копируем метаданные
[FILTER]
    name                modify
    match               docker.*
    copy                docker.hostname hostname
    copy                docker.container_started started
    copy                docker.container_name container_name
    copy                docker.container_name service_name
    copy                docker.container_id container_id
    copy                docker.state state
    copy                docker.stream stream
    copy                docker.line_id line_id
 
    copy                log _raw
    copy                parsed_data _parsed
    # copy                exception_class _class
    # copy                exception_message _message
    # copy                exception_backtrace _backtrace
 
    copy                docker.label_project project
    copy                docker.label_service service
    copy                docker.label_logging logging
    copy                docker.label_logging_jobname logging_jobname
 
# Структурируем через nest
[FILTER]
    name                nest
    match               docker.*
    operation           nest
    wildcard            _*
    nest_under          log
    remove_prefix       _
 
# Добавляем host metadata
[FILTER]
    name                modify
    match               docker.*
    set                 node_id ${NODE_ID}
    set                 node_name ${NODE_NAME}
    set                 host_name ${NODE_NAME}
 
# Перетагиваем только логи с enabled logging
[FILTER]
    name                rewrite_tag
    match               docker.*
    rule                $logging ^enabled$ data.$container_id true
 
# ==================== ROUTER FILTERS ====================
 
# [FILTER]
#     name                parser
#     match               router.*
#     key_name            message
#     parser              router_logs
#     reserve_data        true
 
# [FILTER]
#     name                record_modifier
#     match               router.*
#     record              hostname ${HOSTNAME}
#     record              device_type router
#     record              source asus_merlin
#     record              cluster docker_swarm
 
# [FILTER]
#     name                modify
#     match               router.*
#     rename              host source_host
#     rename              ident facility
#     set                 log_type syslog
#     set                 environment production
 
# ==================== DOCKER FILTERS ====================
 
# Отфильтровываем служебные контейнеры
[FILTER]
    name                grep
    match               docker.*
    exclude             container_name ^/loki.*
    exclude             container_name ^/fluent-bit.*
    exclude             container_name ^/grafana.*
    exclude             container_name ^/traefik.*
 
[FILTER]
    name                record_modifier
    match               data.*
 
    whitelist_key       date
    whitelist_key       log
    whitelist_key       exception_class
    whitelist_key       exception_message
    whitelist_key       exception_backtrace
 
    whitelist_key       method
    whitelist_key       path
    whitelist_key       action
    whitelist_key       status
    whitelist_key       remote_ip
    whitelist_key       controller
    whitelist_key       line_id
 
    whitelist_key       node_id
    whitelist_key       node_name
    whitelist_key       host_name
    whitelist_key       hostname
    whitelist_key       started
    whitelist_key       container_name
    whitelist_key       service_name
    whitelist_key       container_id
    whitelist_key       stream
    whitelist_key       project
    whitelist_key       service
    whitelist_key       logging_jobname
 
# ==================== OUTPUTS ====================
 
# Логи docker в Loki
[OUTPUT]
    name                loki
    match               data.*
    host                loki
    port                3100
    labels              job=$logging_jobname, node_name=$node_name, container_id=$container_id, container_name=$container_name, service_name=$service_name, project=$project, service=$service, level=$stream
    line_format         json
    auto_kubernetes_labels off
 
# Логи роутера в Loki
# [OUTPUT]
#     name                loki
#     match               router.*
#     host                loki
#     port                3100
#     labels              job=asus_router, device_type=router, source=syslog, cluster=docker_swarm
#     line_format         key_value
#     auto_kubernetes_labels off
 
 
# Только для отладки
# [OUTPUT]
#     name                stdout
#     match               data.*
#     format              json

parsers.conf

loki_config

auth_enabled: false

server:
  http_listen_port: 3100

common:
  instance_addr: 127.0.0.1
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2020-10-24
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

limits_config:
  retention_period: 720h
  reject_old_samples: true
  reject_old_samples_max_age: 720h
  allow_structured_metadata: true
  max_query_length: 721h
  
ingester:
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 1h
  max_chunk_age: 1h
  chunk_target_size: 1048576
  chunk_retain_period: 30s

table_manager:
  retention_deletes_enabled: true
  retention_period: 720h

Развертывание

version: "3.8"
 
x-logging: &default-logging
  driver: "json-file"
  options:
    max-size: "10m"
    max-file: "3"
    tag: "{{.Name}}/{{.ImageName}}"
 
x-labels: &default-labels
  logging: "enabled"
  logging_jobname: "docker_swarm"
  monitoring: "true"
 
services:
  loki:
    image: grafana/loki:3.5.8
    ports:
      - "3100:3100"
    configs:
      - source: loki_config
        target: /etc/loki/local-config.yaml
    volumes:
      - loki_data:/loki
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - monitoring
    deploy:
      placement:
        constraints:
          - node.role == manager
    logging: *default-logging
    labels:
      <<: *default-labels
      service: "loki"
      component: "logging"
 
  fluent-bit:
    image: fluent/fluent-bit:4.2.0
    configs:
      - source: fluent_bit_config
        target: /fluent-bit/etc/fluent-bit.conf
      - source: fluent_bit_parser
        target: /fluent-bit/etc/parsers.conf
      - source: docker_metadata
        target: /fluent-bit/bin/docker-metadata.lua
    environment:
      - NODE_ID={{.Node.ID}}
      - NODE_NAME={{.Node.Hostname}}
    ports:
      - "5140:5140/tcp"  # Для приема syslog от роутера
      - "2020:2020"      # HTTP мониторинг Fluent Bit
    networks:
      - monitoring
    volumes:
      - flb_storage:/var/log/flb-storage/
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
    deploy:
      mode: global
 
 
  grafana:
    image: grafana/grafana:12.1.4
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_SECURITY_ADMIN_USER=admin
    volumes:
      - grafana_data:/var/lib/grafana
    networks:
      - monitoring
    deploy:
      placement:
        constraints:
          - node.role == manager
    logging: *default-logging
    labels:
      <<: *default-labels
      service: "grafana"
      component: "monitoring"
 
configs:
  loki_config:
    external: true
  fluent_bit_config:
    external: true
  fluent_bit_parser:
    external: true
  docker_metadata:
    external: true
 
networks:
  monitoring:
    driver: overlay
 
volumes:
  flb_storage:
    driver: local
  loki_data:
    driver: local
  grafana_data:
    driver: local

Проверка работы

# Проверяем конфигурацию
$ docker exec -it <fluentbit_container> /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf --dry-run
 
# Смотрим логи
$ docker service logs logging_fluent-bit
 
# Проверяем метрики
$ curl http://localhost:2020/api/v1/metrics | jq
 
# Тестируем парсеры
$ echo '2025-11-21T02:13:34.366Z {"method":"PUT","path":"/projects","status":500}' | \
$ docker exec -i <fluentbit_container> /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf -i stdin -o stdout

Запросы в Grafana

{job="fluent-bit"} |= "gitlab"
 
{container_name="gitlab"} 
 
{job="fluent-bit"} |~ "(?i)error|exception|fail"
 
{node_name="node-1"}
 
# Все логи GitLab
{container_name=~".*gitlab.*"} 
 
# Логи по компонентам
{container_name=~".*gitlab.*"} | json | component="gitaly.UnaryServerInterceptor"
 
# Ошибки
{container_name=~".*gitlab.*"} | json | level="error"
 
# Запросы с определенным correlation_id
{container_name=~".*gitlab.*"} | json | correlation_id="01KAJ30DCE4BW6JSAT7KHGZ9PX"
 
# Логи Sidekiq
{container_name=~".*gitlab.*"} | json | severity="INFO"
 
# Все логи GitLab с parsed_data
{container_name=~".*gitlab.*"} | json
 
# Логи с ошибками
{environment="production"} | json | level="error" 
 
# Медленные PostgreSQL запросы
{service_name=~".*postgres.*"} | json | duration > 1000
 
# Nginx 5xx ошибки
{container_name=~".*nginx.*"} | json | status >= 500
 
# Sidekiq логи
{container_name=~".*sidekiq.*"} | json | severity="INFO"

Обработка логов syslog + Docker

Anonymous (anonymous@undisclosed.example.com) — 2025-12-07T00:56:48+00:00

Обработка логов syslog + Docker

fluent-bit.conf

[SERVICE]
    flush                     1
    log_level                 info
    daemon                    off
    storage.path              /var/log/flb-storage/
    storage.sync              normal
    storage.checksum          off
    storage.max_chunks_up     128
    storage.backlog.mem_limit 10M
    parsers_file              parsers.conf
    http_server               on
    http_listen               0.0.0.0
    http_port                 2020
    coro_stack_size           24576
    plugins_path              /fluent-bit/bin/
 
# ==================== ASUS ROUTER SYSLOG INPUT ====================
[INPUT]
    Name              syslog
    Listen            0.0.0.0
    Port              5140
    Parser            syslog-rfc5424
    Tag               router.syslog
    Buffer_Chunk_Size 4MB
    Buffer_Max_Size   16MB
    Mode              tcp
 
# ==================== DOCKER LOGS INPUT ====================
[INPUT]
    Name              tail
    Path              /var/lib/docker/containers/*/*.log
    Parser            docker
    Refresh_Interval  5
    Ignore_Older      1h
    Docker_Mode       On
    Tag               docker.<file_name>
    Tag_Regex         (?<file_name>[a-f0-9]*)-json.log
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    DB                /var/log/flb-storage/flb_db.db
    DB.sync           normal
    Storage.Type      filesystem
    Read_from_Head    false
 
# Только для отладки
# [INPUT]
#     Name              tail
#     Path              /var/lib/docker/containers/*/*.log
#     Parser            docker
#     Refresh_Interval  10
#     Docker_Mode       On
#     Tag               docker.<file_name>
#     Tag_Regex         (?<file_name>[a-f0-9]*)-json.log
#     Mem_Buf_Limit     50MB
#     Skip_Long_Lines   On
#     DB                /var/log/flb-storage/flb_db.db
#     DB.sync           normal
#     Storage.Type      filesystem
#     Read_from_Head    true
 
# ==================== DOCKER FILTERS ====================
 
# Фильтруем пустые логи
[FILTER]
    name                grep
    match               docker.*
    Exclude             log ^$
    Exclude             log ^\s*$
    Exclude             log ^==>.+<==$
 
# Извлекаем сырой лог
[FILTER]
    name                modify
    match               docker.*
    copy                log raw_log
    copy                exception.backtrace exception_backtrace
    copy                exception.class exception_class
    copy                exception.message exception_message
 
# Очищаем docker.* - оставляем ТОЛЬКО нужные поля
[FILTER]
    name                record_modifier
    match               docker.*
 
    whitelist_key       date
    whitelist_key       log
    whitelist_key       raw_log
    whitelist_key       exception_backtrace
    whitelist_key       exception_class
    whitelist_key       exception_message
 
    whitelist_key       method
    whitelist_key       path
    whitelist_key       action
    whitelist_key       status
    whitelist_key       remote_ip
    whitelist_key       controller
    whitelist_key       line_id
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              nginx_access
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              gitlab_json
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              gitlab_registry
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              sidekiq_json
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              mysql_error
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              mysql_slow
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              postgresql
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              postgresql_detailed
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              nextcloud_access
    reserve_data        true
    Preserve_Key        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              apache_access
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              php_fpm
    reserve_data        true
 
[FILTER]
    name                parser
    match               docker.*
    key_name            raw_log
    parser              grafana_regex
    reserve_data        true
 
# Структуризация распарсенных полей В raw_log
[FILTER]
    name                nest
    match               docker.*
    operation           nest
    wildcard            remote_*
    wildcard            method
    wildcard            path
    wildcard            status
    wildcard            body_bytes
    wildcard            user_agent
    wildcard            referrer
    wildcard            logger
    wildcard            endpoint
    wildcard            pluginId
    wildcard            dsName
    wildcard            dsUID
    wildcard            uname
    wildcard            level
    wildcard            msg
    wildcard            statusCode
    wildcard            resourcePath
    wildcard            exception
    nest_under          parsed_data
 
# Метаданные Docker
[FILTER]
    name                lua
    match               docker.*
    script              /fluent-bit/bin/docker-metadata.lua
    call                enrich_with_docker_metadata
 
# Копируем метаданные
[FILTER]
    name                modify
    match               docker.*
    copy                docker.hostname hostname
    copy                docker.container_started started
    copy                docker.container_name container_name
    copy                docker.container_name service_name
    copy                docker.container_id container_id
    copy                docker.state state
    copy                docker.stream stream
    copy                docker.line_id line_id
 
    copy                log _raw
    copy                parsed_data _parsed
    # copy                exception_class _class
    # copy                exception_message _message
    # copy                exception_backtrace _backtrace
 
    copy                docker.label_project project
    copy                docker.label_service service
    copy                docker.label_logging logging
    copy                docker.label_logging_jobname logging_jobname
 
# Структурируем через nest
[FILTER]
    name                nest
    match               docker.*
    operation           nest
    wildcard            _*
    nest_under          log
    remove_prefix       _
 
# Добавляем host metadata
[FILTER]
    name                modify
    match               docker.*
    set                 node_id ${NODE_ID}
    set                 node_name ${NODE_NAME}
    set                 host_name ${NODE_NAME}
 
# Перетагиваем только логи с enabled logging
[FILTER]
    name                rewrite_tag
    match               docker.*
    rule                $logging ^enabled$ data.$container_id true
 
# ==================== ROUTER FILTERS ====================
 
[FILTER]
    Name                parser
    Match               router.*
    Key_Name            message
    Parser              router_logs
    Reserve_Data        true
 
[FILTER]
    Name                record_modifier
    Match               router.*
    Record              hostname ${HOSTNAME}
    Record              device_type router
    Record              source asus_merlin
    Record              cluster docker_swarm
 
[FILTER]
    Name                modify
    Match               router.*
    Rename              host source_host
    Rename              ident facility
    Set                 log_type syslog
    Set                 environment production
 
# Отфильтровываем служебные контейнеры
[FILTER]
    Name                grep
    Match               docker.*
    Exclude             container_name ^/loki.*
    Exclude             container_name ^/fluent-bit.*
    Exclude             container_name ^/grafana.*
    Exclude             container_name ^/traefik.*
 
[FILTER]
    name                record_modifier
    match               data.*
 
    whitelist_key       date
    whitelist_key       log
    whitelist_key       exception_class
    whitelist_key       exception_message
    whitelist_key       exception_backtrace
 
    whitelist_key       method
    whitelist_key       path
    whitelist_key       action
    whitelist_key       status
    whitelist_key       remote_ip
    whitelist_key       controller
    whitelist_key       line_id
 
    whitelist_key       node_id
    whitelist_key       node_name
    whitelist_key       host_name
    whitelist_key       hostname
    whitelist_key       started
    whitelist_key       container_name
    whitelist_key       service_name
    whitelist_key       container_id
    whitelist_key       stream
    whitelist_key       project
    whitelist_key       service
    whitelist_key       logging_jobname
 
# Логи docker в Loki
[OUTPUT]
    name                loki
    match               data.*
    host                loki
    port                3100
    labels              job=$logging_jobname, node_name=$node_name, container_id=$container_id, container_name=$container_name, service_name=$service_name, project=$project, service=$service, level=$stream
    line_format         json
    auto_kubernetes_labels off
 
# Логи роутера в Loki
[OUTPUT]
    Name                loki
    Match               router.*
    Host                loki
    Port                3100
    Labels              job=asus_router, device_type=router, source=syslog, cluster=docker_swarm
    Line_Format         key_value
    Auto_Kubernetes_Labels off
 
# Только для отладки
# [OUTPUT]
#     name                stdout
#     match               data.*
#     format              json

loki_config

auth_enabled: false

server:
  http_listen_port: 3100

common:
  instance_addr: 127.0.0.1
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2020-10-24
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

limits_config:
  retention_period: 720h
  reject_old_samples: true
  reject_old_samples_max_age: 720h
  allow_structured_metadata: true
  max_query_length: 721h
  
ingester:
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 1h
  max_chunk_age: 1h
  chunk_target_size: 1048576
  chunk_retain_period: 30s

table_manager:
  retention_deletes_enabled: true
  retention_period: 720h

Развертывание

version: "3.8"
 
x-logging: &default-logging
  driver: "json-file"
  options:
    max-size: "10m"
    max-file: "3"
    tag: "{{.Name}}/{{.ImageName}}"
 
x-labels: &default-labels
  logging: "enabled"
  logging_jobname: "docker_swarm"
  monitoring: "true"
 
services:
  loki:
    image: grafana/loki:3.5.8
    ports:
      - "3100:3100"
    configs:
      - source: loki_config
        target: /etc/loki/local-config.yaml
    volumes:
      - loki_data:/loki
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - monitoring
    deploy:
      placement:
        constraints:
          - node.role == manager
    logging: *default-logging
    labels:
      <<: *default-labels
      service: "loki"
      component: "logging"
 
  fluent-bit:
    image: fluent/fluent-bit:4.2.0
    configs:
      - source: fluent_bit_config
        target: /fluent-bit/etc/fluent-bit.conf
      - source: fluent_bit_parser
        target: /fluent-bit/etc/parsers.conf
      - source: docker_metadata
        target: /fluent-bit/bin/docker-metadata.lua
    environment:
      - NODE_ID={{.Node.ID}}
      - NODE_NAME={{.Node.Hostname}}
    ports:
      - "5140:5140/tcp"  # Для приема syslog от роутера
      - "2020:2020"      # HTTP мониторинг Fluent Bit
    networks:
      - monitoring
    volumes:
      - flb_storage:/var/log/flb-storage/
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
    deploy:
      mode: global
 
 
  grafana:
    image: grafana/grafana:12.1.4
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_SECURITY_ADMIN_USER=admin
    volumes:
      - grafana_data:/var/lib/grafana
    networks:
      - monitoring
    deploy:
      placement:
        constraints:
          - node.role == manager
    logging: *default-logging
    labels:
      <<: *default-labels
      service: "grafana"
      component: "monitoring"
 
configs:
  loki_config:
    external: true
  fluent_bit_config:
    external: true
  fluent_bit_parser:
    external: true
  docker_metadata:
    external: true
 
networks:
  monitoring:
    driver: overlay
 
volumes:
  flb_storage:
    driver: local
  loki_data:
    driver: local
  grafana_data:
    driver: local

Проверка работы

# Проверяем конфигурацию
$ docker exec -it <fluentbit_container> /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf --dry-run
 
# Смотрим логи
$ docker service logs logging_fluent-bit
 
# Проверяем метрики
$ curl http://localhost:2020/api/v1/metrics | jq
 
# Тестируем парсеры
$ echo '2025-11-21T02:13:34.366Z {"method":"PUT","path":"/projects","status":500}' | \
$ docker exec -i <fluentbit_container> /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf -i stdin -o stdout

Запросы в Grafana

{job="fluent-bit"} |= "gitlab"
 
{container_name="gitlab"} 
 
{job="fluent-bit"} |~ "(?i)error|exception|fail"
 
{node_name="node-1"}
 
# Все логи GitLab
{container_name=~".*gitlab.*"} 
 
# Логи по компонентам
{container_name=~".*gitlab.*"} | json | component="gitaly.UnaryServerInterceptor"
 
# Ошибки
{container_name=~".*gitlab.*"} | json | level="error"
 
# Запросы с определенным correlation_id
{container_name=~".*gitlab.*"} | json | correlation_id="01KAJ30DCE4BW6JSAT7KHGZ9PX"
 
# Логи Sidekiq
{container_name=~".*gitlab.*"} | json | severity="INFO"
 
# Все логи GitLab с parsed_data
{container_name=~".*gitlab.*"} | json
 
# Логи с ошибками
{environment="production"} | json | level="error" 
 
# Медленные PostgreSQL запросы
{service_name=~".*postgres.*"} | json | duration > 1000
 
# Nginx 5xx ошибки
{container_name=~".*nginx.*"} | json | status >= 500
 
# Sidekiq логи
{container_name=~".*sidekiq.*"} | json | severity="INFO"

Обработка многострочных логов

Anonymous (anonymous@undisclosed.example.com) — 2025-11-26T00:30:31+00:00

Обработка многострочных логов

fluent-bit.conf

parsers_multiline.conf

Установка

loki_config

Развертывание

Проверка работы

Запросы в Grafana

{job="fluent-bit"} |= "gitlab"

{container_name="gitlab"}

{job="fluent-bit"} |~ "(?i)error|exception|fail"

{node_name="node-1"}

# Все логи GitLab
{container_name=~".*gitlab.*"}

# Логи по компонентам
{container_name=~".*gitlab.*"} | json | component="gitaly.UnaryServerInterceptor"

# Ошибки
{container_name=~".*gitlab.*"} | json | level="error"

# Запросы с определенным correlation_id
{container_name=~".*gitlab.*"} | json | correlation_id="01KAJ30DCE4BW6JSAT7KHGZ9PX"

# Логи Sidekiq
{container_name=~".*gitlab.*"} | json | severity="INFO"

Anonymous (anonymous@undisclosed.example.com) — 2025-11-26T00:28:46+00:00

Страница была перемещена по адресу Fluent-Bit.