Библиотека знаний - software:monitoring:fluent-bit

Настройка логов GitLab в Docker Swarm

Anonymous (anonymous@undisclosed.example.com) — 2025-11-25T23:19:32+00:00

Настройка логов GitLab в Docker Swarm

docker-compose.fluentbit.yml

version: '3.8'

services:
  fluent-bit:
    image: fluent/fluent-bit:latest
    restart: always
    environment:
      TZ: 'Europe/Moscow'
    configs:
      - source: fluent-bit-conf
        target: /fluent-bit/etc/fluent-bit.conf
      - source: parsers-conf
        target: /fluent-bit/etc/parsers.conf
    volumes:
      # Монтируем директорию с контейнерами Docker
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      # Монтируем Docker socket для получения метаданных
      - /var/run/docker.sock:/var/run/docker.sock:ro
      # Для отслеживания позиции в файлах
      - fluent-bit-state:/tail-db
    networks:
      - localnetwork
    
    deploy:
      mode: global  # Запускаем на каждом узле Swarm
      resources:
        limits:
          memory: 300M
        reservations:
          memory: 100M

    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

volumes:
  fluent-bit-state:

configs:
  fluent-bit-conf:
    file: ./fluent-bit.conf
  parsers-conf:
    file: ./parsers.conf

networks:
  localnetwork:
    external: true

fluent-bit.conf

[SERVICE]
    Flush         1
    Log_Level     info
    Daemon        off
    Parsers_File  parsers.conf
    HTTP_Server   On
    HTTP_Listen   0.0.0.0
    HTTP_Port     2020
 
# Вход для Docker JSON логов
[INPUT]
    Name              tail
    Path              /var/lib/docker/containers/*/*.log
    Tag               docker.*
    Parser            docker
    Docker_Mode       On
    Docker_Mode_Flush 4
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10
    DB                /tail-db/docker.db
    DB.sync           normal
 
# Парсинг JSON логов
[FILTER]
    Name          parser
    Match         docker.*
    Key_Name      log
    Parser        json_auto
    Reserve_Data  On
 
# Добавление Docker метаданных
[FILTER]
    Name          kubernetes
    Match         docker.*
    Kube_Tag_Prefix docker.var.log.containers.
    Merge_Log     On
    Merge_Log_Key log_processed
    K8S-Logging.Parser On
    K8S-Logging.Exclude On
 
# Альтернативный фильтр для Docker метаданных (если kubernetes не работает)
[FILTER]
    Name          nest
    Match         docker.*
    Operation     lift
    Nested_under  log_processed
    Add_prefix    log_
 
# Обогащение логов
[FILTER]
    Name          modify
    Match         docker.*
    Add           environment production
    Add           cluster docker_swarm
    Add           log_source docker_json
 
# Выход для отладки
[OUTPUT]
    Name        stdout
    Match       *
    Format      json_lines
 
# Пример выхода в Elasticsearch
# [OUTPUT]
#     Name        es
#     Match       *
#     Host        192.168.1.10
#     Port        9200
#     Index       docker-logs
#     Type        _doc

parsers.conf

[PARSER]
    Name        docker
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On
 
[PARSER]
    Name        json_auto
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On
 
# Парсер для многострочных логов GitLab
[PARSER]
    Name        gitlab_multiline_firstline
    Format      regex
    Regex       ^{"time":"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z"
 
[MULTILINE_PARSER]
    Name          gitlab_multiline
    Type          regex
    Flush_Timeout 1000
    Rule          "start_state"  "^{"time":"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z" "cont"
    Rule          "cont"         "^(?!{"time":"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z).*" "cont"

Установка

# Создайте директорию для конфигов
$ mkdir -p fluentbit-docker
$ cd fluentbit-docker
 
# Создайте файлы конфигурации
$ cat > docker-compose.fluentbit.yml << 'EOF'
# [вставьте содержимое docker-compose.fluentbit.yml выше]
EOF
 
$ cat > fluent-bit.conf << 'EOF'
# [вставьте содержимое fluent-bit.conf выше]
EOF
 
$ cat > parsers.conf << 'EOF'
# [вставьте содержимое parsers.conf выше]
EOF
 
# Разверните стек
$ docker stack deploy -c docker-compose.fluentbit.yml fluentbit

Проверка работы

# Проверьте сервисы
docker service ls
 
# Посмотрите логи Fluent-Bit
docker service logs fluentbit_fluent-bit
 
# Проверьте, что Fluent-Bit видит файлы
docker exec -it $(docker ps -q -f name=fluentbit) ls -la /var/lib/docker/containers/

Fluent-Bit Gitlab

Anonymous (anonymous@undisclosed.example.com) — 2025-11-25T23:19:06+00:00

Fluent-Bit Gitlab

fluent-bit.conf

[SERVICE]
    Flush         1
    Log_Level     info
    Daemon        off
    Parsers_File  parsers.conf
    HTTP_Server   On
    HTTP_Listen   0.0.0.0
    HTTP_Port     2020
 
# Вход для JSON логов GitLab
[INPUT]
    Name              tail
    Path              /var/log/gitlab/gitaly/current
    Tag               gitlab.gitaly
    Parser            json
    DB                /tail-db/gitaly.db
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10
 
[INPUT]
    Name              tail
    Path              /var/log/gitlab/sidekiq/current
    Tag               gitlab.sidekiq
    Parser            json
    DB                /tail-db/sidekiq.db
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10
 
[INPUT]
    Name              tail
    Path              /var/log/gitlab/gitlab-rails/production_json.log
    Tag               gitlab.rails.json
    Parser            json
    DB                /tail-db/rails_json.db
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10
 
[INPUT]
    Name              tail
    Path              /var/log/gitlab/gitlab-rails/sidekiq_client.log
    Tag               gitlab.sidekiq_client
    Parser            json
    DB                /tail-db/sidekiq_client.db
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10
 
[INPUT]
    Name              tail
    Path              /var/log/gitlab/gitlab-workhorse/current
    Tag               gitlab.workhorse
    Parser            json
    DB                /tail-db/workhorse.db
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10
 
# Вход для многострочных текстовых логов (исключения)
[INPUT]
    Name              tail
    Path              /var/log/gitlab/gitlab-rails/production.log
    Tag               gitlab.rails.text
    Parser            gitlab_multiline
    DB                /tail-db/rails_text.db
    Mem_Buf_Limit     50MB
    Multiline         On
    Parser_Firstline  gitlab_multiline_firstline
    Refresh_Interval  10
 
# Вход для логов nginx
[INPUT]
    Name              tail
    Path              /var/log/gitlab/nginx/gitlab_access.log
    Tag               gitlab.nginx
    Parser            nginx
    DB                /tail-db/nginx.db
    Mem_Buf_Limit     50MB
    Refresh_Interval  10
 
# Вход для логов gitlab-exporter
[INPUT]
    Name              tail
    Path              /var/log/gitlab/gitlab-exporter/current
    Tag               gitlab.exporter
    Parser            gitlab_exporter
    DB                /tail-db/exporter.db
    Mem_Buf_Limit     50MB
    Refresh_Interval  10
 
# Обработка и обогащение логов
[FILTER]
    Name          parser
    Match         gitlab.*
    Key_Name      log
    Parser        json
    Reserve_Data  On
 
[FILTER]
    Name          nest
    Match         gitlab.*
    Operation     lift
    Nested_under  log
    Add_prefix    log_
 
# Добавление метаданных Docker Swarm
[FILTER]
    Name          modify
    Match         gitlab.*
    Add           environment production
    Add           service gitlab
    Add           cluster docker_swarm
 
# Выход в stdout для отладки
[OUTPUT]
    Name        stdout
    Match       *
    Format      json_lines
 
# Пример выхода в Elasticsearch (раскомментировать при необходимости)
# [OUTPUT]
#     Name        es
#     Match       gitlab.*
#     Host        192.168.1.10  # Ваш Elasticsearch
#     Port        9200
#     Index       gitlab-logs
#     Type        _doc
#     Retry_Limit False

parsers.conf

[PARSER]
    Name        json
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%LZ
    Time_Keep   On
 
[PARSER]
    Name        nginx
    Format      regex
    Regex       ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
    Time_Key    time
    Time_Format %d/%b/%Y:%H:%M:%S %z
    Time_Keep   On
 
[PARSER]
    Name        gitlab_exporter
    Format      regex
    Regex       ^(?<time>\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}\.\d+)\s+(?<message>.*)$
    Time_Key    time
    Time_Format %Y-%m-%d_%H:%M:%S.%L
    Time_Keep   On
 
[PARSER]
    Name        gitlab_multiline_firstline
    Format      regex
    Regex       ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+\S+\s+\[[^\]]+\]\s+#.*$|^[A-Z][a-z]+::[A-Z][a-zA-Z]+.*$
 
[MULTILINE_PARSER]
    Name          gitlab_multiline
    Type          regex
    Flush_Timeout 1000
    Rule          "start_state"  "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+\S+\s+\[[^\]]+\]\s+#.*$|^[A-Z][a-z]+::[A-Z][a-zA-Z]+.*$" "cont"
    Rule          "cont"         "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+\S+\s+\[[^\]]+\]\s+#.*$|^[A-Z][a-z]+::[A-Z][a-zA-Z]+.*$).*" "cont"

Обработка многострочных логов

Anonymous (anonymous@undisclosed.example.com) — 2025-11-25T23:19:45+00:00

Обработка многострочных логов

fluent-bit.conf

[SERVICE]
    flush        1
    log_level    info
    parsers_file parsers_multiline.conf
 
[INPUT]
    name             tail
    path             test.log
    read_from_head   true
    multiline.parser multiline-regex-test
 
[FILTER]
    name             parser
    match            *
    key_name         log
    parser           named-capture-test
 
[OUTPUT]
    name             stdout
    match            *

parsers_multiline.conf

[MULTILINE_PARSER]
    name          multiline-regex-test
    type          regex
    flush_timeout 1000
    #
    # Regex rules for multiline parsing
    # ---------------------------------
    #
    # configuration hints:
    #
    #  - first state always has the name: start_state
    #  - every field in the rule must be inside double quotes
    #
    # rules |   state name  | regex pattern                  | next state
    # ------|---------------|--------------------------------------------
    rule      "start_state"   "/([a-zA-Z]+ \d+ \d+\:\d+\:\d+)(.*)/"  "cont"
    rule      "cont"          "/^\s+at.*/"                     "cont"
 
[PARSER]
    Name named-capture-test
    Format regex
    Regex /^(?<date>[a-zA-Z]+ \d+ \d+\:\d+\:\d+) (?<message>.*)/m

test.log

single line...
Dec 14 06:41:08 Exception in thread "main" java.lang.RuntimeException: Something has gone wrong, aborting!
    at com.myproject.module.MyProject.badMethod(MyProject.java:22)
    at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:18)
    at com.myproject.module.MyProject.anotherMethod(MyProject.java:14)
    at com.myproject.module.MyProject.someMethod(MyProject.java:10)
    at com.myproject.module.MyProject.main(MyProject.java:6)
another line...

$ fluent-bit -c fluent-bit.conf

[0] tail.0: [1669160706.737650473, {"log"=>"single line...
"}]
[1] tail.0: [1669160706.737657687, {"date"=>"Dec 14 06:41:08", "message"=>"Exception in thread "main" java.lang.RuntimeException: Something has gone wrong, aborting!
    at com.myproject.module.MyProject.badMethod(MyProject.java:22)
    at com.myproject.module.MyProject.oneMoreMethod(MyProject.java:18)
    at com.myproject.module.MyProject.anotherMethod(MyProject.java:14)
    at com.myproject.module.MyProject.someMethod(MyProject.java:10)
    at com.myproject.module.MyProject.main(MyProject.java:6)
"}]
[2] tail.0: [1669160706.737657687, {"log"=>"another line...
"}]