Fluent-Bit Gitlab

fluent-bit.conf

[SERVICE] Flush 1 Log_Level info Daemon off Parsers_File parsers.conf HTTP_Server On HTTP_Listen 0.0.0.0 HTTP_Port 2020 # Вход для JSON логов GitLab [INPUT] Name tail Path /var/log/gitlab/gitaly/current Tag gitlab.gitaly Parser json DB /tail-db/gitaly.db Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 [INPUT] Name tail Path /var/log/gitlab/sidekiq/current Tag gitlab.sidekiq Parser json DB /tail-db/sidekiq.db Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 [INPUT] Name tail Path /var/log/gitlab/gitlab-rails/production_json.log Tag gitlab.rails.json Parser json DB /tail-db/rails_json.db Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 [INPUT] Name tail Path /var/log/gitlab/gitlab-rails/sidekiq_client.log Tag gitlab.sidekiq_client Parser json DB /tail-db/sidekiq_client.db Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 [INPUT] Name tail Path /var/log/gitlab/gitlab-workhorse/current Tag gitlab.workhorse Parser json DB /tail-db/workhorse.db Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 # Вход для многострочных текстовых логов (исключения) [INPUT] Name tail Path /var/log/gitlab/gitlab-rails/production.log Tag gitlab.rails.text Parser gitlab_multiline DB /tail-db/rails_text.db Mem_Buf_Limit 50MB Multiline On Parser_Firstline gitlab_multiline_firstline Refresh_Interval 10 # Вход для логов nginx [INPUT] Name tail Path /var/log/gitlab/nginx/gitlab_access.log Tag gitlab.nginx Parser nginx DB /tail-db/nginx.db Mem_Buf_Limit 50MB Refresh_Interval 10 # Вход для логов gitlab-exporter [INPUT] Name tail Path /var/log/gitlab/gitlab-exporter/current Tag gitlab.exporter Parser gitlab_exporter DB /tail-db/exporter.db Mem_Buf_Limit 50MB Refresh_Interval 10 # Обработка и обогащение логов [FILTER] Name parser Match gitlab.* Key_Name log Parser json Reserve_Data On [FILTER] Name nest Match gitlab.* Operation lift Nested_under log Add_prefix log_ # Добавление метаданных Docker Swarm [FILTER] Name modify Match gitlab.* Add environment production Add service gitlab Add cluster docker_swarm # Выход в stdout для отладки [OUTPUT] Name stdout Match * Format json_lines # Пример выхода в Elasticsearch (раскомментировать при необходимости) # [OUTPUT] # Name es # Match gitlab.* # Host 192.168.1.10 # Ваш Elasticsearch # Port 9200 # Index gitlab-logs # Type _doc # Retry_Limit False

parsers.conf

[PARSER] Name json Format json Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%LZ Time_Keep On [PARSER] Name nginx Format regex Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ Time_Key time Time_Format %d/%b/%Y:%H:%M:%S %z Time_Keep On [PARSER] Name gitlab_exporter Format regex Regex ^(?<time>\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}\.\d+)\s+(?<message>.*)$ Time_Key time Time_Format %Y-%m-%d_%H:%M:%S.%L Time_Keep On [PARSER] Name gitlab_multiline_firstline Format regex Regex ^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+\S+\s+\[[^\]]+\]\s+#.*$|^[A-Z][a-z]+::[A-Z][a-zA-Z]+.*$ [MULTILINE_PARSER] Name gitlab_multiline Type regex Flush_Timeout 1000 Rule "start_state" "^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+\S+\s+\[[^\]]+\]\s+#.*$|^[A-Z][a-z]+::[A-Z][a-zA-Z]+.*$" "cont" Rule "cont" "^(?!\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\s+\S+\s+\[[^\]]+\]\s+#.*$|^[A-Z][a-z]+::[A-Z][a-zA-Z]+.*$).*" "cont"