phyer/k8sConfigs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fluentd 冷热程度过滤器

Browse Source
This commit is contained in:
zhangkun9038@dingtalk.com 2025-03-24 22:35:32 +08:00
parent 488d9f59f5
commit 626feac3bd
1 changed files with 64 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
efk_cluster/fluentd-configMap.yaml
View File

@ -15,27 +15,70 @@ data:
<label @main>
<match sardine.log.**>
@type copy
<store>
@type elasticsearch
@id output_elasticsearch
host elasticsearch
port 9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
logstash_dateformat %Y.%m.%d
flush_interval 5s # 缩短批量写入间隔
@log_level debug
id_key _id
remove_keys _id
</store>
<store>
@type stdout
@id output_stdout
</store>
@type rewrite_tag_filter
<rule>
key @timestamp
pattern /^(202[4-9]|2[1-9]\d{3})/
tag hot.${tag}
</rule>
<rule>
key @timestamp
pattern /^(202[0-3])/
tag warm.${tag}
</rule>
<rule>
key @timestamp
pattern /^(201\d|200\d|19\d{2})/
tag cold.${tag}
</rule>
</match>
<match hot.sardine.log.**>
@type elasticsearch
@id output_elasticsearch_hot
hosts elasticsearch-0.elasticsearch-service.efk.svc.cluster.local:9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
index_date_pattern now/d
include_tag_key true
tag_key @log_name
flush_interval 5s
@log_level debug
</match>
<match warm.sardine.log.**>
@type elasticsearch
@id output_elasticsearch_warm
hosts elasticsearch-1.elasticsearch-service.efk.svc.cluster.local:9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
index_date_pattern now/d
include_tag_key true
tag_key @log_name
flush_interval 10s
@log_level debug
</match>
<match cold.sardine.log.**>
@type elasticsearch
@id output_elasticsearch_cold
hosts elasticsearch-2.elasticsearch-service.efk.svc.cluster.local:9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
index_date_pattern now/d
include_tag_key true
tag_key @log_name
flush_interval 30s
@log_level debug
</match>
</label>