phyer/k8sConfigs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fluentd 冷热程度过滤器

Browse Source
This commit is contained in:
zhangkun9038@dingtalk.com 2025-03-24 22:35:32 +08:00
parent 488d9f59f5
commit 626feac3bd
1 changed files with 64 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
efk_cluster/fluentd-configMap.yaml
View File

@ -15,27 +15,70 @@ data:
<label @main> <label @main>
<match sardine.log.**> <match sardine.log.**>
@type copy @type rewrite_tag_filter
<store> <rule>
@type elasticsearch key @timestamp
@id output_elasticsearch pattern /^(202[4-9]|2[1-9]\d{3})/
host elasticsearch tag hot.${tag}
port 9200 </rule>
scheme http <rule>
user fluentd_user key @timestamp
password fluentd_password pattern /^(202[0-3])/
logstash_format true tag warm.${tag}
logstash_prefix logstash </rule>
logstash_dateformat %Y.%m.%d <rule>
flush_interval 5s # 缩短批量写入间隔 key @timestamp
@log_level debug pattern /^(201\d|200\d|19\d{2})/
id_key _id tag cold.${tag}
remove_keys _id </rule>
</store> </match>
<store>
@type stdout <match hot.sardine.log.**>
@id output_stdout @type elasticsearch
</store> @id output_elasticsearch_hot
hosts elasticsearch-0.elasticsearch-service.efk.svc.cluster.local:9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
index_date_pattern now/d
include_tag_key true
tag_key @log_name
flush_interval 5s
@log_level debug
</match>
<match warm.sardine.log.**>
@type elasticsearch
@id output_elasticsearch_warm
hosts elasticsearch-1.elasticsearch-service.efk.svc.cluster.local:9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
index_date_pattern now/d
include_tag_key true
tag_key @log_name
flush_interval 10s
@log_level debug
</match>
<match cold.sardine.log.**>
@type elasticsearch
@id output_elasticsearch_cold
hosts elasticsearch-2.elasticsearch-service.efk.svc.cluster.local:9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
index_date_pattern now/d
include_tag_key true
tag_key @log_name
flush_interval 30s
@log_level debug
</match> </match>
</label> </label>