79 lines
2.4 KiB
YAML
79 lines
2.4 KiB
YAML
apiVersion: v1
|
||
kind: ConfigMap
|
||
metadata:
|
||
name: fluentd-config
|
||
namespace: efk
|
||
data:
|
||
fluent.conf: |
|
||
<source>
|
||
@type http
|
||
@id input_http
|
||
port 8888
|
||
tag sardine.log # 初始 tag，客户端需指定完整 tag 如 sardine.log.candle.BTC-USDT.15M
|
||
@label @main
|
||
<parse>
|
||
@type json
|
||
</parse>
|
||
</source>
|
||
|
||
<label @main>
|
||
<filter sardine.log.candle.**>
|
||
@type record_transformer
|
||
enable_ruby true
|
||
<record>
|
||
coin_pair ${tag_parts[3] || "UNKNOWN"} # 从 tag 第 4 部分提取，如 BTC-USDT
|
||
timeframe ${tag_parts[4] || "UNKNOWN"} # 从 tag 第 5 部分提取，如 15M
|
||
year ${time.strftime("%Y")} # 从 timestamp 提取年份
|
||
# 生成唯一 _id，例如 BTC-USDT_15M_2024-03-06-12:00:00
|
||
unique_id "${tag_parts[3]}_${tag_parts[4]}_#{record['timestamp'].gsub(/[: ]/, '-')}"
|
||
# 修改索引名称，加入 candle
|
||
index_name "logstash_candle_${tag_parts[3]}_${time.strftime('%Y')}_${tag_parts[4]}"
|
||
</record>
|
||
</filter>
|
||
|
||
<match sardine.log.candle.**>
|
||
@type copy
|
||
<store>
|
||
@type elasticsearch
|
||
@id output_elasticsearch_custom
|
||
host elasticsearch
|
||
port 9200
|
||
scheme http
|
||
user fluentd_user
|
||
password fluentd_password
|
||
logstash_format false
|
||
index_name ${record["index_name"]} # 动态索引，如 logstash_candle_BTC-USDT_2024_15M
|
||
id_key unique_id # 使用 unique_id 作为 _id 去重
|
||
flush_interval 5s
|
||
@log_level debug
|
||
remove_keys _id, coin_pair, timeframe, year, unique_id, index_name
|
||
</store>
|
||
# 保留原有按日期切分的输出（不加 candle）
|
||
<store>
|
||
@type elasticsearch
|
||
@id output_elasticsearch
|
||
host elasticsearch
|
||
port 9200
|
||
scheme http
|
||
user fluentd_user
|
||
password fluentd_password
|
||
logstash_format true
|
||
logstash_prefix logstash
|
||
logstash_dateformat %Y.%m.%d
|
||
id_key unique_id # 同样使用 unique_id 去重
|
||
flush_interval 5s
|
||
@log_level debug
|
||
remove_keys _id, unique_id
|
||
</store>
|
||
<store>
|
||
@type stdout
|
||
@id output_stdout
|
||
</store>
|
||
</match>
|
||
</label>
|
||
|
||
<match **>
|
||
@type stdout
|
||
@id output_stdout_all
|
||
</match>
|