phyer/tanya
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

移除没用的文件

Browse Source
This commit is contained in:
zhangkun9038@dingtalk.com 2025-03-10 22:46:45 +08:00
parent 16ea92317c
commit 20a5d67d34
3 changed files with 0 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
config/fluentd.yaml
View File

@ -1,78 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd-config
namespace: efk
data:
fluent.conf: |
<source>
@type http
@id input_http
port 8888
tag sardine.log # 初始 tag，客户端需指定完整 tag 如 sardine.log.candle.BTC-USDT.15M
@label @main
<parse>
@type json
</parse>
</source>
<label @main>
<filter sardine.log.candle.**>
@type record_transformer
enable_ruby true
<record>
coin_pair ${tag_parts[3] || "UNKNOWN"} # 从 tag 第 4 部分提取，如 BTC-USDT
timeframe ${tag_parts[4] || "UNKNOWN"} # 从 tag 第 5 部分提取，如 15M
year ${time.strftime("%Y")} # 从 timestamp 提取年份
# 生成唯一 _id，例如 BTC-USDT_15M_2024-03-06-12:00:00
unique_id "${tag_parts[3]}_${tag_parts[4]}_#{record['timestamp'].gsub(/[: ]/, '-')}"
# 修改索引名称，加入 candle
index_name "logstash_candle_${tag_parts[3]}_${time.strftime('%Y')}_${tag_parts[4]}"
</record>
</filter>
<match sardine.log.candle.**>
@type copy
<store>
@type elasticsearch
@id output_elasticsearch_custom
host elasticsearch
port 9200
scheme http
user fluentd_user
password fluentd_password
logstash_format false
index_name ${record["index_name"]} # 动态索引，如 logstash_candle_BTC-USDT_2024_15M
id_key unique_id # 使用 unique_id 作为 _id 去重
flush_interval 5s
@log_level debug
remove_keys _id, coin_pair, timeframe, year, unique_id, index_name
</store>
# 保留原有按日期切分的输出（不加 candle）
<store>
@type elasticsearch
@id output_elasticsearch
host elasticsearch
port 9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
logstash_dateformat %Y.%m.%d
id_key unique_id # 同样使用 unique_id 去重
flush_interval 5s
@log_level debug
remove_keys _id, unique_id
</store>
<store>
@type stdout
@id output_stdout
</store>
</match>
</label>
<match **>
@type stdout
@id output_stdout_all
</match>

8
fluentd-deployment.yaml
View File

@ -1,8 +0,0 @@
initContainers:
- name: install-rewrite-tag-filter
image: your-fluentd-image:version
command: ["fluent-gem", "install", "fluent-plugin-rewrite-tag-filter"]
containers:
- name: fluentd
image: your-fluentd-image:version

61
fluentd.conf
View File

@ -1,61 +0,0 @@
<source>
@type http
@id input_http
port 8888
@label @main
</source>
<label @main>
<match sardine.log.**>
@type copy
<store>
@type elasticsearch
@id output_elasticsearch
host elasticsearch
port 9200
scheme http
user fluentd_user
password fluentd_password
logstash_format true
logstash_prefix logstash
logstash_dateformat %Y.%m.%d
flush_interval 5s
@log_level debug
id_key _id
remove_keys _id
</store>
<store>
@type stdout
@id output_stdout
</store>
</match>
<match tanya.**>
@type copy
<store>
@type elasticsearch
@id output_elasticsearch_tanya
host elasticsearch
port 9200
scheme http
user fluentd_user
password fluentd_password
index_name candle_${tag_parts[1]}_${tag_parts[2]}_${tag_parts[3]}_${tag_parts[4]}
flush_interval 5s
@log_level debug
id_key _id
remove_keys _id
include_tag_key true
tag_key @log_name
</store>
<store>
@type stdout
@id output_stdout_tanya
</store>
</match>
</label>
<match **>
@type stdout
@id output_stdout_all
</match>