From 952e8a54707f2bcb8e5df8aa7844198bbfb03ec2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 29 Mar 2025 01:09:18 +0000 Subject: [PATCH] =?UTF-8?q?0,1,3=E5=BD=93=E4=B8=BB=E8=8A=82=E7=82=B9?= =?UTF-8?q?=EF=BC=8C0,1,2=E5=BD=93data=5Fcontent=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- @ | Bin 0 -> 23928 bytes elasticsearch-statefulSet.yaml | 22 +++++++++++++------- indexTemplates | 0 policies/idxTemps/candles/1m.json5 | 32 +++++++++++++++++++++++++++++ policies/idxTemps/candles/5m.json5 | 26 +++++++++++++++++++++++ policies/ilm/15m.json5 | 32 +++++++++++++++++++++++++++++ policies/ilm/1h.json5 | 32 +++++++++++++++++++++++++++++ policies/ilm/1m.json5 | 32 +++++++++++++++++++++++++++++ policies/ilm/2h.json5 | 32 +++++++++++++++++++++++++++++ policies/ilm/30m.json5 | 32 +++++++++++++++++++++++++++++ policies/ilm/5m.json5 | 32 +++++++++++++++++++++++++++++ 11 files changed, 265 insertions(+), 7 deletions(-) create mode 100644 @ create mode 100644 indexTemplates create mode 100644 policies/idxTemps/candles/1m.json5 create mode 100644 policies/idxTemps/candles/5m.json5 create mode 100644 policies/ilm/15m.json5 create mode 100644 policies/ilm/1h.json5 create mode 100644 policies/ilm/1m.json5 create mode 100644 policies/ilm/2h.json5 create mode 100644 policies/ilm/30m.json5 create mode 100644 policies/ilm/5m.json5 diff --git a/@ b/@ new file mode 100644 index 0000000000000000000000000000000000000000..6d7aa2984dd76c00778a8ec0af4c307b693aab1c GIT binary patch literal 23928 zcmeHPZA=`;8Rm?!`8e}cJ{;L@)({dE`#v}@hI8y1L!1!Tt`jG=l(e8_Zh?ipy<>Op zDE*N(CRHo>hd-#)G=J))QsVrJs-%^g#)_PEqNWFF70D`1daBK(Zmax?`Y(U_zOxJX z>^lrMOE#*OALw>x=GkYS_nmiUXLiOjKaxq`KK7H>M+8|=crB(Fo8_b!eJv>{su+_E z?}^j>$GK$C-xl)J{Y55J@j z8==5xEGh&NLkU?+1jf76-k3Zrj;H}a;#Ex~s=zB`)TatcQY3MB>P?U+3P&GlZhz;swlPxfZu*Q@qiz7UAyc7{JKlPz6;|G)3XY>Xp~05=LKai2hKgKB;Lcss<}DO z&{;_Sw5ZxB_W689eKU9RbT2m?Q@Gxk6oG%(VvI^YE}k}9WSC2~89T;BITGeh;w?ax zwbL;v5fyqRUW^)LQ9k&~ws@hy`zLd%7rqN-|^Xlmk`xqrTV^|L>mzy0cq zw|{Ym5f?suiw7!Iwn}bNhlQar1p8 zjwcJ#7S#<=btmUMCsEyB8C5H?p5c{fs9;^!>$~k6>AQC;+ZO5qWgl8i*-5NDctBy& zLia$0ZJ+bBn8L2a2O1uzxcTBkL+Sx!cq z@tUH;7I@?^1cC=?dFb^81)~5AErR~;c7Gc*P8uoBC-%%(eX3i$Y{L7G&3G=qwM^dTo#RYxlztT*j7{`Qom8VE zR~$La(yF965005K6}^mn{-u)!6R4M|8=CWM)yZyqkZR1%AXCxL7*%gkK2I`AbW1@;+<1Gy{zAb%wg>Dcf>pn5i)xysTI~V0F#537r`JBphVzIuh^*~l z!#S!GIyk6hd?W1FNdV=peT#K$0G&iVMY_74P9l44zQ^jUYEy+{bm^oDdDP9X2P3<~ zdrT*r5u@QMt15eSvWp^pXp1|`Iw?iC#$H?C@H+v7+i}zuIFNojg&SCp^w}=(xK+a) zpRETY+k8)0ZnWcz)n_Vl&o=53XT6E;ELojH?(!=ib)XgtDYk|_rIQvraE2_L^l#Ys zG@3JPne-=YIQZ=h)+>E`b&a#frP;oMpYm)%jgwVpXVth-B27g^epVF-{{I=rj+ zqES?C!!r=W+qe1$KCAIOqLcl)JYWM%r!M;|U=HnPuZ8i|GVeV4YHWH~0XkR#m>E3A z%HU~M0OuJvZw2QWIBx~#88~kR=NUL}1?L$!Zw2QWIBx~#88~kR=NUL}1?L$!Zw2QW zIBx~#88~kR=NUL}1?L$!Zw2QWIBx~#88~kRn;2Xvdy7Xo)A`LKpTGk2QGT=JOKOjwi>xi)O!vma5Zr?$#E8j z>nHb~58`U#YO8Au8RInTH=`6{CPD9DKNKB8QxXA=g_{dfp*W*7wqoDjts zW8(vxT6R&KX-3b1(=ZSyQ!X>gyhu^vj3~D#N`kRcxL=riUW-~hkdD9aY&NBxgN+^AetyWFtMLTi*t1-{`Z8Pjen~H3?RAg-8 zC-CCN^YkK2FBX1V0x2F95*AOCZiU~)!UT_3V;5+OfWIft(BEh9_qk^J8%r;q6Diar zEpYkYwD=^y8JAI87Gttu@&}Uht(l==mhs0p#pA!E zcg(|!=^c5b2J(29P8-DYh0Xgvu7vw#G8_-=U%7ed0k~Ng5mgd{<36YSsvtxLM`Lgx zuvzG>-H^U>2~3nlP2{CP^KpZSW#u!fAYVBJS7MTN_lJ{v;O@iO1MoZt?*{4fW?m?V x5+#^t5N~8^57fbps%!|FH&gpc10>aCCxv`wV#9}Pe{?&2(n69xn(l!N{0}^Ii3tDz literal 0 HcmV?d00001 diff --git a/elasticsearch-statefulSet.yaml b/elasticsearch-statefulSet.yaml index cd2f999..6f2afcf 100644 --- a/elasticsearch-statefulSet.yaml +++ b/elasticsearch-statefulSet.yaml @@ -36,7 +36,7 @@ spec: - name: discovery.seed_hosts value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local" - name: cluster.initial_master_nodes - value: "elasticsearch-0" + value: "[elasticsearch-0, elasticsearch-1, elasticsearch-3]" - name: xpack.security.enabled value: "false" - name: ES_JAVA_OPTS @@ -64,6 +64,8 @@ spec: operator: In values: ["elasticsearch"] topologyKey: "kubernetes.io/hostname" + nodeSelector: + kubernetes.io/hostname: miracle volumes: - name: data persistentVolumeClaim: @@ -112,13 +114,13 @@ spec: - name: discovery.seed_hosts value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local" - name: cluster.initial_master_nodes - value: "elasticsearch-0" + value: "[elasticsearch-0, elasticsearch-1, elasticsearch-3]" - name: xpack.security.enabled value: "false" - name: ES_JAVA_OPTS value: "-Xms4g -Xmx4g" - name: node.roles - value: "[data_warm, remote_cluster_client]" + value: "[data_warm, master, data_content,remote_cluster_client]" resources: requests: memory: "4Gi" @@ -140,6 +142,8 @@ spec: operator: In values: ["elasticsearch"] topologyKey: "kubernetes.io/hostname" + nodeSelector: + kubernetes.io/hostname: lydia volumes: - name: data persistentVolumeClaim: @@ -188,13 +192,13 @@ spec: - name: discovery.seed_hosts value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local" - name: cluster.initial_master_nodes - value: "elasticsearch-0" + value: "[elasticsearch-0, elasticsearch-1, elasticsearch-3]" - name: xpack.security.enabled value: "false" - name: ES_JAVA_OPTS value: "-Xms4g -Xmx4g" - name: node.roles - value: "[data_cold, remote_cluster_client]" + value: "[data_cold, remote_cluster_client]" resources: requests: memory: "4Gi" @@ -216,6 +220,8 @@ spec: operator: In values: ["elasticsearch"] topologyKey: "kubernetes.io/hostname" + nodeSelector: + kubernetes.io/hostname: emma volumes: - name: data persistentVolumeClaim: @@ -264,13 +270,13 @@ spec: - name: discovery.seed_hosts value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local" - name: cluster.initial_master_nodes - value: "elasticsearch-0" + value: "[elasticsearch-0, elasticsearch-1, elasticsearch-3]" - name: xpack.security.enabled value: "false" - name: ES_JAVA_OPTS value: "-Xms4g -Xmx4g" - name: node.roles - value: "[data_warm, master, remote_cluster_client]" + value: "[data_warm, master, data_content, remote_cluster_client]" resources: requests: memory: "4Gi" @@ -292,6 +298,8 @@ spec: operator: In values: ["elasticsearch"] topologyKey: "kubernetes.io/hostname" + nodeSelector: + kubernetes.io/hostname: niyon volumes: - name: data persistentVolumeClaim: diff --git a/indexTemplates b/indexTemplates new file mode 100644 index 0000000..e69de29 diff --git a/policies/idxTemps/candles/1m.json5 b/policies/idxTemps/candles/1m.json5 new file mode 100644 index 0000000..0d0b1b4 --- /dev/null +++ b/policies/idxTemps/candles/1m.json5 @@ -0,0 +1,32 @@ +//PUT _ilm/policy/logstash_1m_candle_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "10GB", + "max_age": "30d" // 7 天滚动，频繁数据快速积累 + }, + "allocate": { "_tier_preference": "data_hot" } + } + }, + "warm": { + "min_age": "100d", // 30 天进入 warm + "actions": { + "allocate": { "_tier_preference": "data_warm" } + } + }, + "cold": { + "min_age": "300d", // 90 天进入 cold + "actions": { + "allocate": { "_tier_preference": "data_cold" }, + "shrink": { "number_of_shards": 1 } + } + }, + "delete": { + "min_age": "1000d" // 1 年后删除 + } + } + } +} diff --git a/policies/idxTemps/candles/5m.json5 b/policies/idxTemps/candles/5m.json5 new file mode 100644 index 0000000..76bb25a --- /dev/null +++ b/policies/idxTemps/candles/5m.json5 @@ -0,0 +1,26 @@ +// PUT _index_template/logstash-candle-template +{ + "index_patterns": ["logstash-candle-5m.*"], + "template": { + "settings": { + "number_of_shards": 2, + "number_of_replicas": 1, + "index.lifecycle.name": "logstash-candle-5m_candle_policy" + }, + "mappings": { + "properties": { + "dataTime": { + "type": "date", + "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssXXX" + }, + "close": { "type": "float" }, + "high": { "type": "float" }, + "low": { "type": "float" }, + "open": { "type": "float" }, + "volume": { "type": "float" }, + "volumeCcy": { "type": "float" }, + "timeStamp": { "type": "double" } + } + } + } +} diff --git a/policies/ilm/15m.json5 b/policies/ilm/15m.json5 new file mode 100644 index 0000000..1c13227 --- /dev/null +++ b/policies/ilm/15m.json5 @@ -0,0 +1,32 @@ +// PUT _ilm/policy/logstash_15m_candle_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "10GB", + "max_age": "360d" // 14 天滚动 + }, + "allocate": { "_tier_preference": "data_hot" } + } + }, + "warm": { + "min_age": "500d", // 60 天进入 warm + "actions": { + "allocate": { "_tier_preference": "data_warm" } + } + }, + "cold": { + "min_age": "800d", // 180 天进入 cold + "actions": { + "allocate": { "_tier_preference": "data_cold" }, + "shrink": { "number_of_shards": 1 } + } + }, + "delete": { + "min_age": "2000d" // 3 年后删除 + } + } + } +} diff --git a/policies/ilm/1h.json5 b/policies/ilm/1h.json5 new file mode 100644 index 0000000..397552a --- /dev/null +++ b/policies/ilm/1h.json5 @@ -0,0 +1,32 @@ +// PUT _ilm/policy/logstash_1h_candle_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "10GB", + "max_age": "300d" // 14 天滚动 + }, + "allocate": { "_tier_preference": "data_hot" } + } + }, + "warm": { + "min_age": "600d", // 60 天进入 warm + "actions": { + "allocate": { "_tier_preference": "data_warm" } + } + }, + "cold": { + "min_age": "800d", // 180 天进入 cold + "actions": { + "allocate": { "_tier_preference": "data_cold" }, + "shrink": { "number_of_shards": 1 } + } + }, + "delete": { + "min_age": "2400d" // 3 年后删除 + } + } + } +} diff --git a/policies/ilm/1m.json5 b/policies/ilm/1m.json5 new file mode 100644 index 0000000..d3aac29 --- /dev/null +++ b/policies/ilm/1m.json5 @@ -0,0 +1,32 @@ +// PUT _ilm/policy/logstash_1m_candle_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "20GB", + "max_age": "7d" // 7 天滚动，频繁数据快速积累 + }, + "allocate": { "_tier_preference": "data_hot" } + } + }, + "warm": { + "min_age": "30d", // 30 天进入 warm + "actions": { + "allocate": { "_tier_preference": "data_warm" } + } + }, + "cold": { + "min_age": "90d", // 90 天进入 cold + "actions": { + "allocate": { "_tier_preference": "data_cold" }, + "shrink": { "number_of_shards": 1 } + } + }, + "delete": { + "min_age": "365d" // 1 年后删除 + } + } + } +} diff --git a/policies/ilm/2h.json5 b/policies/ilm/2h.json5 new file mode 100644 index 0000000..b516010 --- /dev/null +++ b/policies/ilm/2h.json5 @@ -0,0 +1,32 @@ +// PUT _ilm/policy/logstash_1h_candle_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "10GB", + "max_age": "700d" // 14 天滚动 + }, + "allocate": { "_tier_preference": "data_hot" } + } + }, + "warm": { + "min_age": "1000d", // 60 天进入 warm + "actions": { + "allocate": { "_tier_preference": "data_warm" } + } + }, + "cold": { + "min_age": "1000d", // 180 天进入 cold + "actions": { + "allocate": { "_tier_preference": "data_cold" }, + "shrink": { "number_of_shards": 1 } + } + }, + "delete": { + "min_age": "2600d" // 3 年后删除 + } + } + } +} diff --git a/policies/ilm/30m.json5 b/policies/ilm/30m.json5 new file mode 100644 index 0000000..b4afc0b --- /dev/null +++ b/policies/ilm/30m.json5 @@ -0,0 +1,32 @@ +// PUT _ilm/policy/logstash_30m_candle_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "10GB", + "max_age": "400d" // 14 天滚动 + }, + "allocate": { "_tier_preference": "data_hot" } + } + }, + "warm": { + "min_age": "650d", // 60 天进入 warm + "actions": { + "allocate": { "_tier_preference": "data_warm" } + } + }, + "cold": { + "min_age": "1000d", // 180 天进入 cold + "actions": { + "allocate": { "_tier_preference": "data_cold" }, + "shrink": { "number_of_shards": 1 } + } + }, + "delete": { + "min_age": "2000d" // 3 年后删除 + } + } + } +} diff --git a/policies/ilm/5m.json5 b/policies/ilm/5m.json5 new file mode 100644 index 0000000..440effc --- /dev/null +++ b/policies/ilm/5m.json5 @@ -0,0 +1,32 @@ +// PUT _ilm/policy/logstash_5m_candle_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "10GB", + "max_age": "120d" // 14 天滚动 + }, + "allocate": { "_tier_preference": "data_hot" } + } + }, + "warm": { + "min_age": "200d", // 60 天进入 warm + "actions": { + "allocate": { "_tier_preference": "data_warm" } + } + }, + "cold": { + "min_age": "365d", // 180 天进入 cold + "actions": { + "allocate": { "_tier_preference": "data_cold" }, + "shrink": { "number_of_shards": 1 } + } + }, + "delete": { + "min_age": "1000d" // 3 年后删除 + } + } + } +}