0,1,3当主节点，0,1,2当data_content节点

elasticsearch-statefulSet.yaml

@@ -36,7 +36,7 @@ spec:
         - name: discovery.seed_hosts
           value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local"
         - name: cluster.initial_master_nodes
-          value: "elasticsearch-0"
+          value: "[elasticsearch-0, elasticsearch-1, elasticsearch-3]"
         - name: xpack.security.enabled
           value: "false"
         - name: ES_JAVA_OPTS
@@ -64,6 +64,8 @@ spec:
                 operator: In
                 values: ["elasticsearch"]
             topologyKey: "kubernetes.io/hostname"
+      nodeSelector:
+        kubernetes.io/hostname: miracle
       volumes:
       - name: data
         persistentVolumeClaim:
@@ -112,13 +114,13 @@ spec:
         - name: discovery.seed_hosts
           value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local"
         - name: cluster.initial_master_nodes
-          value: "elasticsearch-0"
+          value: "[elasticsearch-0, elasticsearch-1, elasticsearch-3]"
         - name: xpack.security.enabled
           value: "false"
         - name: ES_JAVA_OPTS
           value: "-Xms4g -Xmx4g"
         - name: node.roles
-          value: "[data_warm, remote_cluster_client]"
+          value: "[data_warm, master, data_content,remote_cluster_client]"
         resources:
           requests:
             memory: "4Gi"
@@ -140,6 +142,8 @@ spec:
                 operator: In
                 values: ["elasticsearch"]
             topologyKey: "kubernetes.io/hostname"
+      nodeSelector:
+        kubernetes.io/hostname: lydia
       volumes:
       - name: data
         persistentVolumeClaim:
@@ -188,13 +192,13 @@ spec:
         - name: discovery.seed_hosts
           value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local"
         - name: cluster.initial_master_nodes
-          value: "elasticsearch-0"
+          value: "[elasticsearch-0, elasticsearch-1, elasticsearch-3]"
         - name: xpack.security.enabled
           value: "false"
         - name: ES_JAVA_OPTS
           value: "-Xms4g -Xmx4g"
         - name: node.roles
-          value: "[data_cold, remote_cluster_client]"
+          value: "[data_cold,  remote_cluster_client]"
         resources:
           requests:
             memory: "4Gi"
@@ -216,6 +220,8 @@ spec:
                 operator: In
                 values: ["elasticsearch"]
             topologyKey: "kubernetes.io/hostname"
+      nodeSelector:
+        kubernetes.io/hostname: emma
       volumes:
       - name: data
         persistentVolumeClaim:
@@ -264,13 +270,13 @@ spec:
         - name: discovery.seed_hosts
           value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local"
         - name: cluster.initial_master_nodes
-          value: "elasticsearch-0"
+          value: "[elasticsearch-0, elasticsearch-1, elasticsearch-3]"
         - name: xpack.security.enabled
           value: "false"
         - name: ES_JAVA_OPTS
           value: "-Xms4g -Xmx4g"
         - name: node.roles
-          value: "[data_warm, master, remote_cluster_client]"
+          value: "[data_warm, master, data_content, remote_cluster_client]"
         resources:
           requests:
             memory: "4Gi"
@@ -292,6 +298,8 @@ spec:
                 operator: In
                 values: ["elasticsearch"]
             topologyKey: "kubernetes.io/hostname"
+      nodeSelector:
+        kubernetes.io/hostname: niyon
       volumes:
       - name: data
         persistentVolumeClaim:

policies/idxTemps/candles/1m.json5

@@ -0,0 +1,32 @@
+//PUT _ilm/policy/logstash_1m_candle_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "10GB",
+            "max_age": "30d"    // 7 天滚动，频繁数据快速积累
+          },
+          "allocate": { "_tier_preference": "data_hot" }
+        }
+      },
+      "warm": {
+        "min_age": "100d",      // 30 天进入 warm
+        "actions": {
+          "allocate": { "_tier_preference": "data_warm" }
+        }
+      },
+      "cold": {
+        "min_age": "300d",      // 90 天进入 cold
+        "actions": {
+          "allocate": { "_tier_preference": "data_cold" },
+          "shrink": { "number_of_shards": 1 }
+        }
+      },
+      "delete": {
+        "min_age": "1000d"      // 1 年后删除
+      }
+    }
+  }
+}

policies/idxTemps/candles/5m.json5

@@ -0,0 +1,26 @@
+// PUT _index_template/logstash-candle-template
+{
+  "index_patterns": ["logstash-candle-5m.*"],
+  "template": {
+    "settings": {
+      "number_of_shards": 2,
+      "number_of_replicas": 1,
+      "index.lifecycle.name": "logstash-candle-5m_candle_policy"
+    },
+    "mappings": {
+      "properties": {
+        "dataTime": {
+          "type": "date",
+          "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ssZ||yyyy-MM-dd'T'HH:mm:ssXXX"
+        },
+        "close": { "type": "float" },
+        "high": { "type": "float" },
+        "low": { "type": "float" },
+        "open": { "type": "float" },
+        "volume": { "type": "float" },
+        "volumeCcy": { "type": "float" },
+        "timeStamp": { "type": "double" }
+      }
+    }
+  }
+}

policies/ilm/15m.json5

@@ -0,0 +1,32 @@
+// PUT _ilm/policy/logstash_15m_candle_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "10GB",
+            "max_age": "360d"   // 14 天滚动
+          },
+          "allocate": { "_tier_preference": "data_hot" }
+        }
+      },
+      "warm": {
+        "min_age": "500d",      // 60 天进入 warm
+        "actions": {
+          "allocate": { "_tier_preference": "data_warm" }
+        }
+      },
+      "cold": {
+        "min_age": "800d",     // 180 天进入 cold
+        "actions": {
+          "allocate": { "_tier_preference": "data_cold" },
+          "shrink": { "number_of_shards": 1 }
+        }
+      },
+      "delete": {
+        "min_age": "2000d"     // 3 年后删除
+      }
+    }
+  }
+}

policies/ilm/1h.json5

@@ -0,0 +1,32 @@
+// PUT _ilm/policy/logstash_1h_candle_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "10GB",
+            "max_age": "300d"   // 14 天滚动
+          },
+          "allocate": { "_tier_preference": "data_hot" }
+        }
+      },
+      "warm": {
+        "min_age": "600d",      // 60 天进入 warm
+        "actions": {
+          "allocate": { "_tier_preference": "data_warm" }
+        }
+      },
+      "cold": {
+        "min_age": "800d",     // 180 天进入 cold
+        "actions": {
+          "allocate": { "_tier_preference": "data_cold" },
+          "shrink": { "number_of_shards": 1 }
+        }
+      },
+      "delete": {
+        "min_age": "2400d"     // 3 年后删除
+      }
+    }
+  }
+}

policies/ilm/1m.json5

@@ -0,0 +1,32 @@
+// PUT _ilm/policy/logstash_1m_candle_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "20GB",
+            "max_age": "7d"    // 7 天滚动，频繁数据快速积累
+          },
+          "allocate": { "_tier_preference": "data_hot" }
+        }
+      },
+      "warm": {
+        "min_age": "30d",      // 30 天进入 warm
+        "actions": {
+          "allocate": { "_tier_preference": "data_warm" }
+        }
+      },
+      "cold": {
+        "min_age": "90d",      // 90 天进入 cold
+        "actions": {
+          "allocate": { "_tier_preference": "data_cold" },
+          "shrink": { "number_of_shards": 1 }
+        }
+      },
+      "delete": {
+        "min_age": "365d"      // 1 年后删除
+      }
+    }
+  }
+}

policies/ilm/2h.json5

@@ -0,0 +1,32 @@
+// PUT _ilm/policy/logstash_1h_candle_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "10GB",
+            "max_age": "700d"   // 14 天滚动
+          },
+          "allocate": { "_tier_preference": "data_hot" }
+        }
+      },
+      "warm": {
+        "min_age": "1000d",      // 60 天进入 warm
+        "actions": {
+          "allocate": { "_tier_preference": "data_warm" }
+        }
+      },
+      "cold": {
+        "min_age": "1000d",     // 180 天进入 cold
+        "actions": {
+          "allocate": { "_tier_preference": "data_cold" },
+          "shrink": { "number_of_shards": 1 }
+        }
+      },
+      "delete": {
+        "min_age": "2600d"     // 3 年后删除
+      }
+    }
+  }
+}

policies/ilm/30m.json5

@@ -0,0 +1,32 @@
+// PUT _ilm/policy/logstash_30m_candle_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "10GB",
+            "max_age": "400d"   // 14 天滚动
+          },
+          "allocate": { "_tier_preference": "data_hot" }
+        }
+      },
+      "warm": {
+        "min_age": "650d",      // 60 天进入 warm
+        "actions": {
+          "allocate": { "_tier_preference": "data_warm" }
+        }
+      },
+      "cold": {
+        "min_age": "1000d",     // 180 天进入 cold
+        "actions": {
+          "allocate": { "_tier_preference": "data_cold" },
+          "shrink": { "number_of_shards": 1 }
+        }
+      },
+      "delete": {
+        "min_age": "2000d"     // 3 年后删除
+      }
+    }
+  }
+}

policies/ilm/5m.json5

@@ -0,0 +1,32 @@
+// PUT _ilm/policy/logstash_5m_candle_policy
+{
+  "policy": {
+    "phases": {
+      "hot": {
+        "actions": {
+          "rollover": {
+            "max_size": "10GB",
+            "max_age": "120d"   // 14 天滚动
+          },
+          "allocate": { "_tier_preference": "data_hot" }
+        }
+      },
+      "warm": {
+        "min_age": "200d",      // 60 天进入 warm
+        "actions": {
+          "allocate": { "_tier_preference": "data_warm" }
+        }
+      },
+      "cold": {
+        "min_age": "365d",     // 180 天进入 cold
+        "actions": {
+          "allocate": { "_tier_preference": "data_cold" },
+          "shrink": { "number_of_shards": 1 }
+        }
+      },
+      "delete": {
+        "min_age": "1000d"     // 3 年后删除
+      }
+    }
+  }
+}