diff --git a/elasticsearch-ingress.yaml b/elasticsearch-ingress.yaml index 8744bd1..cdad7cd 100644 --- a/elasticsearch-ingress.yaml +++ b/elasticsearch-ingress.yaml @@ -61,6 +61,26 @@ spec: --- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: elasticsearch-3 + namespace: efk +spec: + ingressClassName: traefik + rules: + - host: elastic3.k8s.xunlang.home + http: + paths: + - backend: + service: + name: elasticsearch-3 + port: + number: 9200 + path: / + pathType: Prefix +--- + apiVersion: traefik.containo.us/v1alpha1 kind: IngressRoute diff --git a/elasticsearch-pvandpvc_incluster.yaml b/elasticsearch-pvandpvc_incluster.yaml index c27a453..f1503e7 100644 --- a/elasticsearch-pvandpvc_incluster.yaml +++ b/elasticsearch-pvandpvc_incluster.yaml @@ -28,6 +28,20 @@ spec: path: /root/elasticsearch/warmData --- +# Warm2 节点的 PV +apiVersion: v1 +kind: PersistentVolume +metadata: + name: elasticsearch-warm2-pv +spec: + capacity: + storage: 128Gi + accessModes: + - ReadWriteOnce + nfs: + server: 10.100.93.1 # fenny + path: /root/elasticsearch/warmData2 +--- # Cold 节点的 PV apiVersion: v1 kind: PersistentVolume @@ -73,21 +87,35 @@ spec: storage: 128Gi volumeName: elasticsearch-warm-pv storageClassName: "" # 显式禁用 StorageClass - --- -# PVC for Cold +# PVC for Warm2 apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: elasticsearch-data-pvc-elasticsearch-2 + name: elasticsearch-data-pvc-elasticsearch-3 namespace: efk spec: accessModes: - ReadWriteOnce resources: requests: - storage: 500Gi - volumeName: elasticsearch-cold-pv + storage: 128Gi + volumeName: elasticsearch-warm2-pv + storageClassName: "" # 显式禁用 StorageClass +--- +# PVC for Warm2 Config +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: elasticsearch-config-pvc-elasticsearch-3 + namespace: efk +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + volumeName: elasticsearch-warm2-config-pv storageClassName: "" # 显式禁用 StorageClass --- # PVC for Cold @@ -104,7 +132,6 @@ spec: storage: 500Gi volumeName: elasticsearch-cold-pv storageClassName: "" # 显式禁用 StorageClass - --- # PV for Hot Config apiVersion: v1 @@ -150,7 +177,20 @@ spec: nfs: server: 10.100.93.1 # fenny path: /root/elasticsearch/warmData/config - +--- +# PV for Warm2 Config +apiVersion: v1 +kind: PersistentVolume +metadata: + name: elasticsearch-warm2-config-pv +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + nfs: + server: 10.100.93.1 # fenny + path: /root/elasticsearch/warmData2/config --- # PVC for Warm Config apiVersion: v1 @@ -182,19 +222,4 @@ spec: server: 10.100.93.1 # fenny path: /root/elasticsearch/coldData/config ---- -# PVC for Cold Config -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: elasticsearch-config-pvc-elasticsearch-2 - namespace: efk -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - volumeName: elasticsearch-cold-config-pv - storageClassName: "" # 显式禁用 StorageClass diff --git a/elasticsearch-service.yaml b/elasticsearch-service.yaml index aa356bc..94983e8 100644 --- a/elasticsearch-service.yaml +++ b/elasticsearch-service.yaml @@ -51,6 +51,23 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: elasticsearch-3 + namespace: efk +spec: + ports: + - port: 9200 + targetPort: 9200 + name: http + - port: 9300 + targetPort: 9300 + name: transport + selector: + statefulset.kubernetes.io/pod-name: elasticsearch-3-0 + type: LoadBalancer +--- +apiVersion: v1 +kind: Service metadata: name: elasticsearch-lb namespace: efk diff --git a/elasticsearch-statefulSet.yaml b/elasticsearch-statefulSet.yaml index 9fc1071..cd2f999 100644 --- a/elasticsearch-statefulSet.yaml +++ b/elasticsearch-statefulSet.yaml @@ -34,7 +34,7 @@ spec: - name: node.name value: "elasticsearch-0" - name: discovery.seed_hosts - value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local" + value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local" - name: cluster.initial_master_nodes value: "elasticsearch-0" - name: xpack.security.enabled @@ -42,7 +42,7 @@ spec: - name: ES_JAVA_OPTS value: "-Xms6g -Xmx6g" - name: node.roles - value: "[master, data_hot, data_content]" + value: "[master, data_hot, data_content, remote_cluster_client]" resources: requests: memory: "6Gi" @@ -110,7 +110,7 @@ spec: - name: node.name value: "elasticsearch-1" - name: discovery.seed_hosts - value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local" + value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local" - name: cluster.initial_master_nodes value: "elasticsearch-0" - name: xpack.security.enabled @@ -118,7 +118,7 @@ spec: - name: ES_JAVA_OPTS value: "-Xms4g -Xmx4g" - name: node.roles - value: "[data_warm]" + value: "[data_warm, remote_cluster_client]" resources: requests: memory: "4Gi" @@ -186,7 +186,7 @@ spec: - name: node.name value: "elasticsearch-2" - name: discovery.seed_hosts - value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local" + value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local" - name: cluster.initial_master_nodes value: "elasticsearch-0" - name: xpack.security.enabled @@ -194,7 +194,7 @@ spec: - name: ES_JAVA_OPTS value: "-Xms4g -Xmx4g" - name: node.roles - value: "[data_cold]" + value: "[data_cold, remote_cluster_client]" resources: requests: memory: "4Gi" @@ -223,3 +223,79 @@ spec: - name: config-volume persistentVolumeClaim: claimName: "elasticsearch-config-pvc-elasticsearch-2" + +--- + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch-3 + namespace: efk +spec: + serviceName: elasticsearch-service + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + initContainers: + - name: init-sysctl + image: busybox + command: ["sysctl", "-w", "vm.max_map_count=262144"] + securityContext: + privileged: true + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0 + ports: + - containerPort: 9200 + name: http + - containerPort: 9300 + name: transport + env: + - name: cluster.name + value: "my-es-cluster" + - name: node.name + value: "elasticsearch-3" + - name: discovery.seed_hosts + value: "elasticsearch-0.efk.svc.cluster.local,elasticsearch-1.efk.svc.cluster.local,elasticsearch-2.efk.svc.cluster.local,elasticsearch-3.efk.svc.cluster.local" + - name: cluster.initial_master_nodes + value: "elasticsearch-0" + - name: xpack.security.enabled + value: "false" + - name: ES_JAVA_OPTS + value: "-Xms4g -Xmx4g" + - name: node.roles + value: "[data_warm, master, remote_cluster_client]" + resources: + requests: + memory: "4Gi" + cpu: "1" + limits: + memory: "6Gi" + cpu: "2" + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + - name: config-volume + mountPath: /usr/share/elasticsearch/config + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: ["elasticsearch"] + topologyKey: "kubernetes.io/hostname" + volumes: + - name: data + persistentVolumeClaim: + claimName: "elasticsearch-data-pvc-elasticsearch-3" + - name: config-volume + persistentVolumeClaim: + claimName: "elasticsearch-config-pvc-elasticsearch-3" diff --git a/fluentd-configMap.yaml b/fluentd-configMap.yaml index f9f4dac..34fbb5e 100644 --- a/fluentd-configMap.yaml +++ b/fluentd-configMap.yaml @@ -15,70 +15,27 @@ data: <label @main> <match sardine.log.**> - @type rewrite_tag_filter - <rule> - key @timestamp - pattern /^(202[4-9]|2[1-9]\d{3})/ - tag hot.${tag} - </rule> - <rule> - key @timestamp - pattern /^(202[0-3])/ - tag warm.${tag} - </rule> - <rule> - key @timestamp - pattern /^(201\d|200\d|19\d{2})/ - tag cold.${tag} - </rule> - </match> - - <match hot.sardine.log.**> - @type elasticsearch - @id output_elasticsearch_hot - hosts elasticsearch-0.elasticsearch-service.efk.svc.cluster.local:9200 - scheme http - user fluentd_user - password fluentd_password - logstash_format true - logstash_prefix logstash - index_date_pattern now/d - include_tag_key true - tag_key @log_name - flush_interval 5s - @log_level debug - </match> - - <match warm.sardine.log.**> - @type elasticsearch - @id output_elasticsearch_warm - hosts elasticsearch-1.elasticsearch-service.efk.svc.cluster.local:9200 - scheme http - user fluentd_user - password fluentd_password - logstash_format true - logstash_prefix logstash - index_date_pattern now/d - include_tag_key true - tag_key @log_name - flush_interval 10s - @log_level debug - </match> - - <match cold.sardine.log.**> - @type elasticsearch - @id output_elasticsearch_cold - hosts elasticsearch-2.elasticsearch-service.efk.svc.cluster.local:9200 - scheme http - user fluentd_user - password fluentd_password - logstash_format true - logstash_prefix logstash - index_date_pattern now/d - include_tag_key true - tag_key @log_name - flush_interval 30s - @log_level debug + @type copy + <store> + @type elasticsearch + @id output_elasticsearch + host elasticsearch-lb + port 9200 + scheme http + user fluentd_user + password fluentd_password + logstash_format true + logstash_prefix logstash + logstash_dateformat %Y.%m.%d + flush_interval 5s # 缩短批量写入间隔 + @log_level debug + id_key _id + remove_keys _id + </store> + <store> + @type stdout + @id output_stdout + </store> </match> </label> @@ -86,4 +43,3 @@ data: @type stdout @id output_stdout_all </match> - diff --git a/kibana-deployment.yaml b/kibana-deployment.yaml index ff4d0c0..470dd74 100644 --- a/kibana-deployment.yaml +++ b/kibana-deployment.yaml @@ -22,7 +22,7 @@ spec: - containerPort: 5601 env: - name: ELASTICSEARCH_HOSTS - value: "http://elasticsearch-lb.efk.svc.cluster.local:9200" + value: "http://elasticsearch.efk.svc.cluster.local:9200" - name: XPACK_SECURITY_ENCRYPTIONKEY value: "a_secure_random_string_of_32_characters" - name: XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY @@ -30,4 +30,4 @@ spec: - name: XPACK_REPORTING_ENCRYPTIONKEY value: "yet_another_secure_random_string_of_32_characters" - name: ELASTICSEARCH_SERVICEACCOUNTTOKEN - value: "" + value: "AAEAAWVsYXN0aWMva2liYW5hL215LXRva2VuOlo1dC0xNDFQVHQyajdBeEVtQzZjZ2c"