git revert

efk/fileBeat-config.yaml

@@ -0,0 +1,62 @@
+# filebeat-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: efk
+data:
+  filebeat.yml: |
+    filebeat.inputs:
+    - type: http_endpoint
+      enabled: true
+      listen_address: 0.0.0.0:8888
+      path: "/"
+      json.keys_under_root: true
+
+    processors:
+      # 提取路径中的变量（严格模式）
+      - dissect:
+          tokenizer: "/tanya.candle.%{currency}.%{year}.%{interval}"
+          field: "http.request.path"
+          target_prefix: ""
+          ignore_missing: false  # 关键：关闭忽略缺失
+
+      # 强制设置默认值（即使字段为空）
+      - script:
+          lang: javascript
+          source: |
+            function process(event) {
+              // 先检查字段是否存在，不存在则设置默认值
+              if (!event.containsKey('currency') || event.get('currency') === '') {
+                event.put('currency', 'unknown');
+              }
+              if (!event.containsKey('year') || event.get('year') === '') {
+                event.put('year', '0000');
+              }
+              if (!event.containsKey('interval') || event.get('interval') === '') {
+                event.put('interval', '0D');
+              }
+            }
+
+    output.elasticsearch:
+      hosts: ["http://elasticsearch:9200"]
+      username: "fluentd_user"
+      password: "fluentd_password"
+      indices:
+        - index: "logstash-candle-${currency}-${year}-${interval}"
+          # 严格验证字段值非空
+          when.and:
+            - not.equals:
+                currency: ""
+            - not.equals:
+                year: ""
+            - not.equals:
+                interval: ""
+        - index: "fallback-index"
+          when.or:
+            - equals:
+                currency: ""
+            - equals:
+                year: ""
+            - equals:
+                interval: ""

efk/fileBeat-daemonset.yaml

@@ -0,0 +1,38 @@
+# filebeat-daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: filebeat
+  namespace: efk
+spec:
+  selector:
+    matchLabels:
+      app: filebeat
+  template:
+    metadata:
+      labels:
+        app: filebeat
+    spec:
+      containers:
+      - name: filebeat
+        image: docker.elastic.co/beats/filebeat:8.12.0
+        args: [
+          "-c", "/etc/filebeat.yml",
+          "-e",
+          "-strict.perms=false"
+        ]
+        volumeMounts:
+        - name: config
+          mountPath: /etc/filebeat.yml
+          readOnly: true
+          subPath: filebeat.yml
+        - name: varlog
+          mountPath: /var/log
+          readOnly: true
+      volumes:
+      - name: config
+        configMap:
+          name: filebeat-config
+      - name: varlog
+        hostPath:
+          path: /var/log

efk/fileBeat-ingress.yaml

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: filebeat-ingress
+  namespace: efk
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    # 添加 SSL 终止支持的注释，如果需要 TLS/SSL 支持
+    # nginx.ingress.kubernetes.io/ssl-redirect: "true"
+spec:
+  ingressClassName: traefik
+  rules:
+  - host: filebeat.k8s.xunlang.home
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: filebeat-service
+            port:
+              number: 8888

efk/fileBeat-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: filebeat-service
+  namespace: efk
+spec:
+  type: LoadBalancer
+  selector:
+    k8s-app: filebeat
+  ports:
+  - protocol: TCP
+    port: 8888          # Service 暴露的端口
+    targetPort: 8888    # Fluentd 容器内部的端口
+

efk/fluentd-configMap2.yaml

@@ -10,6 +10,10 @@ data:
       @id input_http
       port 8888
       @label @main
+      @log_level debug
+      <parse>
+        @type json
+      </parse>
     </source>
 
     <label @main>
@@ -36,33 +40,6 @@ data:
           @id output_stdout
         </store>
       </match>
-
-      <match tanya.**>
-        @type copy
-        <store>
-          @type elasticsearch
-          @id output_elasticsearch_tanya
-          host elasticsearch
-          port 9200
-          scheme http
-          user fluentd_user
-          password fluentd_password
-          logstash_format false
-          index_name logstash-candle_${tag_parts[1]}_${tag_parts[2]}_${tag_parts[3]}
-          flush_interval 5s
-          @log_level debug
-          id_key _id
-          remove_keys _id
-          include_tag_key true
-          tag_key @log_name
-        </store>
-        <store>
-          @type stdout
-          @id output_stdout_tanya
-        </store>
-      </match>
-    </label>
-
     <match **>
       @type stdout
       @id output_stdout_all

efk/fluentd-daemonset.yaml

@@ -24,7 +24,7 @@ spec:
         effect: NoSchedule
       containers:
       - name: fluentd
-        image: localhost:32000/efk/fluentd-k8s-daemon-withrewrite
+        image: fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-elasticsearch8-1.0
         env:
           - name: K8S_NODE_NAME
             valueFrom:

efk/init.sh

@@ -11,6 +11,10 @@ sudo su
 mkdir /var/snap/microk8s/common/mnt/data/elasticsearch-data -p || true
 mkdir /var/snap/microk8s/common/mnt/data/elasticsearch-config -p || true
 cp config/* /var/snap/microk8s/common/mnt/data/elasticsearch-config -r
+
+microk8s.ctr image import ~/shared/powerssd/images/docker/x86/fluentd-x86-image.tar
+microk8s.ctr image import ~/shared/powerssd/images/docker/x86/elasticsearch-8-8-0.tar
+
 // 创建 es 资源
 microk8s.kubectl apply -f efk-namespace.yaml
 microk8s.kubectl apply -f elasticsearch-deployment.yaml
@@ -19,7 +23,7 @@ microk8s.kubectl apply -f elasticsearch-pv.yaml
 microk8s.kubectl apply -f elasticsearch-pvc.yaml
 microk8s.kubectl apply -f elasticsearch-service.yaml
 // 这个时候正在创建elasticsearch的pod，需要拉取镜像，大概1个多小时，如果有离线的直接导入离线的镜像 
-sleep 3600
+sleep 60
 ./createSecure_passwd_forES.sh
 ./createFluentdAccoutnIn.sh
 

efk/kibana-deployment.yaml

@@ -30,4 +30,4 @@ spec:
             - name: XPACK_REPORTING_ENCRYPTIONKEY
               value: "yet_another_secure_random_string_of_32_characters"
             - name: ELASTICSEARCH_SERVICEACCOUNTTOKEN
-              value: "AAEAAWVsYXN0aWMva2liYW5hL215LXRva2VuOmlnZWdqMGp5UWI2ODVaZzZaMElVVmc"
+              value: "AAEAAWVsYXN0aWMva2liYW5hL215LXRva2VuOmd3ZG9idU5kVEM2b3BLRUJDS2g5YVE"

efk/myFluentd.Dockerfile

@@ -1,2 +1,4 @@
 FROM fluent/fluentd-kubernetes-daemonset:v1.16-debian-elasticsearch8-2
-RUN fluent-gem install fluent-plugin-rewrite-tag-filter
+USER root
+RUN fluent-gem install fluent-plugin-rewrite-tag-filter fluent-plugin-dynamic
+USER fluent

efk/tem.yaml

@@ -0,0 +1,8 @@
+# filebeat-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: efk
+data:
+  filebeat.yml: |