diff --git a/efk/fileBeat-config.yaml b/efk/fileBeat-config.yaml new file mode 100644 index 0000000..600eee9 --- /dev/null +++ b/efk/fileBeat-config.yaml @@ -0,0 +1,62 @@ +# filebeat-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: efk +data: + filebeat.yml: | + filebeat.inputs: + - type: http_endpoint + enabled: true + listen_address: 0.0.0.0:8888 + path: "/" + json.keys_under_root: true + + processors: + # 提取路径中的变量（严格模式） + - dissect: + tokenizer: "/tanya.candle.%{currency}.%{year}.%{interval}" + field: "http.request.path" + target_prefix: "" + ignore_missing: false # 关键：关闭忽略缺失 + + # 强制设置默认值（即使字段为空） + - script: + lang: javascript + source: | + function process(event) { + // 先检查字段是否存在，不存在则设置默认值 + if (!event.containsKey('currency') || event.get('currency') === '') { + event.put('currency', 'unknown'); + } + if (!event.containsKey('year') || event.get('year') === '') { + event.put('year', '0000'); + } + if (!event.containsKey('interval') || event.get('interval') === '') { + event.put('interval', '0D'); + } + } + + output.elasticsearch: + hosts: ["http://elasticsearch:9200"] + username: "fluentd_user" + password: "fluentd_password" + indices: + - index: "logstash-candle-${currency}-${year}-${interval}" + # 严格验证字段值非空 + when.and: + - not.equals: + currency: "" + - not.equals: + year: "" + - not.equals: + interval: "" + - index: "fallback-index" + when.or: + - equals: + currency: "" + - equals: + year: "" + - equals: + interval: "" diff --git a/efk/fileBeat-daemonset.yaml b/efk/fileBeat-daemonset.yaml new file mode 100644 index 0000000..1a4ea40 --- /dev/null +++ b/efk/fileBeat-daemonset.yaml @@ -0,0 +1,38 @@ +# filebeat-daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: filebeat + namespace: efk +spec: + selector: + matchLabels: + app: filebeat + template: + metadata: + labels: + app: filebeat + spec: + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:8.12.0 + args: [ + "-c", "/etc/filebeat.yml", + "-e", + "-strict.perms=false" + ] + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: varlog + mountPath: /var/log + readOnly: true + volumes: + - name: config + configMap: + name: filebeat-config + - name: varlog + hostPath: + path: /var/log diff --git a/efk/fileBeat-ingress.yaml b/efk/fileBeat-ingress.yaml new file mode 100644 index 0000000..fa81654 --- /dev/null +++ b/efk/fileBeat-ingress.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: filebeat-ingress + namespace: efk + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + # 添加 SSL 终止支持的注释，如果需要 TLS/SSL 支持 + # nginx.ingress.kubernetes.io/ssl-redirect: "true" +spec: + ingressClassName: traefik + rules: + - host: filebeat.k8s.xunlang.home + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: filebeat-service + port: + number: 8888 diff --git a/efk/fileBeat-service.yaml b/efk/fileBeat-service.yaml new file mode 100644 index 0000000..ca32ead --- /dev/null +++ b/efk/fileBeat-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: filebeat-service + namespace: efk +spec: + type: LoadBalancer + selector: + k8s-app: filebeat + ports: + - protocol: TCP + port: 8888 # Service 暴露的端口 + targetPort: 8888 # Fluentd 容器内部的端口 + diff --git a/efk/fluentd-configMap2.yaml b/efk/fluentd-configMap2.yaml index 3e0b6de..e698764 100644 --- a/efk/fluentd-configMap2.yaml +++ b/efk/fluentd-configMap2.yaml @@ -10,6 +10,10 @@ data: @id input_http port 8888 @label @main + @log_level debug + + @type json + - @type stdout @id output_stdout_all diff --git a/efk/fluentd-daemonset.yaml b/efk/fluentd-daemonset.yaml index 06430c8..249b7c4 100644 --- a/efk/fluentd-daemonset.yaml +++ b/efk/fluentd-daemonset.yaml @@ -24,7 +24,7 @@ spec: effect: NoSchedule containers: - name: fluentd - image: localhost:32000/efk/fluentd-k8s-daemon-withrewrite + image: fluent/fluentd-kubernetes-daemonset:v1.17.1-debian-elasticsearch8-1.0 env: - name: K8S_NODE_NAME valueFrom: diff --git a/efk/init.sh b/efk/init.sh index e25c5ee..210f4b0 100644 --- a/efk/init.sh +++ b/efk/init.sh @@ -11,6 +11,10 @@ sudo su mkdir /var/snap/microk8s/common/mnt/data/elasticsearch-data -p || true mkdir /var/snap/microk8s/common/mnt/data/elasticsearch-config -p || true cp config/* /var/snap/microk8s/common/mnt/data/elasticsearch-config -r + +microk8s.ctr image import ~/shared/powerssd/images/docker/x86/fluentd-x86-image.tar +microk8s.ctr image import ~/shared/powerssd/images/docker/x86/elasticsearch-8-8-0.tar + // 创建 es 资源 microk8s.kubectl apply -f efk-namespace.yaml microk8s.kubectl apply -f elasticsearch-deployment.yaml @@ -19,7 +23,7 @@ microk8s.kubectl apply -f elasticsearch-pv.yaml microk8s.kubectl apply -f elasticsearch-pvc.yaml microk8s.kubectl apply -f elasticsearch-service.yaml // 这个时候正在创建elasticsearch的pod，需要拉取镜像，大概1个多小时，如果有离线的直接导入离线的镜像 -sleep 3600 +sleep 60 ./createSecure_passwd_forES.sh ./createFluentdAccoutnIn.sh diff --git a/efk/kibana-deployment.yaml b/efk/kibana-deployment.yaml index 0bce0a6..1e49872 100644 --- a/efk/kibana-deployment.yaml +++ b/efk/kibana-deployment.yaml @@ -30,4 +30,4 @@ spec: - name: XPACK_REPORTING_ENCRYPTIONKEY value: "yet_another_secure_random_string_of_32_characters" - name: ELASTICSEARCH_SERVICEACCOUNTTOKEN - value: "AAEAAWVsYXN0aWMva2liYW5hL215LXRva2VuOmlnZWdqMGp5UWI2ODVaZzZaMElVVmc" + value: "AAEAAWVsYXN0aWMva2liYW5hL215LXRva2VuOmd3ZG9idU5kVEM2b3BLRUJDS2g5YVE" diff --git a/efk/myFluentd.Dockerfile b/efk/myFluentd.Dockerfile index 22ea2ba..5b5fe2b 100644 --- a/efk/myFluentd.Dockerfile +++ b/efk/myFluentd.Dockerfile @@ -1,2 +1,4 @@ FROM fluent/fluentd-kubernetes-daemonset:v1.16-debian-elasticsearch8-2 -RUN fluent-gem install fluent-plugin-rewrite-tag-filter +USER root +RUN fluent-gem install fluent-plugin-rewrite-tag-filter fluent-plugin-dynamic +USER fluent diff --git a/efk/tem.yaml b/efk/tem.yaml new file mode 100644 index 0000000..83a6ff3 --- /dev/null +++ b/efk/tem.yaml @@ -0,0 +1,8 @@ +# filebeat-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: efk +data: + filebeat.yml: |