За основу возьмем официальный Docker образ fluntd
Установим необходимые gem‘ы для GELF протокола и Kubernetes‘а, а так же укажем, что fluentd будет запускаться от пользователя root, иначе будет получать ошибку доступа.
Dockerfile:
FROM fluent/fluentd:latest
USER root
WORKDIR /home/fluent
RUN set -ex \
&& apk add --no-cache --virtual .build-deps \
build-base \
ruby-dev \
&& echo 'gem: --no-document' >> /etc/gemrc \
&& gem install fluent-plugin-secure-forward \
&& gem install fluent-plugin-record-reformer \
&& gem install fluent-plugin-gelf-hs \
&& gem install fluent-plugin-kubernetes_metadata_filter \
&& gem install fluent-plugin-rewrite-tag-filter \
&& apk del .build-deps \
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
# Copy plugins
COPY plugins /fluentd/plugins/
# Environment variables
ENV FLUENTD_OPT=""
ENV FLUENTD_CONF="fluent.conf"
# For Kubernetes. Start Fluentd by user root
ENV FLUENT_UID="0"
# Run Fluentd
CMD fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT
plugins/parser_kubernetes.rb:
#
# Fluentd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# The following Fluentd parser plugin, aims to simplify the parsing of multiline
# logs found in Kubernetes nodes. Since many log files shared the same format and
# in order to simplify the configuration, this plugin provides a 'kubernetes' format
# parser (built on top of MultilineParser).
#
# When tailing files, this 'kubernetes' format should be applied to the following
# log file sources:
#
# - /var/log/kubelet.log
# - /var/log/kube-proxy.log
# - /var/log/kube-apiserver.log
# - /var/log/kube-controller-manager.log
# - /var/log/kube-scheduler.log
# - /var/log/rescheduler.log
# - /var/log/glbc.log
# - /var/log/cluster-autoscaler.log
#
# Usage:
#
# ---- fluentd.conf ----
#
# <source>
# type tail
# format kubernetes
# path ./kubelet.log
# read_from_head yes
# tag kubelet
# </source>
#
# ---- EOF ---
require 'fluent/parser'
module Fluent
class KubernetesParser < Fluent::TextParser::MultilineParser
Fluent::Plugin.register_parser("kubernetes", self)
CONF_FORMAT_FIRSTLINE = %q{/^\w\d{4}/}
CONF_FORMAT1 = %q{/^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/}
CONF_TIME_FORMAT = "%m%d %H:%M:%S.%N"
def configure(conf)
conf['format_firstline'] = CONF_FORMAT_FIRSTLINE
conf['format1'] = CONF_FORMAT1
conf['time_format'] = CONF_TIME_FORMAT
super
end
end
end
После сборки кастомного образа, переходим к настройке Kubernetes‘а
rbac.yaml:
apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-gelf
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
name: kube-gelf-metadata-filter
rules:
- apiGroups:
- ""
resources:
- pods
- pods/exec
- namespaces
verbs: ["get", "list", "watch", "create"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: kube-gelf
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kube-gelf-metadata-filter
subjects:
- kind: ServiceAccount
name: kube-gelf
namespace: kube-system
configmap.yml:
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-gelf
namespace: kube-system
labels:
app: kube-gelf
data:
GELF_HOST: "graylog-svc.staging.svc.cluster.local"
GELF_PORT: "12201"
GELF_PROTOCOL: "udp"
fluent.conf: |
<match fluent.**>
@type null
</match>
<source>
@type tail
path /var/log/containers/*.log
pos_file /pos/containers.pos
time_key time
time_format %Y-%m-%dT%H:%M:%S.%NZ
tag kubernetes.*
format json
read_from_head true
</source>
<filter kubernetes.**>
@type kubernetes_metadata
ca_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file /var/run/secrets/kubernetes.io/serviceaccount/token
</filter>
<filter access>
@type record_transformer
<record>
hostname "#{ENV['NODENAME']}"
</record>
</filter>
### NOT COLLECT LOGS FROM ALL EXCEPT NAMESPACE STAGING ###
<match **fluentd**>
@type null
</match>
<match **kube-system**>
@type null
</match>
<match **monitoring**>
@type null
</match>
<match **cert-manager**>
@type null
</match>
<match **efs-staging**>
@type null
</match>
##########################################################
<match kubernetes.**>
@type rewrite_tag_filter
<rule>
key $['kubernetes']['labels']['app']
pattern ^(.+)$
tag $1
</rule>
</match>
<match **>
@type copy
<store>
@type gelf
include_tag_key true
host "#{ENV['GELF_HOST']}"
port "#{ENV['GELF_PORT']}"
protocol "#{ENV['GELF_PROTOCOL']}"
flush_interval 10s
num_threads 2
use_record_host true
buffer_chunk_limit 4096K
buffer_queue_limit 512
max_retry_wait 300
</store>
</match>
# Valid log_level's are: fatal, error, warn, info, debug, trace
<system>
log_level warn
</system>
В файле «fluent.conf» я исключил все, кроме namespace staging, в котором у меня крутятся все контейнеры.
manifest.yml:
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-gelf
namespace: kube-system
spec:
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
app: kube-gelf
template:
metadata:
labels:
app: kube-gelf
spec:
serviceAccountName: kube-gelf
dnsPolicy: ClusterFirst
containers:
- name: agent
image: my-image-repo:kube-gelf
imagePullPolicy: "IfNotPresent"
env:
- name: GELF_HOST
valueFrom:
configMapKeyRef:
name: kube-gelf
key: GELF_HOST
- name: GELF_PORT
valueFrom:
configMapKeyRef:
name: kube-gelf
key: GELF_PORT
- name: GELF_PROTOCOL
valueFrom:
configMapKeyRef:
name: kube-gelf
key: GELF_PROTOCOL
- name: NODENAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: varlog
mountPath: /var/log
readOnly: true
- name: gelf-pos
mountPath: /pos
readOnly: false
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: kube-gelf-config
mountPath: /fluentd/etc/fluent.conf
subPath: fluent.conf
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: kube-gelf-config
configMap:
name: kube-gelf
items:
- key: fluent.conf
path: fluent.conf
- name: gelf-pos
hostPath:
path: /var/log/pos
Не забываем указать свой путь с собраному образу.