fluent-bit日志转发到S3

2023-04-20 18:24:43

fluent-bit是一个快速的轻量级日志处理器和转发器，它可运行在Linux，OSX，Windows和BSD类操作系统。它使用C语言编写实现。它在设计时就考虑了高性能和低资源消耗。

本次操作是将nginx的日志通过fluent-bit格式化后，输出json并压缩上传到S3中。

一、创建credentials，文件包含S3的存储桶的访问权限。

credentials ^[1]

[default]
  aws_access_key_id = your-aws-ak
  aws_secret_access_key = your-aws-sk

二、parsers是nginx的日志格式定义。

> $ cat parsers.conf

#[PARSER]
#    Name   nginx
#    Format ltsv
#    Time_Key time
#    Time_Format [%d/%b/%Y:%H:%M:%S %z]

[PARSER]
    name nginx
    Format regex
    #Regex ^(?<INT>[^ ]+) (?<FLOAT>[^ ]+) (?<BOOL>[^ ]+) (?<STRING>.+)$
    Regex ^(?<remote>.+)\t(?<name>.+)\t(?<time_local>.+)\t(?<req>.+)\t(?<host>.+)\t(?<code>.+)\t(?<sent>.+)\t(?<reqtime>.+)\t(?<upstream>.+)\t(?<reqbody>.+)\t(?<referer>.+)\t(?<ua>.+)\t(?<xff>.+)$


[PARSER]
    name nginxerr
    Format regex
    Regex ^(?<time>.+) \[error\] (?<msg>.+), client: (?<client>.+), server: (?<server>.+), request: "(?<request>.+)", upstream: "(?<upstream>.+)", host: "(?<host>.+)"$%

三、fluent-bit的主配置文件。

cat fluent-bit.conf

 [SERVICE]
    Flush           5
    Daemon          off
    Log_Level       info
    HTTP_Monitoring On
    HTTP_Port       2020
    Parsers_File parsers.conf

#@INCLUDE inputs.conf
#@INCLUDE outputs.conf
[INPUT]
    #Name forward
    #unix_path /var/run/fluent.sock
    Name       tail
    Tag	      nginx.access
    Path       /wwwlogs/nginx/*-access.log
    Parser     nginx
    DB	       /var/log/access.db
    Mem_Buf_Limit  5MB
    Skip_Long_Lines  On
    Refresh_Interval  10

[INPUT]
    Name        tail
    Tag         nginx.error
    Path 	/wwwlogs/nginx/*-error.log
    Parser      nginxerr
    DB	        /var/log/error.db
    Mem_Buf_Limit  5MB
    Skip_Long_Lines  On
    Refresh_Interval  10

[FILTER]
    Name parser
    Match nginx.error
    Parser nginxerr
    Key_Name log

[FILTER]
    Name parser
    Match nginx.access
    Parser nginx
    Key_Name log

[OUTPUT]
    Name s3
    Match  nginx.error
    region cn-northwest-1
    bucket loook
    total_file_size 50M
    upload_timeout 60m
    store_dir /home
    s3_key_format /samples/%Y/month=%m/day=%d/error-%H-%M-%S
    compression gzip
    use_put_object On
    workers 2

[OUTPUT]
    Name s3
    Match  nginx.access
    region cn-northwest-1
    bucket loook
    total_file_size 50M
    upload_timeout 60m
    store_dir /home
    s3_key_format /samples/%Y/month=%m/day=%d/access-%H-%M-%S
    compression gzip
    use_put_object On
    workers 2

四、kubernetes-kustomize

 configMapGenerator:
- name: aws-config
  files:
  - fluentbit/credentials
- name: fluentbit-config
  files:
  - fluentbit/fluent-bit.conf
  - fluentbit/parsers.conf

五、工作负载定义（部分）

       - image: amazon/aws-for-fluent-bit:2.13.0
        imagePullPolicy: IfNotPresent
        #imagePullPolicy: Always
        name: fluent-bit
        env:
          - name: POD_IP
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: status.podIP
          - name: pod_name
            valueFrom:
              fieldRef:
                apiVersion: v1
                fieldPath: metadata.name
        #workingDir: /fluent-bit
        #command: ["./bin/fluent-bit","-e","./cloudwatch.so","-p","credentials_endpoint=/fluent-bit/etc/cloudwatch-credentials_endpoint"]
        resources:
          limits:
            cpu: 50m
            memory: 64Mi
          requests:
            cpu: 50m
            memory: 64Mi
        volumeMounts:
          - mountPath: /wwwlogs/nginx
            name: volume-log
          - mountPath: /fluent-bit/etc/
            name: fluentbit-config
          - mountPath: /root/.aws/
            name: aws-config