initial commit
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Aleksander Cynarski 2020-09-03 16:05:06 +02:00
commit dad8a7ff4c
12 changed files with 456 additions and 0 deletions

35
.drone.yml Normal file
View File

@ -0,0 +1,35 @@
kind: pipeline
name: default
steps:
- name: docker
image: plugins/docker
settings:
username:
from_secret: hub_username
password:
from_secret: hub_password
repo:
from_secret: hub_repo
tags: latest
when:
branch:
- master
- name: notify
image: appleboy/drone-telegram
settings:
token:
from_secret: telegram_bot_token
to:
from_secret: telegram_group_id
message: >
{{#success build.status}}
build {{build.link}} triggerd by {{commit.link}} from {{commit.author}} succeeded
{{else}}
build {{build.link}} triggerd by {{commit.link}} from {{commit.author}} failed
{{/success}}
when:
status:
- success
- failure

19
DOCKER_DEPLOY.md Normal file
View File

@ -0,0 +1,19 @@
# Drone docker deploy
## Użycie
```
cat ${args[0]} |awk -F\= '{system("drone secret add --repository='${args[1]}' --name="$1 " --data="$2)}'
```
Wymagane zmienne:
```
hub_username=
hub_password=
hub_repo=
telegram_bot_token=
telegram_group_id=
```

56
Dockerfile Normal file
View File

@ -0,0 +1,56 @@
FROM debian:stretch-slim as builder
# Fluent Bit version
ENV FLB_MAJOR 1
ENV FLB_MINOR 5
ENV FLB_PATCH 4
ENV FLB_VERSION 1.5.4
ARG FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v$FLB_VERSION.tar.gz
ENV FLB_SOURCE $FLB_TARBALL
RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log /tmp/fluent-bit-master/
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && \
apt-get install -y curl \
ca-certificates build-essential \
cmake make bash sudo wget unzip dh-make \
libsystemd-dev zlib1g-dev flex bison \
libssl1.1 libssl-dev libpq-dev postgresql-server-dev-all && \
apt-get install -y --reinstall lsb-base lsb-release \
&& curl -L -o "/tmp/fluent-bit.tar.gz" ${FLB_SOURCE} \
&& cd tmp/ && mkdir fluent-bit \
&& tar zxfv fluent-bit.tar.gz -C ./fluent-bit --strip-components=1 \
&& cd fluent-bit/build/ \
&& rm -rf /tmp/fluent-bit/build/*
WORKDIR /tmp/fluent-bit/build/
RUN cmake -DFLB_DEBUG=On \
-DFLB_TRACE=Off \
-DFLB_JEMALLOC=On \
-DFLB_TLS=On \
-DFLB_SHARED_LIB=Off \
-DFLB_EXAMPLES=Off \
-DFLB_HTTP_SERVER=On \
-DFLB_IN_SYSTEMD=On \
-DFLB_OUT_KAFKA=On \
-DFLB_OUT_PGSQL=On ..
RUN make -j $(getconf _NPROCESSORS_ONLN)
RUN install bin/fluent-bit /fluent-bit/bin/
# Configuration files
COPY conf/fluent-bit.conf \
conf/parsers.conf \
conf/parsers_ambassador.conf \
conf/parsers_java.conf \
conf/parsers_extra.conf \
conf/parsers_openstack.conf \
conf/parsers_cinder.conf \
conf/plugins.conf \
/fluent-bit/etc/
FROM gcr.io/distroless/cc
COPY --from=builder /fluent-bit /fluent-bit

48
conf/fluent-bit.conf Normal file
View File

@ -0,0 +1,48 @@
[SERVICE]
# Flush
# =====
# Set an interval of seconds before to flush records to a destination
Flush 5
# Daemon
# ======
# Instruct Fluent Bit to run in foreground or background mode.
Daemon Off
# Log_Level
# =========
# Set the verbosity level of the service, values can be:
#
# - error
# - warning
# - info
# - debug
# - trace
#
# By default 'info' is set, that means it includes 'error' and 'warning'.
Log_Level info
# Parsers_File
# ============
# Specify an optional 'Parsers' configuration file
Parsers_File parsers.conf
Plugins_File plugins.conf
# HTTP Server
# ===========
# Enable/Disable the built-in HTTP Server for metrics
HTTP_Server Off
HTTP_Listen 0.0.0.0
HTTP_Port 2020
[INPUT]
Name cpu
Tag cpu.local
# Interval Sec
# ====
# Read interval (sec) Default: 1
Interval_Sec 1
[OUTPUT]
Name stdout
Match *

117
conf/parsers.conf Normal file
View File

@ -0,0 +1,117 @@
[PARSER]
Name apache
Format regex
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name apache2
Format regex
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>.*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name apache_error
Format regex
Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
[PARSER]
Name nginx
Format regex
Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
# https://rubular.com/r/P8zgLD5K73fp2n
Name k8s-nginx-ingress
Format regex
Regex ^(?<host>[^ ]*) - - \[(?<time>[^\]]*)\] \\*"(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?\\*" (?<code>[^ ]*) (?<size>[^ ]*) \\*"(?<referer>[^\"]*)\\*" \\*"(?<agent>[^\"]*)\\*" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (\[(?<proxy_alternative_upstream_name>[^ ]*)\] )?(?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<reg_id>[^ ]*).*$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name json
Format json
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
# --
# Since Fluent Bit v1.2, if you are parsing Docker logs and using
# the Kubernetes filter, it's not longer required to decode the
# 'log' key.
#
# Command | Decoder | Field | Optional Action
# =============|==================|=================
#Decode_Field_As json log
[PARSER]
Name docker-daemon
Format regex
Regex time="(?<time>[^ ]*)" level=(?<level>[^ ]*) msg="(?<msg>[^ ].*)"
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
[PARSER]
Name syslog-rfc5424
Format regex
Regex ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Time_Keep On
[PARSER]
Name syslog-rfc3164-local
Format regex
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
Time_Key time
Time_Format %b %d %H:%M:%S
Time_Keep On
[PARSER]
Name syslog-rfc3164
Format regex
Regex /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
Time_Key time
Time_Format %b %d %H:%M:%S
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
[PARSER]
Name mongodb
Format regex
Regex ^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<component>[^ ]+)\s+\[(?<context>[^\]]+)]\s+(?<message>.*?) *(?<ms>(\d+))?(:?ms)?$
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
Time_Key time
[PARSER]
# https://rubular.com/r/3fVxCrE5iFiZim
Name envoy
Format regex
Regex ^\[(?<start_time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)? (?<protocol>\S+)" (?<code>[^ ]*) (?<response_flags>[^ ]*) (?<bytes_received>[^ ]*) (?<bytes_sent>[^ ]*) (?<duration>[^ ]*) (?<x_envoy_upstream_service_time>[^ ]*) "(?<x_forwarded_for>[^ ]*)" "(?<user_agent>[^\"]*)" "(?<request_id>[^\"]*)" "(?<authority>[^ ]*)" "(?<upstream_host>[^ ]*)"
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
Time_Keep On
Time_Key start_time
[PARSER]
# http://rubular.com/r/tjUt3Awgg4
Name cri
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
[PARSER]
Name kube-custom
Format regex
Regex (?<tag>[^.]+)?\.?(?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$

View File

@ -0,0 +1,6 @@
# Ambassador - open source Kubernetes-native API gateway for microservices built on the Envoy Proxy https://www.getambassador.io
[PARSER]
Name ambassador
Format regex
Regex ^(?<type>\S+) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?) (?<protocol>\S+)?" (?<response_code>\S+) (?<response_flags>\S+) (?<bytes_received>\S+) (?<bytes_sent>\S+) (?<duration>\S+) (?<x_envoy_upstream_service_time>\S+) "(?<x_forwarded_for>[^\"]*)" "(?<user_agent>[^\"]*)" "(?<x_request_id>[^\"]*)" "(?<authority>[^\"]*)" "(?<upstream_host>[^\"]*)"

10
conf/parsers_cinder.conf Normal file
View File

@ -0,0 +1,10 @@
[PARSER]
# http://rubular.com/r/IvZVElTgNl
Name ceph
Format regex
Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<message>.*)$
Time_Format %Y-%m-%d %H:%M:%S.%L
Time_Keep Off
Time_Key log_time

78
conf/parsers_extra.conf Normal file
View File

@ -0,0 +1,78 @@
# Extra set of common parsers
[PARSER]
# http://rubular.com/r/cCVd1HLCAO
Name crowbar
Format regex
Regex ^.*\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+?(?<severity>[^ ]\w+)([\s-]*):?\s+(?<message>.*)
Time_Format %Y-%m-%dT%H:%M:%S
Time_Keep Off
Time_Key log_time
[PARSER]
# http://rubular.com/r/frDgnElXW9
Name chefclient
Format regex
Regex ^\[(?<log_time>[^ ][-.\d\+:]+T[:\d]*)([^\]])*?\]\s+(?<severity>[^ ]\w+):\s+(?<message>.*)$
Time_Format %Y-%m-%dT%H:%M:%S
Time_Keep Off
Time_Key log_time
[PARSER]
Name mysql_error
Format regex
#Regex ^(?<log_time>[^ +][ -:0-9TZ]+|[[:upper:]][[:lower:]]{2})(\+\d+:\d+[TZ]*){0,1}\s*(?<myid>[^ ]\d+)\s+\[(?<severity>[^ ]\w+)\](\s+(?<subsystem>[^ ]\w+):){0,1}\s+(?<message>.*)$
Regex ^(?<log_time>[^ +][-\d]+[\ T]*[:\dZ]+)\s*(?<myid>[^ ]\d+)\s+\[(?<severity>[^ ]\w+)\](\s+(?<subsystem>[^ ]\w+):){0,1}\s+(?<message>.*)$
Time_Format %Y-%m-%d %H:%M:%S
Time_Keep Off
Time_Key log_time
[PARSER]
Name mysql_slow
Format regex
Regex ^# User\@Host:\s+(?<user>[^\@][\w\[\]]+)[@\s]+(?<dbhost>[^ ][-.\w]+)\s+(\[(?<dbhost_address>[.\d]+)\]){0,1}\s+(?<message>.*)$
[PARSER]
Name pacemaker
Format regex
Regex ^\s*(?<log_time>[^ ]* {1,2}[^ ]* [^ ]*) \[(?<pid>\d+)\] (?<node>[\-\w]*)\s*(?<component>\w*):\s+(?<severity>\w+):\s+(?<message>.*)$
#Time_Format %Y-%m-%dT%H:%M:%S
Time_Format %b %d %H:%M:%S
Time_Keep Off
Time_Key log_time
#Types pid:integer
[PARSER]
Name rabbitmq
Format regex
Regex ^=(?<severity>[^ ]\w+)\s+REPORT[=\s]*(?<log_time>[^ =][-:.\d\w]+)[\s=]+(?<message>.*)$
Time_Format %d-%b-%Y::%H:%M:%S
Time_Keep Off
Time_Key log_time
[PARSER]
Name http_statement
Format regex
Regex ^.*((?<req_method>GET|POST|PUT|DELETE|CONNECT|OPTIONS|HEAD[^ ]\w+)\s*(?<req_path>[^ ][-._?=%&\/[:alnum:]]*)\s*(?<req_protocol>[^ ][.\/\dHTFSP]+){0,1})(['"\s]*){0,1}((\s*status:\s*(?<req_status>[^ ]\d+)){0,1}(\s*len:\ (?<req_len>[^ ]\d+)){0,1}(\s*time:\s*(?<req_log_time>[^ ][.\d]+)){0,1}(\s*microversion:\s*(?<req_mver>[^ ][.\d]+)){0,1}){0,1}$
[PARSER]
Name universal
Format regex
Regex ^(?<message>.*)$
[PARSER]
Name uuid
Format regex
Regex (?<uuid>[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})
#UUID v1 :
#/^[0-9A-F]{8}-[0-9A-F]{4}-[1][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
#UUID v2 :
#/^[0-9A-F]{8}-[0-9A-F]{4}-[2][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
#UUID v3 :
#/^[0-9A-F]{8}-[0-9A-F]{4}-[3][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
#UUID v4 :
#/^[0-9A-F]{8}-[0-9A-F]{4}-[4][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i
#UUID v5 :
#/^[0-9A-F]{8}-[0-9A-F]{4}-[5][0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i

6
conf/parsers_java.conf Normal file
View File

@ -0,0 +1,6 @@
[PARSER]
Name java_multiline
Format regex
Regex /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S

26
conf/parsers_mult.conf Normal file
View File

@ -0,0 +1,26 @@
[PARSER]
Name mult_first
Format regex
Regex Started (?<method>[^ ]+) "(?<path>[^"]+)" for (?<host>[^ ]+) at (?<time>[^ ]+ [^ ]+ [^ ]+)
Time_Key time
Time_Format %Y-%m-%d %H:%M:%S %z
[PARSER]
Name mult_1
Format regex
Regex /Processing by (?<controller>[^\u0023]+)\u0023(?<controller_method>[^ ]+) as (?<format>[^ ]+?)$/
[PARSER]
Name mult_2
Format regex
Regex ( Parameters: (?<parameters>[^ ]+))?
[PARSER]
Name mult_3
Format regex
Regex / Rendered (?<template>[^ ]+) within (?<layout>.+) \([\d\.]+ms\)/
[PARSER]
Name mult_4
Format regex
Regex Completed (?<code>[^ ]+) [^ ]+ in (?<runtime>[\d\.]+)ms \(Views: (?<view_runtime>[\d\.]+)ms \| ActiveRecord: (?<ar_runtime>[\d\.]+)ms\)

View File

@ -0,0 +1,53 @@
# Example openstack parsers, see source for recent updates:
# https://github.com/salt-formulas/salt-formula-fluentbit/tree/master/metadata/service/config/parser
# Take these parsers as an example, as they may not fully fit all possible
# openstack deployments from all vendors, etc..
[PARSER]
Name cinder
Format regex
Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$
Time_Format %Y-%m-%d %H:%M:%S.%L
Time_Keep Off
Time_Key log_time
[PARSER]
Name glance
Format regex
Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$
Time_Format %Y-%m-%d %H:%M:%S.%L
Time_Keep Off
Time_Key log_time
[PARSER]
Name heat
Format regex
Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$
Time_Format %Y-%m-%d %H:%M:%S.%L
Time_Keep Off
Time_Key log_time
[PARSER]
Name keystone
Format regex
Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$
Time_Format %Y-%m-%d %H:%M:%S.%L
Time_Keep Off
Time_Key log_time
[PARSER]
Name neutron
Format regex
Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$
Time_Format %Y-%m-%d %H:%M:%S.%L
Time_Keep Off
Time_Key log_time
[PARSER]
Name nova
Format regex
Regex ^(?<log_time>[^ ][-.\d\+:T]+[ ]*[.:\d]*)\s+(?<pid>[^ ]\d+)\s+(?<severity>[^ ][.-_\w]+)\s+(?<component>[^ ][.-_\w]+)(\s+\[(-|(?<req_id>[^ ][-\w]*) (?<req_user>[^ ][-\w]*) (?<req_project>[^ ][-\w]*) (?<req_domain>[^ ][-\w]*) (?<req_user_domain>[^ ][-\w]*) (?<req_project_domain>[^ ][-\w]*))\]){1}\s+(?<message>.*)$
Time_Format %Y-%m-%d %H:%M:%S.%L
Time_Keep Off
Time_Key log_time

2
conf/plugins.conf Normal file
View File

@ -0,0 +1,2 @@
[PLUGINS]
# Path /path/to/out_gstdout.so