LogstashExample1

Пример

Это продолжение статьи https://noname.com.ua/mediawiki/index.php?title=Logstash

Задача

Задача - отправка логов от тестового приложения в Logstash

• Для тестовых устновок - писать максимально подробные логи приложения
  • Nginx - настолько подробно насколько это возможно
  • Логи которые пишет бекенд (Ruby)
• Для Staging/Prod - менее подробные логи (что бы не перегрузить Elasticsearch)

nginx

Nginx умеет писать логи в Json (для максимально подробного логгирования требуется поддержка Lua)

• Как логгировать хедеры: https://stackoverflow.com/questions/24380123/how-to-log-all-headers-in-nginx
• Пишем все что возможно: https://noname.com.ua/mediawiki/index.php/Nginx_Log_Post

Конфигурация (только значимые части)=

   # Log in JSON Format
    log_format nginxlog_json escape=json
    '{ '
    '"nginx_http_user_agent": "$http_user_agent",'
    '"nginx_ancient_browser": "$ancient_browser",'
    '"nginx_body_bytes_sent": "$body_bytes_sent",'
    '"nginx_bytes_sent": "$bytes_sent",'
    '"nginx_connection": "$connection",'
    '"nginx_connection_requests": "$connection_requests",'
    '"nginx_connections_active": "$connections_active",'
    '"nginx_connections_reading": "$connections_reading",'
    '"nginx_connections_waiting": "$connections_waiting",'
    '"nginx_connections_writing": "$connections_writing",'
    '"nginx_content_length": "$content_length",'
    '"nginx_content_type": "$content_type",'
    '"nginx_cookie_": "$cookie_",'
    '"nginx_document_root": "$document_root",'
    '"nginx_document_uri": "$document_uri",'
    '"nginx_fastcgi_path_info": "$fastcgi_path_info",'
    '"nginx_fastcgi_script_name": "$fastcgi_script_name",'
    '"nginx_host": "$host",'
    '"nginx_hostname": "$hostname",'
    '"nginx_https": "$https",'
    '"nginx_invalid_referer": "$invalid_referer",'
    '"nginx_is_args": "$is_args",'
    '"nginx_limit_conn_status": "$limit_conn_status",'
    '"nginx_limit_rate": "$limit_rate",'
    '"nginx_limit_req_status": "$limit_req_status",'
    '"nginx_modern_browser": "$modern_browser",'
    '"nginx_msec": "$msec",'
    '"nginx_msie": "$msie",'
    '"nginx_nginx_version": "$nginx_version",'
    '"nginx_proxy_add_x_forwarded_for": "$proxy_add_x_forwarded_for",'
    '"nginx_proxy_host": "$proxy_host",'
    '"nginx_proxy_port": "$proxy_port",'
    '"nginx_proxy_protocol_addr": "$proxy_protocol_addr",'
    '"nginx_proxy_protocol_port": "$proxy_protocol_port",'
    '"nginx_proxy_protocol_server_addr": "$proxy_protocol_server_addr",'
    '"nginx_proxy_protocol_server_port": "$proxy_protocol_server_port",'
    '"nginx_query_string": "$query_string",'
    '"nginx_realip_remote_addr": "$realip_remote_addr",'
    '"nginx_realip_remote_port": "$realip_remote_port",'
    '"nginx_remote_addr": "$remote_addr",'
    '"nginx_remote_port": "$remote_port",'
    '"nginx_remote_user": "$remote_user",'
    '"nginx_request": "$request",'
    '"nginx_request_headers": "$request_headers",'
    '"nginx_request_body": "$request_body",'
    '"nginx_request_id": "$request_id",'
    '"nginx_request_length": "$request_length",'
    '"nginx_request_method": "$request_method",'
    '"nginx_request_time": "$request_time",'
    '"nginx_request_uri": "$request_uri",'
    '"nginx_scheme": "$scheme",'
    '"nginx_server_addr": "$server_addr",'
    '"nginx_server_name": "$server_name",'
    '"nginx_server_port": "$server_port",'
    '"nginx_server_port": "$server_port",'
    '"nginx_server_protocol": "$server_protocol",'
    '"nginx_ssl_cipher": "$ssl_cipher",'
    '"nginx_ssl_ciphers": "$ssl_ciphers",'
    '"nginx_ssl_client_cert": "$ssl_client_cert",'
    '"nginx_ssl_client_escaped_cert": "$ssl_client_escaped_cert",'
    '"nginx_ssl_client_fingerprint": "$ssl_client_fingerprint",'
    '"nginx_ssl_client_i_dn": "$ssl_client_i_dn",'
    '"nginx_ssl_client_raw_cert": "$ssl_client_raw_cert",'
    '"nginx_ssl_client_s_dn": "$ssl_client_s_dn",'
    '"nginx_ssl_client_serial": "$ssl_client_serial",'
    '"nginx_ssl_client_v_end": "$ssl_client_v_end",'
    '"nginx_ssl_client_v_remain": "$ssl_client_v_remain",'
    '"nginx_ssl_client_v_start": "$ssl_client_v_start",'
    '"nginx_ssl_client_verify": "$ssl_client_verify",'
    '"nginx_ssl_early_data": "$ssl_early_data",'
    '"nginx_ssl_protocol": "$ssl_protocol",'
    '"nginx_ssl_server_name": "$ssl_server_name",'
    '"nginx_ssl_session_id": "$ssl_session_id",'
    '"nginx_ssl_session_reused": "$ssl_session_reused",'
    '"nginx_status": "$status",'
    '"nginx_tcpinfo_rtt": "$tcpinfo_rtt",'
    '"nginx_tcpinfo_rttvar": "$tcpinfo_rttvar",'
    '"nginx_tcpinfo_snd_cwnd": "$tcpinfo_snd_cwnd",'
    '"nginx_tcpinfo_rcv_space": "$tcpinfo_rcv_space",'
    '"nginx_time_iso8601": "$time_iso8601",'
    '"nginx_time_local": "$time_local",'
    '"nginx_uid_got": "$uid_got",'
    '"nginx_uid_reset": "$uid_reset",'
    '"nginx_uid_set": "$uid_set",'
    '"nginx_upstream_addr": "$upstream_addr",'
    '"nginx_upstream_bytes_received": "$upstream_bytes_received",'
    '"nginx_upstream_bytes_sent": "$upstream_bytes_sent",'
    '"nginx_upstream_bytes_sent": "$upstream_bytes_sent",'
    '"nginx_upstream_cache_status": "$upstream_cache_status",'
    '"nginx_upstream_connect_time": "$upstream_connect_time",'
    '"nginx_upstream_cookie_": "$upstream_cookie_",'
    '"nginx_upstream_header_time": "$upstream_header_time",'
    '"nginx_upstream_http_": "$upstream_http_",'
    '"nginx_upstream_response_length": "$upstream_response_length",'
    '"nginx_upstream_response_time": "$upstream_response_time",'
    '"nginx_upstream_status": "$upstream_status",'
    '"nginx_uri": "$uri",'
    '"nginx_response_body": "$response_body"'
    '}';

    server {
        listen 443 ssl;
        root /var/www/backend;
        server_name elk.domain.tld;
        access_log /var/log/nginx/elk.domain.tld-access.log.ssl nginxlog_json;
        error_log /var/log/nginx/elk.domain.tld-error.log.ssl;
        client_max_body_size 500M;
        keepalive_timeout 0;
        ssl_certificate     ...;
        ssl_certificate_key ...;

        lua_need_request_body on;

        set $response_body "";
        body_filter_by_lua '
            local response_body = string.sub(ngx.arg[1], 1, 1000)
            ngx.ctx.buffered = (ngx.ctx.buffered or "") .. response_body
            if ngx.arg[2] then
                ngx.var.response_body = ngx.ctx.buffered
            end
        ';

        set_by_lua_block $request_headers{
            local h = ngx.req.get_headers()
            local request_headers_all = ""
            for k, v in pairs(h) do
                local rowtext = ""
                rowtext = string.format(" %s='%s' ", k, v)
                request_headers_all = request_headers_all .. rowtext

            end
            return request_headers_all
        }
...

Результат

111

В лог попадает:

tail -1 /var/log/nginx/access.log.ssl   | jq .
{
  "timestamp": "2021-08-08T08:47:59+00:00",
  "remote_addr": "159.224.49.4",
  "body_bytes_sent": 361,
  "request": "POST /internal/bsearch HTTP/1.1",
  "request_method": "POST",
  "request_time": 0.152,
  "response_status": 200,
  "upstream_status": 200,
  "upstream_response_time": 0.028,
  "upstream_connect_time": 0,
  "upstream_header_time": 0.028,
  "upstream_addr": "127.0.0.1:5601",
  "host": "elk.domain.tld",
  "http_x_forwarded_for": "",
  "http_referrer": "https://elk.domain.tld/app/discover",
  "http_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15",
  "http_version": "HTTP/1.1",
  "nginx_access": true
}

Filebeat

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/elk.domain.tld-access.log.ssl
  exclude_files: ['\.gz$']
filebeat.config.modules:
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 1
setup.kibana:
output.logstash:
  hosts: ["elk.domain.tld:5400"]
  ssl.certificate_authorities: ["/etc/elk-certs/elk-ssl.crt"]
  ssl.certificate: "/etc/elk-certs/elk-ssl.crt"
  ssl.key: "/etc/elk-certs/elk-ssl.key"
processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~
logging.level: debug
logging.selectors: ["*"]

В примере мониторится всего один файл (исключение очевидно лишнее)
Результат пересылается на удаленный хост в Logstash Важно Сертификат должен соответствовать хостнейму - тут нельзя указать IP адрес

hosts: ["elk.domain.tld:5400"]

Logstash

Базовый пример без преобразований

Минимальный конфиг - слушать Filebeat на порту 5400 (без сертификата - не будет работать)
Результат - писать в файл без всяких преобразований

input {
    beats {
        port => 5400
        ssl => true
        ssl_certificate_authorities => ["/etc/elk-certs/elk-ssl.crt"]
        ssl_certificate => "/etc/elk-certs/elk-ssl.crt"
        ssl_key => "/etc/elk-certs/elk-ssl.key"
        ssl_verify_mode => "force_peer"
    }
}
output {
    file {
        flush_interval => 5
            gzip => false
            path => "/var/log/logstash/logstash_debug.log"
    }
}

Результат записи в файл не удовлетворительный

root@elk:/var/log/logstash# tail -1 logstash_debug.log  | jq .

Сообщение не было разобрано на поля

{
  "message": "{ \"timestamp\": \"2021-08-08T09:20:47+00:00\", \"remote_addr\": \"61.219.11.151\", \"body_bytes_sent\": 163, \"request\": \"GET / HTTP/1.1\", \"request_method\": \"GET\", \"request_time\": 0.213, \"response_status\": 400, \"upstream_status\": ,\"upstream_response_time\": ,\"upstream_connect_time\": ,\"upstream_header_time\": ,\"upstream_addr\": \"\",\"host\": \"elk.arturhaunt.ninja\",\"http_x_forwarded_for\": \"\",\"http_referrer\": \"\", \"http_user_agent\": \"\", \"http_version\": \"HTTP/1.1\", \"nginx_access\": true }",
  "cloud": {
    "availability_zone": "us-east-1f",
    "region": "us-east-1",
    "instance": {
      "id": "i-075866580c26fd42c"
    },
    "service": {
      "name": "EC2"
    },
    "machine": {
      "type": "t3.large"
    },
    "image": {
      "id": "ami-09e67e426f25ce0d7"
    },
    "account": {
      "id": "543591064633"
    },
    "provider": "aws"
  },
  "@timestamp": "2021-08-08T09:20:52.684Z",
  "agent": {
    "hostname": "elk.domain.tld",
    "id": "aef24fda-f14c-40cf-a388-fa0af443f5d7",
    "type": "filebeat",
    "version": "7.14.0",
    "ephemeral_id": "c53f02a8-0b37-4dae-a3c7-cbf03eabca6a",
    "name": "elk.arturhaunt.ninja"
  },
  "@version": "1",
  "log": {
    "offset": 4126,
    "file": {
      "path": "/var/log/nginx/elk.domain.tld-access.log.ssl"
    }
  },
  "host": {
    "hostname": "elk.domain.tld",
    "id": "ec2385ad84aeea5eb425460c41a5b866",
    "os": {
      "type": "linux",
      "codename": "focal",
      "name": "Ubuntu",
      "version": "20.04.2 LTS (Focal Fossa)",
      "kernel": "5.4.0-1045-aws",
      "platform": "ubuntu",
      "family": "debian"
    },
    "ip": [
      "172.31.91.220",
      "fe80::14b5:eff:feac:ae79"
    ],
    "name": "elk.domain.tld",
    "mac": [
      "16:b5:0e:ac:ae:79"
    ],
    "architecture": "x86_64",
    "containerized": false
  },
  "input": {
    "type": "log"
  },
  "ecs": {
    "version": "1.10.0"
  },
  "tags": [
    "beats_input_codec_plain_applied"
  ]
}

Базовый пример с JSON

Для того что бы разобрать на поля JSON

filter {
  json {
    source => "message"
  }
}

В результате сообщение будет разобрано на поля - но в этом случае в поле host

  "host": {
    "hostname": "elk.domain.tld",
    "id": "ec2385ad84aeea5eb425460c41a5b866",
    "os": {
      "type": "linux",
      "codename": "focal",
      "name": "Ubuntu",
      "version": "20.04.2 LTS (Focal Fossa)",
      "kernel": "5.4.0-1045-aws",
      "platform": "ubuntu",
      "family": "debian"
    },

будет записан заголовок HOST что не cовсем то что надо - оригинально там содержаться данные о том какой хост отправил логи

Пример с дебагом

Усложняем - добавляем максимальное количесво информации в логи. В том числе тело запроса и тело ответа и все заголовки которые возможно.

Nginx

Пишем все что возможно: https://noname.com.ua/mediawiki/index.php/Nginx_Log_Post

   # Log in JSON Format
    log_format nginxlog_json escape=json
    '{ '
    '"nginx_http_user_agent": "$http_user_agent",'
    '"nginx_ancient_browser": "$ancient_browser",'
    '"nginx_body_bytes_sent": "$body_bytes_sent",'
    '"nginx_bytes_sent": "$bytes_sent",'
    '"nginx_connection": "$connection",'
    '"nginx_connection_requests": "$connection_requests",'
    '"nginx_connections_active": "$connections_active",'
    '"nginx_connections_reading": "$connections_reading",'
    '"nginx_connections_waiting": "$connections_waiting",'
    '"nginx_connections_writing": "$connections_writing",'
    '"nginx_content_length": "$content_length",'
    '"nginx_content_type": "$content_type",'
    '"nginx_cookie_": "$cookie_",'
    '"nginx_document_root": "$document_root",'
    '"nginx_document_uri": "$document_uri",'
    '"nginx_fastcgi_path_info": "$fastcgi_path_info",'
    '"nginx_fastcgi_script_name": "$fastcgi_script_name",'
    '"nginx_host": "$host",'
    '"nginx_hostname": "$hostname",'
    '"nginx_https": "$https",'
    '"nginx_invalid_referer": "$invalid_referer",'
    '"nginx_is_args": "$is_args",'
    '"nginx_limit_conn_status": "$limit_conn_status",'
    '"nginx_limit_rate": "$limit_rate",'
    '"nginx_limit_req_status": "$limit_req_status",'
    '"nginx_modern_browser": "$modern_browser",'
    '"nginx_msec": "$msec",'
    '"nginx_msie": "$msie",'
    '"nginx_nginx_version": "$nginx_version",'
    '"nginx_proxy_add_x_forwarded_for": "$proxy_add_x_forwarded_for",'
    '"nginx_proxy_host": "$proxy_host",'
    '"nginx_proxy_port": "$proxy_port",'
    '"nginx_proxy_protocol_addr": "$proxy_protocol_addr",'
    '"nginx_proxy_protocol_port": "$proxy_protocol_port",'
    '"nginx_proxy_protocol_server_addr": "$proxy_protocol_server_addr",'
    '"nginx_proxy_protocol_server_port": "$proxy_protocol_server_port",'
    '"nginx_query_string": "$query_string",'
    '"nginx_realip_remote_addr": "$realip_remote_addr",'
    '"nginx_realip_remote_port": "$realip_remote_port",'
    '"nginx_remote_addr": "$remote_addr",'
    '"nginx_remote_port": "$remote_port",'
    '"nginx_remote_user": "$remote_user",'
    '"nginx_request": "$request",'
    '"nginx_request_headers": "$request_headers",'
    '"nginx_request_body": "$request_body",'
    '"nginx_request_id": "$request_id",'
    '"nginx_request_length": "$request_length",'
    '"nginx_request_method": "$request_method",'
    '"nginx_request_time": "$request_time",'
    '"nginx_request_uri": "$request_uri",'
    '"nginx_scheme": "$scheme",'
    '"nginx_server_addr": "$server_addr",'
    '"nginx_server_name": "$server_name",'
    '"nginx_server_port": "$server_port",'
    '"nginx_server_port": "$server_port",'
    '"nginx_server_protocol": "$server_protocol",'
    '"nginx_ssl_cipher": "$ssl_cipher",'
    '"nginx_ssl_ciphers": "$ssl_ciphers",'
    '"nginx_ssl_client_cert": "$ssl_client_cert",'
    '"nginx_ssl_client_escaped_cert": "$ssl_client_escaped_cert",'
    '"nginx_ssl_client_fingerprint": "$ssl_client_fingerprint",'
    '"nginx_ssl_client_i_dn": "$ssl_client_i_dn",'
    '"nginx_ssl_client_raw_cert": "$ssl_client_raw_cert",'
    '"nginx_ssl_client_s_dn": "$ssl_client_s_dn",'
    '"nginx_ssl_client_serial": "$ssl_client_serial",'
    '"nginx_ssl_client_v_end": "$ssl_client_v_end",'
    '"nginx_ssl_client_v_remain": "$ssl_client_v_remain",'
    '"nginx_ssl_client_v_start": "$ssl_client_v_start",'
    '"nginx_ssl_client_verify": "$ssl_client_verify",'
    '"nginx_ssl_early_data": "$ssl_early_data",'
    '"nginx_ssl_protocol": "$ssl_protocol",'
    '"nginx_ssl_server_name": "$ssl_server_name",'
    '"nginx_ssl_session_id": "$ssl_session_id",'
    '"nginx_ssl_session_reused": "$ssl_session_reused",'
    '"nginx_status": "$status",'
    '"nginx_tcpinfo_rtt": "$tcpinfo_rtt",'
    '"nginx_tcpinfo_rttvar": "$tcpinfo_rttvar",'
    '"nginx_tcpinfo_snd_cwnd": "$tcpinfo_snd_cwnd",'
    '"nginx_tcpinfo_rcv_space": "$tcpinfo_rcv_space",'
    '"nginx_time_iso8601": "$time_iso8601",'
    '"nginx_time_local": "$time_local",'
    '"nginx_uid_got": "$uid_got",'
    '"nginx_uid_reset": "$uid_reset",'
    '"nginx_uid_set": "$uid_set",'
    '"nginx_upstream_addr": "$upstream_addr",'
    '"nginx_upstream_bytes_received": "$upstream_bytes_received",'
    '"nginx_upstream_bytes_sent": "$upstream_bytes_sent",'
    '"nginx_upstream_bytes_sent": "$upstream_bytes_sent",'
    '"nginx_upstream_cache_status": "$upstream_cache_status",'
    '"nginx_upstream_connect_time": "$upstream_connect_time",'
    '"nginx_upstream_cookie_": "$upstream_cookie_",'
    '"nginx_upstream_header_time": "$upstream_header_time",'
    '"nginx_upstream_http_": "$upstream_http_",'
    '"nginx_upstream_response_length": "$upstream_response_length",'
    '"nginx_upstream_response_time": "$upstream_response_time",'
    '"nginx_upstream_status": "$upstream_status",'
    '"nginx_uri": "$uri",'
    '"nginx_response_body": "$response_body"'
    '}';

    server {
        listen 443 ssl;
        root /var/www/backend;
        server_name elk.domain.tld;
        access_log /var/log/nginx/elk.domain.tld-access.log.ssl nginxlog_json;
        error_log /var/log/nginx/elk.domain.tld-error.log.ssl;
        client_max_body_size 500M;
        keepalive_timeout 0;
        ssl_certificate     ...;
        ssl_certificate_key ...;

        lua_need_request_body on;

        set $response_body "";
        body_filter_by_lua '
            local response_body = string.sub(ngx.arg[1], 1, 1000)
            ngx.ctx.buffered = (ngx.ctx.buffered or "") .. response_body
            if ngx.arg[2] then
                ngx.var.response_body = ngx.ctx.buffered
            end
        ';

        set_by_lua_block $request_headers{
            local h = ngx.req.get_headers()
            local request_headers_all = ""
            for k, v in pairs(h) do
                local rowtext = ""
                rowtext = string.format(" %s='%s' ", k, v)
                request_headers_all = request_headers_all .. rowtext

            end
            return request_headers_all
        }
...

Logstash

filter {
    json {
        source => "message"
    }
    mutate {
        remove_field => [ "message" ]
    }
    json {
        source => "nginx_request_body"
        target => "nginx_request_body_json"
        tag_on_failure => "request_body_is_not_json"
    }

    json {
        source => "nginx_response_body"
        target => "nginx_response_body_json"
        tag_on_failure => "response_body_is_not_json"
    }

    kv {
        source => "nginx_request_headers"
        target => "nginx_request_headers"
    }
}

Разобрать message как JSON и записать на верхний уровень. Это значит что все поля из сообщения станут полями верхнего уровня (и перепишут если такие поля уже были - по этой причине в логи nginx добавлен префикс nginx_

json {
    source => "message"
}

Удалить оригинальное поле

mutate {
    remove_field => [ "message" ]
}

Разобрать (если это возможно) тело запроса и записать в ключ nginx_request_body_json

json {
    source => "nginx_request_body"
    target => "nginx_request_body_json"
    tag_on_failure => "request_body_is_not_json"
}

Разобрать хедеры и перезаписать в то же поле

kv {
   source => "nginx_request_headers"
   target => "nginx_request_headers"
}

Хедеры в логе выглядят так

"nginx_request_headers": " host='elk.arturhaunt.ninja'  accept-language='en-gb'  accept-encoding='gzip, deflate, br'  connection='keep-alive'  content-type='application/json'  user-agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'  authorization='Basic a2liYW5hYWRtaW46a2liYW5hYWRtaW4='  kbn-system-request='true' ...

После разбора:

  "nginx_request_headers": {
    "accept": "*/*",
    "connection": "keep-alive",
    "accept-encoding": "gzip, deflate, br",
    "kbn-version": "7.14.0",
    "referer": "https://elk.domain.tld/app/discover",
    "accept-language": "en-gb",
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15",
    "host": "elk.domain.tld",
    "content-type": "application/json",
    "kbn-system-request": "true",
    "authorization": "Basic a2liYW5hYWRtaW46a2liYW5hYWRtaW4="
  },