LogstashExample-nginx-config

Конфигурация тестовых окружений с Nginx для максимально подробного логгирования (только значимые части)

Это часть статьи LogstashExample1

nginx

Nginx умеет писать логи в Json (для максимально подробного логгирования требуется поддержка Lua)

• Как логгировать хедеры: https://stackoverflow.com/questions/24380123/how-to-log-all-headers-in-nginx
• Пишем все что возможно: Nginx_Log_Post

config

   # Log in JSON Format
    log_format nginxlog_json escape=json
    '{ '
    '"nginx_http_user_agent": "$http_user_agent",'
    '"nginx_ancient_browser": "$ancient_browser",'
    '"nginx_body_bytes_sent": "$body_bytes_sent",'
    '"nginx_bytes_sent": "$bytes_sent",'
    '"nginx_connection": "$connection",'
    '"nginx_connection_requests": "$connection_requests",'
    '"nginx_connections_active": "$connections_active",'
    '"nginx_connections_reading": "$connections_reading",'
    '"nginx_connections_waiting": "$connections_waiting",'
    '"nginx_connections_writing": "$connections_writing",'
    '"nginx_content_length": "$content_length",'
    '"nginx_content_type": "$content_type",'
    '"nginx_cookie_": "$cookie_",'
    '"nginx_document_root": "$document_root",'
    '"nginx_document_uri": "$document_uri",'
    '"nginx_fastcgi_path_info": "$fastcgi_path_info",'
    '"nginx_fastcgi_script_name": "$fastcgi_script_name",'
    '"nginx_host": "$host",'
    '"nginx_hostname": "$hostname",'
    '"nginx_https": "$https",'
    '"nginx_invalid_referer": "$invalid_referer",'
    '"nginx_is_args": "$is_args",'
    '"nginx_limit_conn_status": "$limit_conn_status",'
    '"nginx_limit_rate": "$limit_rate",'
    '"nginx_limit_req_status": "$limit_req_status",'
    '"nginx_modern_browser": "$modern_browser",'
    '"nginx_msec": "$msec",'
    '"nginx_msie": "$msie",'
    '"nginx_nginx_version": "$nginx_version",'
    '"nginx_proxy_add_x_forwarded_for": "$proxy_add_x_forwarded_for",'
    '"nginx_proxy_host": "$proxy_host",'
    '"nginx_proxy_port": "$proxy_port",'
    '"nginx_proxy_protocol_addr": "$proxy_protocol_addr",'
    '"nginx_proxy_protocol_port": "$proxy_protocol_port",'
    '"nginx_proxy_protocol_server_addr": "$proxy_protocol_server_addr",'
    '"nginx_proxy_protocol_server_port": "$proxy_protocol_server_port",'
    '"nginx_query_string": "$query_string",'
    '"nginx_realip_remote_addr": "$realip_remote_addr",'
    '"nginx_realip_remote_port": "$realip_remote_port",'
    '"nginx_remote_addr": "$remote_addr",'
    '"nginx_remote_port": "$remote_port",'
    '"nginx_remote_user": "$remote_user",'
    '"nginx_request": "$request",'
    '"nginx_request_headers": "$request_headers",'
    '"nginx_request_body": "$request_body",'
    '"nginx_request_id": "$request_id",'
    '"nginx_request_length": "$request_length",'
    '"nginx_request_method": "$request_method",'
    '"nginx_request_time": "$request_time",'
    '"nginx_request_uri": "$request_uri",'
    '"nginx_scheme": "$scheme",'
    '"nginx_server_addr": "$server_addr",'
    '"nginx_server_name": "$server_name",'
    '"nginx_server_port": "$server_port",'
    '"nginx_server_port": "$server_port",'
    '"nginx_server_protocol": "$server_protocol",'
    '"nginx_ssl_cipher": "$ssl_cipher",'
    '"nginx_ssl_ciphers": "$ssl_ciphers",'
    '"nginx_ssl_client_cert": "$ssl_client_cert",'
    '"nginx_ssl_client_escaped_cert": "$ssl_client_escaped_cert",'
    '"nginx_ssl_client_fingerprint": "$ssl_client_fingerprint",'
    '"nginx_ssl_client_i_dn": "$ssl_client_i_dn",'
    '"nginx_ssl_client_raw_cert": "$ssl_client_raw_cert",'
    '"nginx_ssl_client_s_dn": "$ssl_client_s_dn",'
    '"nginx_ssl_client_serial": "$ssl_client_serial",'
    '"nginx_ssl_client_v_end": "$ssl_client_v_end",'
    '"nginx_ssl_client_v_remain": "$ssl_client_v_remain",'
    '"nginx_ssl_client_v_start": "$ssl_client_v_start",'
    '"nginx_ssl_client_verify": "$ssl_client_verify",'
    '"nginx_ssl_early_data": "$ssl_early_data",'
    '"nginx_ssl_protocol": "$ssl_protocol",'
    '"nginx_ssl_server_name": "$ssl_server_name",'
    '"nginx_ssl_session_id": "$ssl_session_id",'
    '"nginx_ssl_session_reused": "$ssl_session_reused",'
    '"nginx_status": "$status",'
    '"nginx_tcpinfo_rtt": "$tcpinfo_rtt",'
    '"nginx_tcpinfo_rttvar": "$tcpinfo_rttvar",'
    '"nginx_tcpinfo_snd_cwnd": "$tcpinfo_snd_cwnd",'
    '"nginx_tcpinfo_rcv_space": "$tcpinfo_rcv_space",'
    '"nginx_time_iso8601": "$time_iso8601",'
    '"nginx_time_local": "$time_local",'
    '"nginx_uid_got": "$uid_got",'
    '"nginx_uid_reset": "$uid_reset",'
    '"nginx_uid_set": "$uid_set",'
    '"nginx_upstream_addr": "$upstream_addr",'
    '"nginx_upstream_bytes_received": "$upstream_bytes_received",'
    '"nginx_upstream_bytes_sent": "$upstream_bytes_sent",'
    '"nginx_upstream_bytes_sent": "$upstream_bytes_sent",'
    '"nginx_upstream_cache_status": "$upstream_cache_status",'
    '"nginx_upstream_connect_time": "$upstream_connect_time",'
    '"nginx_upstream_cookie_": "$upstream_cookie_",'
    '"nginx_upstream_header_time": "$upstream_header_time",'
    '"nginx_upstream_http_": "$upstream_http_",'
    '"nginx_upstream_response_length": "$upstream_response_length",'
    '"nginx_upstream_response_time": "$upstream_response_time",'
    '"nginx_upstream_status": "$upstream_status",'
    '"nginx_uri": "$uri",'
    '"nginx_response_body": "$response_body"'
    '}';

    server {
        listen 443 ssl;
        root /var/www/backend;
        server_name elk.domain.tld;
        access_log /var/log/nginx/elk.domain.tld-access.log.ssl nginxlog_json;
        error_log /var/log/nginx/elk.domain.tld-error.log.ssl;
        client_max_body_size 500M;
        keepalive_timeout 0;
        ssl_certificate     ...;
        ssl_certificate_key ...;

        lua_need_request_body on;

        set $response_body "";
        body_filter_by_lua '
            local response_body = string.sub(ngx.arg[1], 1, 1000)
            ngx.ctx.buffered = (ngx.ctx.buffered or "") .. response_body
            if ngx.arg[2] then
                ngx.var.response_body = ngx.ctx.buffered
            end
        ';

        set_by_lua_block $request_headers{
            local h = ngx.req.get_headers()
            local request_headers_all = ""
            for k, v in pairs(h) do
                local rowtext = ""
                rowtext = string.format(" %s='%s' ", k, v)
                request_headers_all = request_headers_all .. rowtext

            end
            return request_headers_all
        }
...

Результат

tail -1 access.log | jq .

Пример лога (не все поля)

{
  ...
  "nginx_http_user_agent": "python-requests/2.23.0",
  "nginx_request_headers": " connection='keep-alive'  accept='*/*'  accept-encoding='gzip, deflate'  host='login-anastasiia-env.arturhaunt.com'  user-agent='python-requests/2.23.0' ",
  "nginx_time_iso8601": "2021-08-10T03:09:57+00:00",
  "nginx_time_local": "10/Aug/2021:03:09:57 +0000",
  "nginx_response_body": "<html>\r\n<head><title>301 Moved Permanently</title></head>\r\n<body>\r\n<center><h1>301 Moved Permanently</h1></center>\r\n<hr><center>openresty/1.19.3.2</center>\r\n</body>\r\n</html>\r\n",
  ...
}