nginx日志每日生成单个文件nginx日志分割是nginx本身自带不需重新编译。直接在server中添加即可。log_format main $remote_addr $remote_user [$time_local] $request $status $body_bytes_sent $http_referer $http_user_agent $http_x_forwarded_for $request_time $upstream_response_time $upstream_addr $upstream_status; #定义一个日志格式main此main可自己随机定义必须与一下main一致 if ($time_iso8601 ~ ^(\d{4})-(\d{2})-(\d{2})) { set $year $1; set $month $2; set $day $3; } #用于日志分割时间年月日 access_log /var/logs/xxxx/access/xxxxx_xx_access_$year-$month-$day.log main; #日志存放位置及生成名称。 main名称与日志格式名称一致。log_format main$remote_addr $remote_user [$time_local] $request $status $body_bytes_sent $http_referer $http_user_agent $http_x_forwarded_for $request_time $upstream_response_time $upstream_addr $upstream_status;# 从 ISO8601 时间提取年-月-日map$time_iso8601$log_date{~^(\d{4})-(\d{2})-(\d{2})$1-$2-$3;}access_log /usr/local/nginx/nginx/logs/access_$log_date.log main;nginx js/css…开启文件缓存location ~ .*\.(js|css|ico|gif|jpg|jpeg|png|bmp|swf)$ { expires 7d; access_log off; proxy_pass http://wwwserv; #如果图片和网站放在一起必须加这个转向 } location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$ location ~匹配的意思以后面结尾的缓存30不记录 {undefined expires 30d; access_log off; } location ~ .*\.(js|css)$ js/css结尾缓存12个小时并且不记录访问日志 {undefined expires 12h; access_log off; }日志分割命令split -l 行数 -b 大小日志分析过滤 $7 “页面索引” $9 状态码 $10 流量 $11 url***访问量*** 统计ip前十 awk {print $1} xaa |sort |uniq -c |sort -nr | head -10 awk {a[$1]}END{for(i in a)print a[i],i|sort -k1 -nr|head -n10} access.log 统计单日 grep 05/Nov/2019 xaa |awk {print $1} |sort |uniq -c |sort -nr | head -10 统计某个特定时间之后awk ($405/Nov/2019){print $1} xaa |sort |uniq -c |sort -nr | head -10 小时单位里ip连接数最多的10个时段awk -vFS[:] {gsub(-.*,,$1);num[$2 $1]}END{for(i in num)print i,num[i]} log_file | sort -n -k 3 -r | head -10 统计访问最多的前10 awk {a[$7]}END{for(i in a)print a[i],i|sort -k1 -nr|head -n10} access.log ***流量*** 统计单日流量 grep 05/Nov/2019 xaa | awk {sum$10}END{print sum} 统计网站流量G) cat access.log |awk {sum$10} END {print sum/1024/1024/1024} ***资源*** 指定资源统计 awk ($7~/\.html$/){print $1 $7 $9} xaa##第七列已html结尾的 将每个IP访问的页面数进行从小到大排序awk {S[$1]} END {for (a in S) print S[a],a} log_file | sort -n ***状态码*** 按状态码 awk ($9200){print $1 $7 $9} xaa 统计状态码 awk {print $9} xaa |sort |uniq -c |sort -nr | head -10 统计访问状态码为404的IP及出现次数 awk {if($9~/404/)a[$1 $9]}END{for(i in a)print i,a[i]} access.lognginx配合时间任务、自动封堵ip shellcrontab -e * * * * * cd /usr/local/nginx/logs/ sh ip_test.sh 每一分钟运行一次#不能把别人IP一直封着吧这里就清除掉了 echo /usr/local/nginx/conf/blockip.conf #前面最开始编写的统计数据功能 ip_list$(awk{print $1} access.log | sort | uniq -cd | awk {if($160)print $0})#判断这个变量是否为空 if test -z$ip_listthen #为空写入 11.log中并重新启动ngnix echo为空 /usr/local/nginx/logs/11.log /usr/local/nginx/sbin/nginx -s reload else #如果不为空 前面加上 deny格式和ip写入blockip.conf中 echodeny$ip_list /usr/local/nginx/conf/blockip.conf #因为前面携带了行数所有我们需要去除掉前面的行数写入后在读取一次 ip_list2$(awk{print $3} /usr/local/nginx/conf/blockip.conf)#最后再把读取出来的值在次写入到blockip.conf中 echodeny$ip_list2; /usr/local/nginx/conf/blockip.conf #重启ngnix /usr/local/nginx/sbin/nginx -s reload #清空之前的日志从最新的开始截取 echo /usr/local/nginx/logs/access.log fi日志json格式log_format json_analytics escapejson { msec: $msec, # request unixtime in seconds with a milliseconds resolution connection: $connection, # connection serial number connection_requests: $connection_requests, # number of requests made in connection pid: $pid, # process pid request_id: $request_id, # the unique request id request_length: $request_length, # request length (including headers and body) remote_addr: $remote_addr, # client IP remote_user: $remote_user, # client HTTP username remote_port: $remote_port, # client port time_local: $time_local, time_iso8601: $time_iso8601, # local time in the ISO 8601 standard format request: $request, # full path no arguments if the request request_uri: $request_uri, # full path and arguments if the request args: $args, # args status: $status, # response status code body_bytes_sent: $body_bytes_sent, # the number of body bytes exclude headers sent to a client bytes_sent: $bytes_sent, # the number of bytes sent to a client http_referer: $http_referer, # HTTP referer http_user_agent: $http_user_agent, # user agent http_x_forwarded_for: $http_x_forwarded_for, # http_x_forwarded_for http_host: $http_host, # the request Host: header server_name: $server_name, # the name of the vhost serving the request request_time: $request_time, # request processing time in seconds with msec resolution upstream: $upstream_addr, # upstream backend server for proxied requests upstream_connect_time: $upstream_connect_time, # upstream handshake time incl. TLS upstream_header_time: $upstream_header_time, # time spent receiving upstream headers upstream_response_time: $upstream_response_time, # time spend receiving upstream body upstream_response_length: $upstream_response_length, # upstream response length upstream_cache_status: $upstream_cache_status, # cache HIT/MISS where applicable ssl_protocol: $ssl_protocol, # TLS protocol ssl_cipher: $ssl_cipher, # TLS cipher scheme: $scheme, # http or https request_method: $request_method, # request method server_protocol: $server_protocol, # request protocol, like HTTP/1.1 or HTTP/2.0 pipe: $pipe, # p if request was pipelined, . otherwise gzip_ratio: $gzip_ratio, http_cf_ray: $http_cf_ray, geoip_country_code: $geoip_country_code };json格式简化版log_format main $remote_addr - $remote_user [$time_local] $request $status $body_bytes_sent $http_referer $http_user_agent $http_x_forwarded_for $request_time $upstream_addr $upstream_response_time; log_format json {timestamp:$time_iso8601, clientip:$remote_addr, clientip2:$proxy_protocol_addr, responsetime:$request_time, upstreamtime:$upstream_response_time, upstreamhost:$upstream_addr, domain:$host, url:$uri, referer:$http_referer, status:$status}; access_log /var/log/nginx/access.log json;
