用于日志分析(分析日志文件)

应急响应工具-日志分析

rizhi_find.py

#!/usr/bin/env python

# -*- coding:utf-8 -*-

import re

import requests

from retrying import retry

import os

from collections import Counter

# 需要分析的日志名称

# print dir(rizhifenxi_auto)

# 1、打开日志文件,将需要分析的日志保存log目录下

def open_file():

fpath = os.getcwd() + "/log/"

files = os.listdir(fpath)

for f in files:

for f in files: #

if os.path.isfile(fpath + f):

fhandle = open(fpath + f,'r')

aa = fhandle.readlines()

return aa,len(aa)

file,count=open_file()

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko','Accept': 'text/html,application/xhtml+xml,image/jxr,*/*','Accept-Language': 'zh-CN','Connection': 'close'}

# 输出结果保存在result目录下

def save_path():

path = os.getcwd() + "/result/"

if not os.path.exists(path):

os.makedirs(path)

else:

pass

return path

weicha_ip=list()

guowai_ip=set()

ip_address = list()

path = save_path()

yichang_ip = list()

guowai_ip_lianjie = open(path + "guowai_ip_lianjie.txt","w")

# 在进行ip地址归属地查询过程中，可能由于查询归属地接口的原因，造成ip归属地查询失败，查询未成功的ip地址会保存在shibai_ip.txt中

@retry(stop_max_attempt_number=3)

def check_ip(dizhi):

URL='http://ip-api.com/json/'+dizhi+'?lang=zh-CN'

# try:

r = requests.get(URL,timeout=3,headers=headers)

json_data = r.json()

if json_data[u'status'] == 'success':

country = json_data[u'country'].encode('utf-8')

provice=json_data[u'regionName'].encode('utf-8')

city = json_data[u'city'].encode('utf-8')

if country not in "中国":

guowai_ip.add(dizhi)

aa= country,provice,city

# print aa

return aa

# except :

# yichang_ip.append(dizhi)

ip_list=list()

# 1、提取日志中ip地址

def ip_tiqu():

for i in file:

ip=re.findall(r'd+.d+.d+.d+',i.strip())

ip_list.append(ip[0])

return ip_list

# 2、对提取到的ip地址进行频率统计，并判断相关地址归属地

def ip_guishudi():

ip_tiqu()

result = Counter(ip_list)

result1 = dict(result)

res = sorted(result1.items(),key=lambda d: d[1],reverse=True)

for i in range(len(res)):

# print len (res);

ip=res[i][0];

if res[i][1]>20:

try:

bb=check_ip(dizhi=ip);

result= i,ip,bb,res[i][1]

ip_address.append(result)

except:

result = i,'查询失败',res[i][1]

ip_address.append(result)

return ip_address

# 4、根据输入的关键字进行搜索,支持最多两个关键字同时查找

def find_str():

find_result=open(path+"find_result.txt","w")

canshu1,canshu2,dingzhi=raw_input("please input keywords1:").split(',')

# print len(canshu2)

if len(canshu1)>0 and len(canshu2)==0 and len(dingzhi)==0:

for i in file:

if canshu1 in i:

# print i.strip()

find_result.write(i.strip()+'n')

elif len(canshu1)>0 and len(canshu2)>0 and dingzhi is str(1):

for i in file:

if canshu1 in i.strip() and canshu2 in i.strip():

print i.strip()

find_result.write(i.strip() + 'n')

elif len(canshu1)>0 and len(canshu2)>0 and dingzhi is str(2):

for i in file:

if canshu1 in i or canshu2 in i:

# print i.strip()

find_result.write(i.strip() + 'n')

else:

print "type error,please check your input"

find_result.close()

# 5、对查找到的结果进行ip地址提取，将提取结果保存在列表ip1中

fenxi_result=open(path+"find_result.txt","r").readlines()

for i1 in fenxi_result:

ip = re.findall(r'd+.d+.d+.d+',i1.strip())[0]

ip1.add(ip)

return ip1

# 6、查看ip1中的地址列表还访问过哪些url,并将结果保存在log.txt中

def url_tongji():

log=open(path+"log.txt","w")

for ip in find_str():

# print ip

for i in file:

if ip in i.strip() :

if str(200) or str(500) in i.strip():

# print i.strip()

log.write(i.strip()+'n')

# 7、筛选国外ip访问过的链接

def guowai_lianjie():

for i in guowai_ip:

for i in file:

for i1 in guowai_ip:

if i1 in i and i.split(' ')[8] in "200":

print i1.strip

guowai_ip_lianjie.write(i.strip()+'n')

if __name__=='__main__':

ip1=set()

ip_list=list()

# find_str()

url_tongji()

print "请在"+path+"文件夹下查看相关结果"

rizhifenxi.py

#!/usr/bin/env python

# -*- coding:utf-8 -*-

# 对日志中的访问路径进行汇总，根据数量进行排名，并将访问量前20的统计情况输出保存到表格中。

import xlwt

import re

from collections import Counter

import os

import rizhi_find

# _*_coding:utf-8_*_

url_list = list()

bb = list()

# 2、对日志url汇总规则：1、不对以css、txt等下面列表中后缀的url统计；2、通过空格分割，统计日志中的访问路径，存在吗？的路径只选取？前面的路径；3、由于可能存在不标准的日志格式，此处的切割规则也会不同，可以根据实际情况再做修改；

def countfiles():

info,count=rizhi_find.open_file()

for i in info:

if len(i.strip().split(' '))>6 and i.strip().split(' ')[6].endswith(('css','CSS','dae','DAE','eot','EOT','gif','GIF','ico','ICO','jpeg','JPEG','jpg','JPG','js','JS','map','MAP','mp3','MP3','pdf','PDF','png','PNG','svg','SVG','swf','SWF','ttf','TTF','txt','TXT','woff','WOFF')) == False:

path=i.strip().split('"')[1].rstrip(' HTTP/1.1')

# print path.rstrip(' HTTP/1.1')

time=i.strip().split(' ')[3]

# print time

url=path.split('?')[0]

url_list.append(url)

result = Counter(url_list)

result1 = dict(result)

# print result1

res = sorted(result1.items(),reverse=True)

# 3、这里是保存了url访问量前20的数据，也可以在做修改。

for i in range(0,20):

fangwen_url= res[i][0]

bili="%.2f%%" % (round(float(res[i][1])/float(count),2) * 100)

fangwen_cishu=res[i][1]

aa= i,fangwen_url,fangwen_cishu,bili

bb.append(aa)

return bb

# 4、将输出结果保存到url_tongji.xls的表格中

def baocun_url_biaoge():

path=os.getcwd() + "/result/"

if not os.path.exists(path):

os.makedirs(path)

else:

pass

workbook = xlwt.Workbook(encoding='utf-8')

sheet = workbook.add_sheet('url_tongji',cell_overwrite_ok=True)

sheet1 = workbook.add_sheet('ip_tongji',cell_overwrite_ok=True)

head = ['序号','访问次数','访问占比','url'] # sheet表头

head1 = ['序号','ip','ip归属地'] # sheet表头

alignment = xlwt.Alignment()

alignment.horz = xlwt.Alignment.HORZ_CENTER

alignment.vert = xlwt.Alignment.VERT_CENTER

style = xlwt.XFStyle() # Create Style

font = xlwt.Font() # 为样式创建字体

font.name = 'SimSun'

font.bold = False # 黑体

style.font = font

style.alignment = alignment # Add Alignment to Style

for h in range(len(head)):

sheet.write(0,h,head[h],style)

result=countfiles()

sheet.col(0).width = 256 * 5

sheet.col(1).width = 256 * 15

sheet.col(2).width = 256 * 15

sheet.col(3).width = 256 * 50

alignment = xlwt.Alignment()

alignment.horz = xlwt.Alignment.HORZ_CENTER

alignment.vert = xlwt.Alignment.VERT_CENTER

i=1

for a in result:

sheet.write(i,a[0],style)

sheet.write(i,1,a[2],2,a[3],3,a[1],style)

i += 1

# 5、生成ip查询情况统计表,此处只是统计了出现20次以上的ip地址；

for h3 in range(len(head1)):

sheet1.write(0,h3,head1

,style)
result1=rizhi_find.ip_guishudi()
sheet1.col(0).width = 256 * 5
sheet1.col(1).width = 256 * 20
sheet1.col(2).width = 256 * 15
sheet1.col(3).width = 256 * 50
alignment = xlwt.Alignment()
alignment.horz = xlwt.Alignment.HORZ_CENTER
alignment.vert = xlwt.Alignment.VERT_CENTER
h=1
for b in range(len(result1)):
quzhi= result1[b]
# print quzhi
sheet1.write(h,quzhi[0],style)
sheet1.write(h,quzhi[1],quzhi[3],quzhi[2],style)
h += 1
workbook.save(path+'tongji.xls')
print "请在"+path+"文件夹下查看相关结果"
if name=="main":
baocun_url_biaoge()
# countfiles()

[h2]使用手册

一、概述

在系统被入侵后，要想了解下系统被入侵的过程，最好的途径大概就是通过查看日志，对日志进行分析，来还原整个过程的来龙去脉。每次对几百兆的日志进行查看时确实头疼，尤其是对关键字进行搜索时还会出现编辑器卡顿的情况。所以就想着能不能利用脚本去完成一些常规的排查过程，来辅助完成日志分析工作，先简单说下日志分析的思路，此处只是对于web日志进行分析。

二、功能简述

1、根据关键字进行搜索

1) 目的：尝试通过在日志中搜索后门名称、时间等关键字，更快找到有用信息。

2) 使用方法：运行rizhi_find.py,该模块最多支持两个关键字搜索，关键字之间以逗号隔开，

当最后一个关键字为1时，表示对关键字1和关键字2同时进行搜索；

当最后一个关键字为2时，表示搜索满足关键字1或关键字2的日志；

当最后一个关键字为空时，表示只是对关键字1进行搜索；

3) 结果保存：搜索出的结果均会保存在当前目录下的find_result.txt文档中。

如搜索test.php，输入格式test.php,输出如下：