下面是编程之家 jb51.cc 通过网络收集整理的代码片段。
编程之家小编现在分享给大家,也给大家做个参考。
#!usr/bin/env python # -*- coding:UTF-8 -*- import re from lxml import etree from bs4 import BeautifulSoup as sp import requests import urllib2 import StringIO import sys reload(sys) sys.setdefaultencoding("utf-8") headers={'User-Agent' : 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'} def urlread(url): try: req=requests.get(url,headers=headers) req.encoding="utf-8" return req.text.encode("utf-8") except: req=urllib2.Request(url,headers=headers) response=urllib2.urlopen(req) return response.read().encode("utf-8") class Item: def __init__(self,title,link,date,description): self.title=title.strip() self.link=link.strip() self.pubDate=date.strip() self.decription=self.filter(description).strip() def filter(self,description): description=re.sub("<.*?>",'',description) description=re.sub("\r",description) description=re.sub("\n",description) description=re.sub(" "," ",description) if len(description)>240: description=description[:240]+'...' return description def __str__(self): return "%s\n%s\n%s\n<%s>\n" % ( self.title,self.link,self.decription,self.pubDate ) __repr__=__str__ class BSParser(object): #url='' def __init__(self,url): xml=urlread(url) self.reset(xml) def reset(self,xml=None): if xml==None: self.soup=sp("<xml> </xml>") else: self.soup=sp(xml,"xml") def callback(self,method,obj,tags): rst=None attr=method.lower() for tag in tags: try: rst=getattr(obj,attr)(tag) except: continue if rst: break return rst def getfields(self,tags=["item",'entry']): return self.callback(method="FIND_ALL",obj=self.soup,tags=tags) def gettitle(self,tags=["title"]): return self.callback("FIND",tags).text def getlink(self,tags=["link"]): rst=self.callback("FIND",tags).text if not rst: rst=self.callback("FIND",tags).get("href") return rst def getdate(self,tags=["pubDate","published"]): return self.callback("FIND",tags).text def getdescription(self,tags=["description","content"]): return self.callback("FIND",tags).text def run(self): for item in self.getfields(): title=self.gettitle(item) link=self.getlink(item) date=self.getdate(item) description=self.getdescription(item) newsitem=Item(title,description) yield newsitem def test(): parser=Parser() for item in parser.run(): print item if __name__=="__main__": test()
以上是编程之家(jb51.cc)为你收集整理的全部代码内容,希望文章能够帮你解决所遇到的程序开发问题。
如果觉得编程之家网站内容还不错,欢迎将编程之家网站推荐给程序员好友。
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。