home *** CD-ROM | disk | FTP | other *** search
/ Maximum CD 2010 November / maximum-cd-2010-11.iso / DiscContents / calibre-0.7.13.msi / file_4149 < prev    next >
Encoding:
Text File  |  2010-06-25  |  7.5 KB  |  160 lines

  1. #!/usr/bin/env  python
  2.  
  3. __license__   = 'GPL v3'
  4. __copyright__ = '2009, Pu Bo <pubo at pubolab.com>'
  5. '''
  6. zaobao.com
  7. '''
  8. from calibre.web.feeds.news import BasicNewsRecipe
  9. from calibre.web.feeds import feeds_from_index
  10.  
  11. class ZAOBAO(BasicNewsRecipe):
  12.     title          = u'\u8054\u5408\u65e9\u62a5\u7f51 zaobao.com'
  13.     __author__     = 'Pu Bo'
  14.     description    = 'News from zaobao.com'
  15.     no_stylesheets = True
  16.     recursions     = 1
  17.     language = 'zh'
  18.     encoding     = 'gbk'
  19. #    multithreaded_fetch = True
  20.  
  21.     keep_only_tags    = [
  22.                         dict(name='td', attrs={'class':'text'}),
  23.                         dict(name='span', attrs={'class':'page'}),
  24.                         dict(name='div', attrs={'id':'content'})
  25.                     ]
  26.  
  27.     remove_tags    = [
  28.                         dict(name='table', attrs={'cellspacing':'9'}),
  29.                         dict(name='fieldset'),
  30.                         dict(name='div', attrs={'width':'30%'}),
  31.                     ]
  32.  
  33.     extra_css      = '\n\
  34.             @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}\n\
  35.             body{font-family: serif1, serif}\n\
  36.             .article_description{font-family: serif1, serif}\n\
  37.             p{font-family: serif1, serif}\n\
  38.             h1 {font-weight: bold; font-size: large;}\n\
  39.             h2 {font-size: large;}\n\
  40.             .title {font-size: large;}\n\
  41.             .article {font-size:medium}\n\
  42.             .navbar {font-size: small}\n\
  43.             .feed{font-size: medium}\n\
  44.             .small{font-size: small;padding-right: 8pt}\n\
  45.             .text{padding-right: 8pt}\n\
  46.             p{text-indent: 0cm}\n\
  47.             div#content{padding-right: 10pt}'
  48.  
  49.     INDEXES           = [
  50.                        (u'\u65b0\u95fb\u56fe\u7247', u'http://www.zaobao.com/photoweb/photoweb_idx.shtml')
  51.                     ]
  52.     MAX_ITEMS_IN_INDEX = 10
  53.  
  54.     DESC_SENSE     = u'\u8054\u5408\u65e9\u62a5\u7f51'
  55.  
  56.     feeds          = [
  57.                         (u'\u5373\u65f6\u62a5\u9053', u'http://realtime.zaobao.com/news.xml'),
  58.                         (u'\u4e2d\u56fd\u65b0\u95fb', u'http://www.zaobao.com/zg/zg.xml'),
  59.                         (u'\u56fd\u9645\u65b0\u95fb', u'http://www.zaobao.com/gj/gj.xml'),
  60.                         (u'\u4e16\u754c\u62a5\u520a\u6587\u8403', u'http://www.zaobao.com/wencui/wencui.xml'),
  61.                         (u'\u4e1c\u5357\u4e9a\u65b0\u95fb', u'http://www.zaobao.com/yx/yx.xml'),
  62.                         (u'\u65b0\u52a0\u5761\u65b0\u95fb', u'http://www.zaobao.com/sp/sp.xml'),
  63.                         (u'\u4eca\u65e5\u89c2\u70b9', u'http://www.zaobao.com/yl/yl.xml'),
  64.                         (u'\u4e2d\u56fd\u8d22\u7ecf', u'http://www.zaobao.com/cz/cz.xml'),
  65.                         (u'\u72ee\u57ce\u8d22\u7ecf', u'http://www.zaobao.com/cs/cs.xml'),
  66.                         (u'\u5168\u7403\u8d22\u7ecf', u'http://www.zaobao.com/cg/cg.xml'),
  67.                         (u'\u65e9\u62a5\u4f53\u80b2', u'http://www.zaobao.com/ty/ty.xml'),
  68.                         (u'\u65e9\u62a5\u526f\u520a', u'http://www.zaobao.com/fk/fk.xml'),
  69.                     ]
  70.  
  71.     def preprocess_html(self, soup):
  72.         for tag in soup.findAll(name='a'):
  73.             if tag.has_key('href'):
  74.                 tag_url = tag['href']
  75.                 if tag_url.find('http://') != -1 and tag_url.find('zaobao.com') == -1:
  76.                     del tag['href']
  77.         return soup
  78.  
  79.     def postprocess_html(self, soup, first):
  80.         for tag in soup.findAll(name=['table', 'tr', 'td']):
  81.             tag.name = 'div'
  82.         return soup
  83.  
  84.     def parse_feeds(self):
  85.         self.log_debug(_('ZAOBAO overrided parse_feeds()'))
  86.         parsed_feeds = BasicNewsRecipe.parse_feeds(self)
  87.  
  88.         for id, obj in enumerate(self.INDEXES):
  89.             title, url = obj
  90.             articles = []
  91.             soup = self.index_to_soup(url)
  92.  
  93.             for i, item in enumerate(soup.findAll('li')):
  94.                 if i >= self.MAX_ITEMS_IN_INDEX:
  95.                     break
  96.                 a = item.find('a')
  97.                 if a and a.has_key('href'):
  98.                     a_url = a['href']
  99.                     a_title = self.tag_to_string(a)
  100.                     date = ''
  101.                     description = ''
  102.                     self.log_debug(_('adding %s at %s')%(a_title,a_url))
  103.                     articles.append({
  104.                                     'title':a_title,
  105.                                     'date':date,
  106.                                     'url':a_url,
  107.                                     'description':description
  108.                                     })
  109.  
  110.             pfeeds = feeds_from_index([(title, articles)], oldest_article=self.oldest_article,
  111.                                      max_articles_per_feed=self.max_articles_per_feed)
  112.  
  113.             self.log_debug(_('adding %s to feed')%(title))
  114.             for feed in pfeeds:
  115.                 self.log_debug(_('adding feed: %s')%(feed.title))
  116.                 feed.description = self.DESC_SENSE
  117.                 parsed_feeds.append(feed)
  118.                 for a, article in enumerate(feed):
  119.                     self.log_debug(_('added article %s from %s')%(article.title, article.url))
  120.                 self.log_debug(_('added feed %s')%(feed.title))
  121.  
  122.         for i, feed in enumerate(parsed_feeds):
  123.             # workaorund a strange problem: Somethimes the xml encoding is not apllied correctly by parse()
  124.             weired_encoding_detected = False
  125.             if not isinstance(feed.description, unicode) and self.encoding and feed.description:
  126.                 self.log_debug(_('Feed %s is not encoded correctly, manually replace it')%(feed.title))
  127.                 feed.description = feed.description.decode(self.encoding, 'replace')
  128.             elif feed.description.find(self.DESC_SENSE) == -1 and self.encoding and feed.description:
  129.                 self.log_debug(_('Feed %s is weired encoded, manually redo all')%(feed.title))
  130.                 feed.description = feed.description.encode('cp1252', 'replace').decode(self.encoding, 'replace')
  131.                 weired_encoding_detected = True
  132.  
  133.             for a, article in enumerate(feed):
  134.                 if not isinstance(article.title, unicode) and self.encoding:
  135.                     article.title = article.title.decode(self.encoding, 'replace')
  136.                 if not isinstance(article.summary, unicode) and self.encoding and article.summary:
  137.                     article.summary = article.summary.decode(self.encoding, 'replace')
  138.                     article.text_summary = article.summary
  139.                 if not isinstance(article.text_summary, unicode) and self.encoding and article.text_summary:
  140.                     article.text_summary = article.text_summary.decode(self.encoding, 'replace')
  141.                     article.summary = article.text_summary
  142.                 if weired_encoding_detected:
  143.                     if article.title:
  144.                         article.title = article.title.encode('cp1252', 'replace').decode(self.encoding, 'replace')
  145.                     if article.summary:
  146.                         article.summary = article.summary.encode('cp1252', 'replace').decode(self.encoding, 'replace')
  147.                     if article.text_summary:
  148.                         article.text_summary = article.text_summary.encode('cp1252', 'replace').decode(self.encoding, 'replace')
  149.  
  150.                 if article.title == "Untitled article":
  151.                     self.log_debug(_('Removing empty article %s from %s')%(article.title, article.url))
  152.                     # remove the article
  153.                     feed.articles[a:a+1] = []
  154.         return parsed_feeds
  155.  
  156.     def get_browser(self):
  157.         br = BasicNewsRecipe.get_browser()
  158.         br.addheaders.append(('Pragma', 'no-cache'))
  159.         return br
  160.