home *** CD-ROM | disk | FTP | other *** search
/ Chip 2011 November / CHIP_2011_11.iso / Programy / Narzedzia / Calibre / calibre-0.8.18.msi / file_280 / chr_mon.recipe < prev    next >
Encoding:
Text File  |  2011-09-09  |  6.7 KB  |  135 lines
  1. #!/usr/bin/env  python
  2. __license__     = 'GPL v3'
  3. __author__      = 'Kovid Goyal and Sujata Raman, Lorenzo Vigentini'
  4. __copyright__   = '2009, Kovid Goyal and Sujata Raman'
  5. __version__     = 'v1.02'
  6. __date__        = '10, January 2010'
  7. __description__ = 'Providing context and clarity on national and international news, peoples and cultures'
  8.  
  9. '''csmonitor.com'''
  10.  
  11. import re
  12. from calibre.web.feeds.news import BasicNewsRecipe
  13.  
  14.  
  15. class ChristianScienceMonitor(BasicNewsRecipe):
  16.  
  17.     __author__    = 'Kovid Goyal'
  18.     description   = 'Providing context and clarity on national and international news, peoples and cultures'
  19.  
  20.     cover_url      = 'http://www.csmonitor.com/extension/csm_base/design/csm_design/images/csmlogo_179x46.gif'
  21.     title          = 'Christian Science Monitor'
  22.     publisher      = 'The Christian Science Monitor'
  23.     category       = 'News, politics, culture, economy, general interest'
  24.  
  25.     language = 'en'
  26.     encoding = 'utf-8'
  27.     timefmt        = '[%a, %d %b, %Y]'
  28.  
  29.     oldest_article        = 16
  30.     max_articles_per_feed = 20
  31.     use_embedded_content  = False
  32.     recursion             = 10
  33.  
  34.     remove_javascript     = True
  35.     no_stylesheets = True
  36.  
  37.     def append_page(self, soup, appendtag, position):
  38.         nav = soup.find('div',attrs={'class':'navigation'})
  39.         if nav:
  40.             pager = nav.findAll('a')
  41.             for part in pager:
  42.                 if 'Next' in part:
  43.                     nexturl = ('http://www.csmonitor.com' +
  44.                            re.findall(r'href="(.*?)"', str(part))[0])
  45.                     soup2 = self.index_to_soup(nexturl)
  46.                     texttag = soup2.find('div',
  47.                                  attrs={'class': re.compile('list-article-.*')})
  48.                     trash_c = soup2.findAll(attrs={'class': 'list-description'})
  49.                     trash_h = soup2.h1
  50.                     for tc in trash_c: tc.extract()
  51.                     trash_h.extract()
  52.  
  53.                     newpos = len(texttag.contents)
  54.                     self.append_page(soup2, texttag, newpos)
  55.                     texttag.extract()
  56.                     appendtag.insert(position, texttag)
  57.  
  58.     def preprocess_html(self, soup):
  59.         PRINT_RE = re.compile(r'/layout/set/print/content/view/print/[0-9]*')
  60.         html = str(soup)
  61.         try:
  62.             print_found = PRINT_RE.findall(html)
  63.         except Exception:
  64.             pass
  65.         if print_found:
  66.             print_url = 'http://www.csmonitor.com' + print_found[0]
  67.             print_soup = self.index_to_soup(print_url)
  68.         else:
  69.             self.append_page(soup, soup.body, 3)
  70.  
  71.             trash_a = soup.findAll(attrs={'class': re.compile('navigation.*')})
  72.             trash_b = soup.findAll(attrs={'style': re.compile('.*')})
  73.             trash_d = soup.findAll(attrs={'class': 'sByline'})
  74.             for ta in trash_a: ta.extract()
  75.             for tb in trash_b: tb.extract()
  76.             for td in trash_d: td.extract()
  77.  
  78.             print_soup = soup
  79.         return print_soup
  80.  
  81.     preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
  82.         [
  83.             (r'<!--.*?-->', lambda match : ''),
  84.         (r'<body.*?<div id="story"', lambda match : '<body><div id="story"'),
  85.         (r'<div class="pubdate">.*?</div>', lambda m: ''),
  86.         (r'Full HTML version of this story which may include photos, graphics, and related links.*</body>',
  87.               lambda match : '</body>'),
  88.         ]]
  89.     extra_css      = '''
  90.                         h1{ color:#000000;font-family: Georgia,Times,"Times New Roman",serif; font-size: large}
  91.                         .sub{ color:#000000;font-family: Georgia,Times,"Times New Roman",serif; font-size: small;}
  92.                         .byline{ font-family:Arial,Helvetica,sans-serif ; color:#999999; font-size: x-small;}
  93.                         .postdate{color:#999999 ;  font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
  94.                         h3{color:#999999 ;  font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
  95.                         .photoCutline{ color:#333333 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
  96.                         .photoCredit{ color:#999999 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
  97.                         #story{font-family:Arial,Tahoma,Verdana,Helvetica,sans-serif ; font-size: small; }
  98.                         #main{font-family:Arial,Tahoma,Verdana,Helvetica,sans-serif ; font-size: small; }
  99.                         #photo-details{ font-family:Arial,Helvetica,sans-serif ; color:#999999; font-size: x-small;}
  100.                         span.name{color:#205B87;font-family: Georgia,Times,"Times New Roman",serif; font-size: x-small}
  101.                         p#dateline{color:#444444 ;  font-family:Arial,Helvetica,sans-serif ; font-style:italic;} '''
  102.  
  103.     feeds          = [(u'Top Stories', u'http://rss.csmonitor.com/feeds/top'),
  104.                         (u'World' , u'http://rss.csmonitor.com/feeds/world'),
  105.                         (u'USA' , u'http://rss.csmonitor.com/feeds/usa'),
  106.                         (u'Commentary' , u'http://rss.csmonitor.com/feeds/commentary'),
  107.                         (u'Money' , u'http://rss.csmonitor.com/feeds/wam'),
  108.                         (u'Learning' , u'http://rss.csmonitor.com/feeds/learning'),
  109.                         (u'Living', u'http://rss.csmonitor.com/feeds/living'),
  110.                         (u'Innovation', u'http://rss.csmonitor.com/feeds/scitech'),
  111.                         (u'Gardening', u'http://rss.csmonitor.com/feeds/gardening'),
  112.                         (u'Environment',u'http://rss.csmonitor.com/feeds/environment'),
  113.                         (u'Arts', u'http://rss.csmonitor.com/feeds/arts'),
  114.                         (u'Books', u'http://rss.csmonitor.com/feeds/books'),
  115.                         (u'Home Forum' , u'http://rss.csmonitor.com/feeds/homeforum')
  116.                      ]
  117.  
  118.     keep_only_tags = [dict(name='div', attrs={'id':'mainColumn'}), ]
  119.  
  120.     remove_tags    = [
  121.                         dict(name='div', attrs={'id':['story-tools','videoPlayer','storyRelatedBottom','enlarge-photo','photo-paginate']}),
  122.                         dict(name=['div','a'], attrs={'class':
  123.                             ['storyToolbar cfx','podStoryRel','spacer3',
  124.                                 'divvy spacer7','comment','storyIncludeBottom',
  125.                                 'hide', 'podBrdr']}),
  126.                         dict(name='ul', attrs={'class':[ 'centerliststories']}) ,
  127.                         dict(name='form', attrs={'id':[ 'commentform']}) ,
  128.           dict(name='div', attrs={'class': ['ui-comments']})
  129.                     ]
  130.  
  131.     remove_tags_after = [ dict(name='div', attrs={'class':[ 'ad csmAd']}),
  132.               dict(name='div', attrs={'class': [re.compile('navigation.*')]}),
  133.               dict(name='div', attrs={'style': [re.compile('.*')]})
  134.                         ]
  135.