home *** CD-ROM | disk | FTP | other *** search
/ Chip 2011 November / CHIP_2011_11.iso / Programy / Narzedzia / Calibre / calibre-0.8.18.msi / file_280 / guardian.recipe < prev    next >
Encoding:
Text File  |  2011-09-09  |  5.0 KB  |  127 lines
  1. #!/usr/bin/env  python
  2. __license__   = 'GPL v3'
  3. __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
  4. __docformat__ = 'restructuredtext en'
  5.  
  6. '''
  7. www.guardian.co.uk
  8. '''
  9. from calibre import strftime
  10. from calibre.web.feeds.news import BasicNewsRecipe
  11. from datetime import date
  12.  
  13. class Guardian(BasicNewsRecipe):
  14.  
  15.     title = u'The Guardian and The Observer'
  16.     if date.today().weekday() == 6:
  17.         base_url = "http://www.guardian.co.uk/theobserver"
  18.     else:
  19.         base_url = "http://www.guardian.co.uk/theguardian"
  20.  
  21.     __author__ = 'Seabound and Sujata Raman'
  22.     language = 'en_GB'
  23.  
  24.     oldest_article = 7
  25.     max_articles_per_feed = 100
  26.     remove_javascript = True
  27.  
  28.     # List of section titles to ignore
  29.     # For example: ['Sport']
  30.     ignore_sections = []
  31.  
  32.     timefmt = ' [%a, %d %b %Y]'
  33.     keep_only_tags = [
  34.                       dict(name='div', attrs={'id':["content","article_header","main-article-info",]}),
  35.                            ]
  36.     remove_tags = [
  37.                         dict(name='div', attrs={'class':["video-content","videos-third-column"]}),
  38.                         dict(name='div', attrs={'id':["article-toolbox","subscribe-feeds",]}),
  39.                         dict(name='div', attrs={'class':["guardian-tickets promo-component",]}),
  40.                         dict(name='ul', attrs={'class':["pagination"]}),
  41.                         dict(name='ul', attrs={'id':["content-actions"]}),
  42.                         #dict(name='img'),
  43.                         ]
  44.     use_embedded_content    = False
  45.  
  46.     no_stylesheets = True
  47.     extra_css = '''
  48.                     .article-attributes{font-size: x-small; font-family:Arial,Helvetica,sans-serif;}
  49.                     .h1{font-size: large ;font-family:georgia,serif; font-weight:bold;}
  50.                     .stand-first-alone{color:#666666; font-size:small; font-family:Arial,Helvetica,sans-serif;}
  51.                     .caption{color:#666666; font-size:x-small; font-family:Arial,Helvetica,sans-serif;}
  52.                     #article-wrapper{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
  53.                     .main-article-info{font-family:Arial,Helvetica,sans-serif;}
  54.                     #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
  55.                     #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
  56.                 '''
  57.  
  58.     def get_article_url(self, article):
  59.           url = article.get('guid', None)
  60.           if '/video/' in url or '/flyer/' in url or '/quiz/' in url or \
  61.               '/gallery/' in url  or 'ivebeenthere' in url or \
  62.               'pickthescore' in url or 'audioslideshow' in url :
  63.               url = None
  64.           return url
  65.  
  66.     def preprocess_html(self, soup):
  67.  
  68.           for item in soup.findAll(style=True):
  69.               del item['style']
  70.  
  71.           for item in soup.findAll(face=True):
  72.               del item['face']
  73.           for tag in soup.findAll(name=['ul','li']):
  74.                 tag.name = 'div'
  75.  
  76.           return soup
  77.  
  78.     def find_sections(self):
  79.         # soup = self.index_to_soup("http://www.guardian.co.uk/theobserver")
  80.         soup = self.index_to_soup(self.base_url)
  81.         # find cover pic
  82.         img = soup.find( 'img',attrs ={'alt':'Guardian digital edition'})
  83.         if img is not None:
  84.             self.cover_url = img['src']
  85.         # end find cover pic
  86.  
  87.         idx = soup.find('div', id='book-index')
  88.         for s in idx.findAll('strong', attrs={'class':'book'}):
  89.             a = s.find('a', href=True)
  90.             section_title = self.tag_to_string(a)
  91.             if not section_title in self.ignore_sections:
  92.                 prefix = ''
  93.                 if section_title != 'Main section':
  94.                     prefix = section_title + ': '
  95.                 for subsection in s.parent.findAll('a', attrs={'class':'book-section'}):
  96.                     yield (prefix + self.tag_to_string(subsection), subsection['href'])
  97.  
  98.     def find_articles(self, url):
  99.         soup = self.index_to_soup(url)
  100.         div = soup.find('div', attrs={'class':'book-index'})
  101.         for ul in div.findAll('ul', attrs={'class':'trailblock'}):
  102.             for li in ul.findAll('li'):
  103.                 a = li.find(href=True)
  104.                 if not a:
  105.                     continue
  106.                 title = self.tag_to_string(a)
  107.                 url = a['href']
  108.                 if not title or not url:
  109.                     continue
  110.                 tt = li.find('div', attrs={'class':'trailtext'})
  111.                 if tt is not None:
  112.                     for da in tt.findAll('a'): da.extract()
  113.                     desc = self.tag_to_string(tt).strip()
  114.                 yield {
  115.                         'title': title, 'url':url, 'description':desc,
  116.                         'date' : strftime('%a, %d %b'),
  117.                         }
  118.  
  119.     def parse_index(self):
  120.         try:
  121.             feeds = []
  122.             for title, href in self.find_sections():
  123.                 feeds.append((title, list(self.find_articles(href))))
  124.             return feeds
  125.         except:
  126.             raise NotImplementedError
  127.