home *** CD-ROM | disk | FTP | other *** search
/ Maximum CD 2010 November / maximum-cd-2010-11.iso / DiscContents / calibre-0.7.13.msi / file_3782 < prev    next >
Encoding:
Text File  |  2010-04-21  |  4.8 KB  |  126 lines

  1. #!/usr/bin/env  python
  2. __license__   = 'GPL v3'
  3. __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
  4. __docformat__ = 'restructuredtext en'
  5.  
  6. '''
  7. www.guardian.co.uk
  8. '''
  9. from calibre import strftime
  10. from calibre.web.feeds.news import BasicNewsRecipe
  11.  
  12. class Guardian(BasicNewsRecipe):
  13.  
  14.     title = u'The Guardian'
  15.     __author__ = 'Seabound and Sujata Raman'
  16.     language = 'en_GB'
  17.  
  18.     oldest_article = 7
  19.     max_articles_per_feed = 100
  20.     remove_javascript = True
  21.  
  22.     timefmt = ' [%a, %d %b %Y]'
  23.     keep_only_tags = [
  24.                       dict(name='div', attrs={'id':["content","article_header","main-article-info",]}),
  25.                            ]
  26.     remove_tags = [
  27.                         dict(name='div', attrs={'class':["video-content","videos-third-column"]}),
  28.                         dict(name='div', attrs={'id':["article-toolbox","subscribe-feeds",]}),
  29.                         dict(name='ul', attrs={'class':["pagination"]}),
  30.                         dict(name='ul', attrs={'id':["content-actions"]}),
  31.                         ]
  32.     use_embedded_content    = False
  33.  
  34.     no_stylesheets = True
  35.     extra_css = '''
  36.                     .article-attributes{font-size: x-small; font-family:Arial,Helvetica,sans-serif;}
  37.                     .h1{font-size: large ;font-family:georgia,serif; font-weight:bold;}
  38.                     .stand-first-alone{color:#666666; font-size:small; font-family:Arial,Helvetica,sans-serif;}
  39.                     .caption{color:#666666; font-size:x-small; font-family:Arial,Helvetica,sans-serif;}
  40.                     #article-wrapper{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
  41.                     .main-article-info{font-family:Arial,Helvetica,sans-serif;}
  42.                     #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
  43.                     #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
  44.                 '''
  45.  
  46.     feeds = [
  47.         ('Front Page', 'http://www.guardian.co.uk/rss'),
  48.         ('Business', 'http://www.guardian.co.uk/business/rss'),
  49.         ('Sport', 'http://www.guardian.co.uk/sport/rss'),
  50.         ('Culture', 'http://www.guardian.co.uk/culture/rss'),
  51.         ('Money', 'http://www.guardian.co.uk/money/rss'),
  52.         ('Life & Style', 'http://www.guardian.co.uk/lifeandstyle/rss'),
  53.         ('Travel', 'http://www.guardian.co.uk/travel/rss'),
  54.         ('Environment', 'http://www.guardian.co.uk/environment/rss'),
  55.         ('Comment','http://www.guardian.co.uk/commentisfree/rss'),
  56.         ]
  57.  
  58.     def get_article_url(self, article):
  59.           url = article.get('guid', None)
  60.           if '/video/' in url or '/flyer/' in url or '/quiz/' in url or \
  61.               '/gallery/' in url  or 'ivebeenthere' in url or \
  62.               'pickthescore' in url or 'audioslideshow' in url :
  63.               url = None
  64.           return url
  65.  
  66.     def preprocess_html(self, soup):
  67.  
  68.           for item in soup.findAll(style=True):
  69.               del item['style']
  70.  
  71.           for item in soup.findAll(face=True):
  72.               del item['face']
  73.           for tag in soup.findAll(name=['ul','li']):
  74.                 tag.name = 'div'
  75.  
  76.           return soup
  77.  
  78.     def find_sections(self):
  79.         soup = self.index_to_soup('http://www.guardian.co.uk/theguardian')
  80.         # find cover pic
  81.         img = soup.find( 'img',attrs ={'alt':'Guardian digital edition'})
  82.         if img is not None:
  83.             self.cover_url = img['src']
  84.         # end find cover pic
  85.  
  86.         idx = soup.find('div', id='book-index')
  87.         for s in idx.findAll('strong', attrs={'class':'book'}):
  88.             a = s.find('a', href=True)
  89.             yield (self.tag_to_string(a), a['href'])
  90.  
  91.     def find_articles(self, url):
  92.         soup = self.index_to_soup(url)
  93.         div = soup.find('div', attrs={'class':'book-index'})
  94.         for ul in div.findAll('ul', attrs={'class':'trailblock'}):
  95.             for li in ul.findAll('li'):
  96.                 a = li.find(href=True)
  97.                 if not a:
  98.                     continue
  99.                 title = self.tag_to_string(a)
  100.                 url = a['href']
  101.                 if not title or not url:
  102.                     continue
  103.                 tt = li.find('div', attrs={'class':'trailtext'})
  104.                 if tt is not None:
  105.                     for da in tt.findAll('a'): da.extract()
  106.                     desc = self.tag_to_string(tt).strip()
  107.                 yield {
  108.                         'title': title, 'url':url, 'description':desc,
  109.                         'date' : strftime('%a, %d %b'),
  110.                         }
  111.  
  112.     def parse_index(self):
  113.         try:
  114.             feeds = []
  115.             for title, href in self.find_sections():
  116.                 feeds.append((title, list(self.find_articles(href))))
  117.             return feeds
  118.         except:
  119.             raise NotImplementedError
  120.  
  121.  
  122.     def postprocess_html(self,soup,first):
  123.         return soup.findAll('html')[0]
  124.  
  125.  
  126.