home *** CD-ROM | disk | FTP | other *** search
/ Maximum CD 2010 November / maximum-cd-2010-11.iso / DiscContents / calibre-0.7.13.msi / file_3614 < prev    next >
Encoding:
Text File  |  2010-03-03  |  4.3 KB  |  100 lines
  1.  
  2. __license__   = 'GPL v3'
  3. __copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
  4. '''
  5. arstechnica.com
  6. '''
  7.  
  8. import re
  9. from calibre.web.feeds.news import BasicNewsRecipe
  10. from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
  11.  
  12. class ArsTechnica2(BasicNewsRecipe):
  13.     title                 = u'Ars Technica'
  14.     language              = 'en'
  15.     __author__            = 'Darko Miletic and Sujata Raman'
  16.     description           = 'The art of technology'
  17.     publisher             = 'Ars Technica'
  18.     category              = 'news, IT, technology'
  19.     oldest_article        = 2
  20.     max_articles_per_feed = 100
  21.     no_stylesheets        = True
  22.     encoding              = 'utf-8'
  23.     use_embedded_content  = False
  24.     extra_css             = ' body {font-family: Arial,Helvetica,sans-serif} .title{text-align: left} .byline{font-weight: bold; line-height: 1em; font-size: 0.625em; text-decoration: none} '
  25.  
  26.     conversion_options = {
  27.                              'comments'  : description
  28.                             ,'tags'      : category
  29.                             ,'language'  : language
  30.                             ,'publisher' : publisher
  31.                          }
  32.  
  33.  
  34.     preprocess_regexps = [
  35.                 (re.compile(r'<div class="news-item-figure', re.DOTALL|re.IGNORECASE),lambda match: '<div class="news-item-figure"')
  36.                ,(re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')
  37.                          ]
  38.  
  39.     keep_only_tags = [dict(name='div', attrs={'id':['story','etc-story']})]
  40.  
  41.     remove_tags = [
  42.                      dict(name=['object','link','embed'])
  43.                     ,dict(name='div', attrs={'class':'read-more-link'})
  44.                   ]
  45.     remove_attributes=['width','height']
  46.  
  47.     feeds = [
  48.               (u'Infinite Loop (Apple content)'        , u'http://feeds.arstechnica.com/arstechnica/apple/'      )
  49.              ,(u'Opposable Thumbs (Gaming content)'    , u'http://feeds.arstechnica.com/arstechnica/gaming/'     )
  50.              ,(u'Gear and Gadgets'                     , u'http://feeds.arstechnica.com/arstechnica/gadgets/'    )
  51.              ,(u'Chipster (Hardware content)'          , u'http://feeds.arstechnica.com/arstechnica/hardware/'   )
  52.              ,(u'Uptime (IT content)'                  , u'http://feeds.arstechnica.com/arstechnica/business/'   )
  53.              ,(u'Open Ended (Open Source content)'     , u'http://feeds.arstechnica.com/arstechnica/open-source/')
  54.              ,(u'One Microsoft Way'                    , u'http://feeds.arstechnica.com/arstechnica/microsoft/'  )
  55.              ,(u'Nobel Intent (Science content)'       , u'http://feeds.arstechnica.com/arstechnica/science/'    )
  56.              ,(u'Law & Disorder (Tech policy content)' , u'http://feeds.arstechnica.com/arstechnica/tech-policy/')
  57.             ]
  58.  
  59.     def append_page(self, soup, appendtag, position):
  60.         pager = soup.find('div',attrs={'class':'pager'})
  61.         if pager:
  62.            for atag in pager.findAll('a',href=True):
  63.                str = self.tag_to_string(atag)
  64.                if str.startswith('Next'):
  65.                   nurl = 'http://arstechnica.com' + atag['href']
  66.                   rawc = self.index_to_soup(nurl,True)
  67.                   soup2 = BeautifulSoup(rawc, fromEncoding=self.encoding)
  68.  
  69.                   readmoretag = soup2.find('div', attrs={'class':'read-more-link'})
  70.                   if readmoretag:
  71.                      readmoretag.extract()
  72.                   texttag = soup2.find('div', attrs={'class':'body'})
  73.                   for it in texttag.findAll(style=True):
  74.                       del it['style']
  75.  
  76.                   newpos = len(texttag.contents)
  77.                   self.append_page(soup2,texttag,newpos)
  78.                   texttag.extract()
  79.                   pager.extract()
  80.                   appendtag.insert(position,texttag)
  81.  
  82.  
  83.     def preprocess_html(self, soup):
  84.         ftag = soup.find('div', attrs={'class':'byline'})
  85.         if ftag:
  86.            brtag = Tag(soup,'br')
  87.            brtag2 = Tag(soup,'br')
  88.            ftag.insert(4,brtag)
  89.            ftag.insert(5,brtag2)
  90.  
  91.         for item in soup.findAll(style=True):
  92.            del item['style']
  93.  
  94.         self.append_page(soup, soup.body, 3)
  95.  
  96.         return soup
  97.  
  98.     def get_article_url(self, article):
  99.         return article.get('guid',  None).rpartition('?')[0]
  100.