home *** CD-ROM | disk | FTP | other *** search
/ Maximum CD 2010 November / maximum-cd-2010-11.iso / DiscContents / calibre-0.7.13.msi / file_3759 < prev    next >
Encoding:
Text File  |  2010-01-03  |  4.6 KB  |  125 lines
  1. from calibre.web.feeds.news import BasicNewsRecipe
  2. import re
  3. from calibre.ptempfile import PersistentTemporaryFile
  4.  
  5. class ForeignAffairsRecipe(BasicNewsRecipe):
  6.     __license__  = 'GPL v3'
  7.     __author__ = 'kwetal'
  8.     language = 'en'
  9.     version = 1
  10.  
  11.     title = u'Foreign Affairs (Subcription or (free) Registration)'
  12.     publisher = u'Council on Foreign Relations'
  13.     category = u'USA, Foreign Affairs'
  14.     description = u'The leading forum for serious discussion of American foreign policy and international affairs.'
  15.  
  16.     no_stylesheets = True
  17.     remove_javascript = True
  18.  
  19.     INDEX = 'http://www.foreignaffairs.com'
  20.  
  21.     remove_tags = []
  22.     remove_tags.append(dict(name = 'base'))
  23.     #remove_tags.append(dict(name = '', attrs = {'': ''}))
  24.  
  25.     remove_tags_before = dict(name = 'h1', attrs = {'class': 'print-title'})
  26.  
  27.     remove_tags_after = dict(name = 'div', attrs = {'class': 'print-footer'})
  28.  
  29.     extra_css = '''
  30.                 body{font-family:verdana,arial,helvetica,geneva,sans-serif;}
  31.                 div.print-footer {font-size: x-small; color: #696969;}
  32.                 '''
  33.  
  34.     conversion_options = {'comments': description, 'tags': category, 'language': 'en',
  35.                           'publisher': publisher}
  36.  
  37.     temp_files = []
  38.     articles_are_obfuscated = True
  39.  
  40.     def get_obfuscated_article(self, url):
  41.         br = self.get_browser()
  42.         br.open(url)
  43.  
  44.         response = br.follow_link(url_regex = r'/print/[0-9]+', nr = 0)
  45.         html = response.read()
  46.  
  47.         self.temp_files.append(PersistentTemporaryFile('_fa.html'))
  48.         self.temp_files[-1].write(html)
  49.         self.temp_files[-1].close()
  50.  
  51.         return self.temp_files[-1].name
  52.  
  53.     def parse_index(self):
  54.         soup = self.index_to_soup('http://www.foreignaffairs.com/magazine')
  55.         articles = []
  56.         answer = []
  57.         content = soup.find('div', attrs = {'class': 'center-wrapper'})
  58.         if content:
  59.             for div in content.findAll('div', attrs = {'class': re.compile(r'view-row\s+views-row-[0-9]+\s+views-row-[odd|even].*')}):
  60.                 tag = div.find('div', attrs = {'class': 'views-field-title'})
  61.                 if tag:
  62.                     a = tag.find('a')
  63.                     if a:
  64.                         title = self.tag_to_string(a)
  65.                         url = self.INDEX + a['href']
  66.  
  67.                         author = self.tag_to_string(div.find('div', attrs = {'class': 'views-field-field-article-display-authors-value'}))
  68.                         tag = div.find('span', attrs = {'class': 'views-field-field-article-summary-value'})
  69.                         # If they ever fix their markup, this will break :-(
  70.                         summary = self.tag_to_string(tag.findNextSibling('p'))
  71.                         description = author  + '<br/>' + summary
  72.  
  73.                         articles.append({'title': title, 'date': None, 'url': url, 'description': description})
  74.                     else:
  75.                         continue
  76.                 else:
  77.                     continue
  78.  
  79.             answer.append(('Magazine', articles))
  80.  
  81.             ul = content.find('ul')
  82.             if ul:
  83.                 articles = []
  84.                 for li in ul.findAll('li'):
  85.                     tag = li.find('div', attrs = {'class': 'views-field-title'})
  86.                     if tag:
  87.                         a = tag.find('a')
  88.                         if a:
  89.                             title = self.tag_to_string(a)
  90.                             url = self.INDEX + a['href']
  91.                             description = ''
  92.                             tag = li.find('div', attrs = {'class': 'views-field-field-article-display-authors-value'})
  93.                             if tag:
  94.                                 description = self.tag_to_string(tag)
  95.  
  96.                             articles.append({'title': title, 'date': None, 'url': url, 'description': description})
  97.                         else:
  98.                             continue
  99.                     else:
  100.                         continue
  101.  
  102.                 answer.append(('Letters to the Editor', articles))
  103.  
  104.         return answer
  105.  
  106.     def preprocess_html(self, soup):
  107.         for img in soup.findAll('img', attrs = {'src': True}):
  108.             if not img['src'].startswith('http://'):
  109.                 img['src'] = self.INDEX + img['src']
  110.  
  111.         return soup
  112.  
  113.     needs_subscription = True
  114.  
  115.     def get_browser(self):
  116.         br = BasicNewsRecipe.get_browser()
  117.         if self.username is not None and self.password is not None:
  118.             br.open('https://www.foreignaffairs.com/user?destination=home')
  119.             br.select_form(nr = 1)
  120.             br['name']   = self.username
  121.             br['pass'] = self.password
  122.             br.submit()
  123.         return br
  124.  
  125.