home *** CD-ROM | disk | FTP | other *** search
/ Chip 2011 November / CHIP_2011_11.iso / Programy / Narzedzia / Calibre / calibre-0.8.18.msi / file_280 / huffingtonpost.recipe < prev    next >
Encoding:
Text File  |  2011-09-09  |  6.7 KB  |  125 lines
  1. from calibre.web.feeds.news import BasicNewsRecipe
  2. import re
  3.  
  4. class HuffingtonPostRecipe(BasicNewsRecipe):
  5.     __license__  = 'GPL v3'
  6.     __author__ = 'kwetal and Archana Raman'
  7.     language = 'en'
  8.     version = 2
  9.  
  10.     title = u'The Huffington Post'
  11.     publisher = u'huffingtonpost.com'
  12.     category = u'News, Politics'
  13.     description = u'Political Blog'
  14.  
  15.     oldest_article = 1.1
  16.     max_articles_per_feed = 100
  17.     #use_embedded_content = True
  18.  
  19.     encoding = 'utf-8'
  20.     remove_empty_feeds = True
  21.     no_stylesheets = True
  22.     remove_javascript = True
  23.  
  24.     # Feeds from: http://www.huffingtonpost.com/syndication/
  25.     feeds = []
  26.     feeds.append((u'Latest News', u'http://feeds.huffingtonpost.com/huffingtonpost/LatestNews'))
  27.  
  28.     feeds.append((u'Politics', u'http://www.huffingtonpost.com/feeds/verticals/politics/index.xml'))
  29.     #feeds.append((u'Politics: News', u'http://www.huffingtonpost.com/feeds/verticals/politics/news.xml'))
  30.     #feeds.append((u'Politics: Blog', u'http://www.huffingtonpost.com/feeds/verticals/politics/blog.xml'))
  31.  
  32.     feeds.append((u'Media', u'http://www.huffingtonpost.com/feeds/verticals/media/index.xml'))
  33.     #feeds.append((u'Media: News', u'http://www.huffingtonpost.com/feeds/verticals/media/news.xml'))
  34.     #feeds.append((u'Media: Blog', u'http://www.huffingtonpost.com/feeds/verticals/media/blog.xml'))
  35.  
  36.     feeds.append((u'Business', u'http://www.huffingtonpost.com/feeds/verticals/business/index.xml'))
  37.     #feeds.append((u'Business: News', u'http://www.huffingtonpost.com/feeds/verticals/business/news.xml'))
  38.     #feeds.append((u'Business: Blogs', u'http://www.huffingtonpost.com/feeds/verticals/business/blog.xml'))
  39.  
  40.     feeds.append((u'Entertainment', u'http://www.huffingtonpost.com/feeds/verticals/entertainment/index.xml'))
  41.     #feeds.append((u'Entertainment: News', u'http://www.huffingtonpost.com/feeds/verticals/business/news.xml'))
  42.     #feeds.append((u'Entertainment: Blog', u'http://www.huffingtonpost.com/feeds/verticals/entertainment/blog.xml'))
  43.  
  44.     feeds.append((u'Living', u'http://www.huffingtonpost.com/feeds/verticals/living/index.xml'))
  45.     #feeds.append((u'Living: News', u'http://www.huffingtonpost.com/feeds/verticals/living/news.xml'))
  46.     #feeds.append((u'Living: Blog', u'http://www.huffingtonpost.com/feeds/verticals/living/blog.xml'))
  47.  
  48.     feeds.append((u'Style', u'http://www.huffingtonpost.com/feeds/verticals/style/index.xml'))
  49.     #feeds.append((u'Style: News', u'http://www.huffingtonpost.com/feeds/verticals/style/news.xml'))
  50.     #feeds.append((u'Style: Blog', u'http://www.huffingtonpost.com/feeds/verticals/style/blog.xml'))
  51.  
  52.     feeds.append((u'Green', u'http://www.huffingtonpost.com/feeds/verticals/green/index.xml'))
  53.     #feeds.append((u'Green: News', u'http://www.huffingtonpost.com/feeds/verticals/green/news.xml'))
  54.     #feeds.append((u'Green: Blog', u'http://www.huffingtonpost.com/feeds/verticals/green/blog.xml'))
  55.  
  56.     feeds.append((u'Technology', u'http://www.huffingtonpost.com/feeds/verticals/technology/index.xml'))
  57.     #feeds.append((u'Technology: News', u'http://www.huffingtonpost.com/feeds/verticals/technology/news.xml'))
  58.     #feeds.append((u'Technology: Blog', u'http://www.huffingtonpost.com/feeds/verticals/technology/blog.xml'))
  59.  
  60.     feeds.append((u'Comedy', u'http://www.huffingtonpost.com/feeds/verticals/comedy/index.xml'))
  61.     #feeds.append((u'Comedy: News', u'http://www.huffingtonpost.com/feeds/verticals/comedy/news.xml'))
  62.     #feeds.append((u'Comedy: Blog', u'http://www.huffingtonpost.com/feeds/verticals/comedy/blog.xml'))
  63.  
  64.     feeds.append((u'World', u'http://www.huffingtonpost.com/feeds/verticals/world/index.xml'))
  65.     #feeds.append((u'World: News', u'http://www.huffingtonpost.com/feeds/verticals/world/news.xml'))
  66.     #feeds.append((u'World: Blog', u'http://www.huffingtonpost.com/feeds/verticals/world/blog.xml'))
  67.  
  68.     feeds.append((u'Original Reporting', u'http://www.huffingtonpost.com/tag/huffpolitics/feed'))
  69.     #feeds.append((u'Original Posts', u'http://www.huffingtonpost.com/feeds/original_posts/index.xml'))
  70.  
  71.  
  72.     remove_tags = []
  73.     remove_tags.append(dict(name='a', attrs={'href' : re.compile('http://feedads\.g\.doubleclick.net.*')}))
  74.     remove_tags.append(dict(name='div', attrs={'class' : 'feedflare'}))
  75.     remove_tags.append(dict(name='a', attrs={'class' : 'home_pixie'}))
  76.     remove_tags.append(dict(name='div', attrs={'id' : ["top_nav",'threeup_top_wrapper','breaking_news_container',"hp_social_network"]}))
  77.     remove_tags.append(dict(name='img', alt="Connect"))
  78.     remove_tags.append(dict(name='div', attrs={'class' : ['logo']}))    #'share_boxes_box_block_b_wraper',
  79.     remove_tags.append(dict(name='div', attrs={'class' :[ 'read_more with_verticals','chicklets_box_outter_v05','blogger_menu_content','chicklets_bar']}))
  80.     remove_tags.append(dict(name='div', attrs={'class' : ['sidebar_blog_first_design','sidebar_blog_second_design',]}))
  81.     remove_tags.append(dict(name='div', attrs={'class' : ['main_big_news_ontop','login-menu','sidebar_blog_third_design','read_more']}))
  82.  
  83.  
  84.     remove_tags_after = [dict(name='div', attrs={'class' : 'entry_content'}) ]
  85.    # remove_attributes = ['style']
  86.  
  87.  
  88.     extra_css = '''
  89.                     h1{font-family :Arial,Helvetica,sans-serif; font-size:large;}
  90.                     h2{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#000000;}
  91.                     h3{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#000000;}
  92.                     body{font-family:verdana,arial,helvetica,geneva,sans-serif ;}
  93.                     #title_permalink{color:black;font-size:large;}
  94.                     .date{color:#858585;font-family:"Times New Roman",sans-serif;}
  95.                     .comments_datetime v05{color:#696969;}
  96.                     .teaser_permalink{font-style:italic;font-size:xx-small;}
  97.                     .blog_posted_date{color:#696969;font-size:xx-small;font-weight: bold;}
  98.                     '''
  99. #a[href]{color: blue; text-decoration: none; cursor: pointer;}
  100.     def get_article_url(self, article):
  101.         """
  102.             Workaround for Feedparser behaviour. If an item has more than one <link/> element, article.link is empty and
  103.             article.links contains a list of dictionaries.
  104.             Todo: refactor to searching this list to avoid the hardcoded zero-index
  105.         """
  106.         link = article.get('link')
  107.         print("Link:"+link)
  108.         if not link:
  109.             links = article.get('links')
  110.             if links:
  111.                 link = links[0]['href']
  112.                 if not links[0]['href']:
  113.                     link = links[1]['href']
  114.  
  115.         return link
  116.  
  117.     def postprocess_html(self, soup, first_fetch):
  118.         for tag in soup.findAll('div',text = "What's Your Reaction?"):
  119.                 tag.extract()
  120.  
  121.         for tg in soup.findAll('blockquote'):
  122.             tg.extract()
  123.  
  124.         return soup
  125.