Got it! Updated Huffington Post recipe, with less junk showing, and more pure news-like substance:
Code:
from calibre.web.feeds.news import BasicNewsRecipe
import re
class HuffingtonPostRecipe(BasicNewsRecipe):
__license__ = 'GPL v3'
__author__ = 'kwetal and Archana Raman'
language = 'en'
version = 2
title = u'__The Huffington Post'
publisher = u'huffingtonpost.com'
category = u'News, Politics'
description = u'Political Blog'
oldest_article = 1.1
max_articles_per_feed = 100
encoding = 'utf-8'
remove_empty_feeds = True
no_stylesheets = True
remove_javascript = True
# Feeds from: http://www.huffingtonpost.com/syndication/
feeds = []
feeds.append((u'Latest News', u'http://feeds.huffingtonpost.com/huffingtonpost/LatestNews'))
feeds.append((u'Politics', u'http://www.huffingtonpost.com/feeds/verticals/politics/index.xml'))
#feeds.append((u'Politics: News', u'http://www.huffingtonpost.com/feeds/verticals/politics/news.xml'))
#feeds.append((u'Politics: Blog', u'http://www.huffingtonpost.com/feeds/verticals/politics/blog.xml'))
feeds.append((u'Media', u'http://www.huffingtonpost.com/feeds/verticals/media/index.xml'))
#feeds.append((u'Media: News', u'http://www.huffingtonpost.com/feeds/verticals/media/news.xml'))
#feeds.append((u'Media: Blog', u'http://www.huffingtonpost.com/feeds/verticals/media/blog.xml'))
feeds.append((u'Business', u'http://www.huffingtonpost.com/feeds/verticals/business/index.xml'))
#feeds.append((u'Business: News', u'http://www.huffingtonpost.com/feeds/verticals/business/news.xml'))
#feeds.append((u'Business: Blogs', u'http://www.huffingtonpost.com/feeds/verticals/business/blog.xml'))
feeds.append((u'Entertainment', u'http://www.huffingtonpost.com/feeds/verticals/entertainment/index.xml'))
#feeds.append((u'Entertainment: News', u'http://www.huffingtonpost.com/feeds/verticals/business/news.xml'))
#feeds.append((u'Entertainment: Blog', u'http://www.huffingtonpost.com/feeds/verticals/entertainment/blog.xml'))
feeds.append((u'Living', u'http://www.huffingtonpost.com/feeds/verticals/living/index.xml'))
#feeds.append((u'Living: News', u'http://www.huffingtonpost.com/feeds/verticals/living/news.xml'))
#feeds.append((u'Living: Blog', u'http://www.huffingtonpost.com/feeds/verticals/living/blog.xml'))
feeds.append((u'Style', u'http://www.huffingtonpost.com/feeds/verticals/style/index.xml'))
#feeds.append((u'Style: News', u'http://www.huffingtonpost.com/feeds/verticals/style/news.xml'))
#feeds.append((u'Style: Blog', u'http://www.huffingtonpost.com/feeds/verticals/style/blog.xml'))
feeds.append((u'Green', u'http://www.huffingtonpost.com/feeds/verticals/green/index.xml'))
#feeds.append((u'Green: News', u'http://www.huffingtonpost.com/feeds/verticals/green/news.xml'))
#feeds.append((u'Green: Blog', u'http://www.huffingtonpost.com/feeds/verticals/green/blog.xml'))
feeds.append((u'Technology', u'http://www.huffingtonpost.com/feeds/verticals/technology/index.xml'))
#feeds.append((u'Technology: News', u'http://www.huffingtonpost.com/feeds/verticals/technology/news.xml'))
#feeds.append((u'Technology: Blog', u'http://www.huffingtonpost.com/feeds/verticals/technology/blog.xml'))
feeds.append((u'Comedy', u'http://www.huffingtonpost.com/feeds/verticals/comedy/index.xml'))
#feeds.append((u'Comedy: News', u'http://www.huffingtonpost.com/feeds/verticals/comedy/news.xml'))
#feeds.append((u'Comedy: Blog', u'http://www.huffingtonpost.com/feeds/verticals/comedy/blog.xml'))
feeds.append((u'World', u'http://www.huffingtonpost.com/feeds/verticals/world/index.xml'))
#feeds.append((u'World: News', u'http://www.huffingtonpost.com/feeds/verticals/world/news.xml'))
#feeds.append((u'World: Blog', u'http://www.huffingtonpost.com/feeds/verticals/world/blog.xml'))
feeds.append((u'Original Reporting', u'http://www.huffingtonpost.com/tag/huffpolitics/feed'))
#feeds.append((u'Original Posts', u'http://www.huffingtonpost.com/feeds/original_posts/index.xml'))
keep_only_tags = [
dict(name='div', attrs={'id':['blog_title']}),
dict(name='div', attrs={'class':['col entry_right full','col entry_right full wide_format','comments_datetime v05','entry_body_text','float_left fixed_width_author']})]
remove_tags = [
dict(name='div', attrs={'id':['liveblog_heading','liveblog_container','chicklets','sidebar_digg_block']}),
dict(name='div', attrs={'class':lambda x: x and 'facebookvote_v2' in x}),
dict(name='div', attrs={'class':lambda x: x and 'reaction_pannel_v3' in x}),
dict(name='div', attrs={'class':lambda x: x and 'facebookvote_reaction' in x}),
dict(name='div', attrs={'class':['chicklets','chicklets_bar','hidden','liveblog_entry','reaction_pannel_v3','read_more','share_boxes_box_block_b_wraper','sidebar_share_block','sidebarHeader',]}),
dict(name='div', attrs={'class':['facebook-like-box float_left','facebookvote_reaction','facebookvote_v2','liveblog_entry hidden','read_more with_verticals',]}),
dict(name='span', attrs={'class':['get_huffpo','email_huffpo']}),
dict(name='a', attrs={'class':'home_pixie'}),
dict(name=['script', 'noscript', 'style'])]
extra_css = '''
h1{font-family :Arial,Helvetica,sans-serif; font-size:large;}
h2{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#000000;}
h3{font-family :Arial,Helvetica,sans-serif; font-size:medium; color:#000000;}
body{font-family:verdana,arial,helvetica,geneva,sans-serif ;}
.date{color:#858585;font-family:"Times New Roman",sans-serif;}
.comments_datetime v05{color:#696969;}
.teaser_permalink{font-style:italic;font-size:xx-small;}
.blog_posted_date{color:#696969;font-size:xx-small;font-weight: bold;}
'''
def get_article_url(self, article):
"""
Workaround for Feedparser behaviour. If an item has more than one <link/> element, article.link is empty and
article.links contains a list of dictionaries.
Todo: refactor to searching this list to avoid the hardcoded zero-index
"""
link = article.get('link')
print("Link:"+link)
if not link:
links = article.get('links')
if links:
link = links[0]['href']
if not links[0]['href']:
link = links[1]['href']
return link
I'm going to run it for the next few weeks and see if it needs any further tweaking, but the tests I did look pretty good overall. Thanks for the help and patience, Kovid.