This is my first time trying this out, hopefully I haven't butchered things too badly.
I've removed the comments and cleaned out most of the extraneous stuff using this:
Code:
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup
class SmithsonianMagazine(BasicNewsRecipe):
title = u'Smithsonian Magazine'
language = 'en'
__author__ = 'Krittika Goyal (mod by TerminalVeracity)'
oldest_article = 31#days
max_articles_per_feed = 50
use_embedded_content = False
recursions = 1
cover_url = 'http://sphotos.xx.fbcdn.net/hphotos-snc7/431147_10150602715983253_764313347_n.jpg'
match_regexps = ['&page=[2-9]$']
#preprocess_regexps = [
(re.compile(r'<p style="font-family: Arial, Helvetica, sans-serif; font-weight:bold; color:#000;"> <a href="/subArticleBottomWeb" style="color:#900">Subscribe now</a> for more of Smithsonian\'s coverage on history, science and nature. </p>', re.I|re.DOTALL), lambda match:''),
]
remove_stylesheets = True
remove_tags_after = dict(name='div', attrs={'class':['post','articlePaginationWrapper']})
remove_tags = [
dict(name='iframe'),
dict(name='div', attrs={'class':'article_sidebar_border'}),
dict(name='div', attrs={'id':['article_sidebar_border', 'most-popular_large', 'most-popular-body_large']}),
dict(name='ul', attrs={'class':'cat-breadcrumb col three last'}),
dict(name='div', attrs={'class':'addtoany_share_save_container'}),
dict(name='div', attrs={'class':'meta'}),
dict(name='div', attrs={'class':'social'}),
dict(name='h4', attrs={'id':'related-topics'}),
dict(name='table'),
dict(name='div', attrs={'class':'OUTBRAIN'}),
dict(name='div', attrs={'id':'comment_section'}),
dict(name='div', attrs={'id':'article-related'}),
dict(name='div', attrs={'class':'related-articles-inpage'}),
]
feeds = [
('History and Archeology',
'http://feeds.feedburner.com/smithsonianmag/history-archaeology'),
('People and Places',
'http://feeds.feedburner.com/smithsonianmag/people-places'),
('Science and Nature',
'http://feeds.feedburner.com/smithsonianmag/science-nature'),
('Arts and Culture',
'http://feeds.feedburner.com/smithsonianmag/arts-culture'),
('Travel',
'http://feeds.feedburner.com/smithsonianmag/travel'),
]
def preprocess_html(self, soup):
story = soup.find(name='div', attrs={'id':'article-body'})
soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
body = soup.find(name='body')
body.insert(0, story)
return soup
However, there's one short bit of text that shows up in some articles. It's untagged and I haven't been able to figure out how to remove it via regex:
Code:
<p style="font-family: Arial, Helvetica, sans-serif; font-weight:bold; color:#000;"> <a href="/subArticleBottomMag" style="color:#900">Subscribe now</a> for more of Smithsonian's coverage on history, science and nature. </p>
Any hints on how to get rid of this last bit of cruft?
Also, thanks Kovid for an awesome program!
