Update: fixed missing articles, better processing of article categories, minor bug fixes.
Code:
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
from collections import OrderedDict
class Smithsonian(BasicNewsRecipe):
title = 'Smithsonian Magazine'
__author__ = 'Rick Shang'
description = 'This magazine chronicles the arts, environment, sciences and popular culture of the times. It is edited for modern, well-rounded individuals with diverse, general interests. With your order, you become a National Associate Member of the Smithsonian. Membership benefits include your subscription to Smithsonian magazine, a personalized membership card, discounts from the Smithsonian catalog, and more.'
language = 'en'
category = 'news'
encoding = 'UTF-8'
keep_only_tags = [dict(attrs={'id':['articleTitle', 'subHead', 'byLine', 'articleImage', 'article-text']})]
remove_tags = [dict(attrs={'class':['related-articles-inpage', 'viewMorePhotos']})]
no_javascript = True
no_stylesheets = True
def parse_index(self):
#Go to the issue
soup0 = self.index_to_soup('http://www.smithsonianmag.com/issue/archive/')
div = soup0.find('div',attrs={'id':'archives'})
issue = div.find('ul',attrs={'class':'clear-both'})
current_issue_url = issue.find('a', href=True)['href']
soup = self.index_to_soup(current_issue_url)
#Go to the main body
div = soup.find ('div', attrs={'id':'article-body'})
#Find date
date = re.sub('.*\:\W*', "", self.tag_to_string(div.find('h2')).strip())
self.timefmt = u' [%s]'%date
#Find cover
self.cover_url = div.find('img',src=True)['src']
feeds = OrderedDict()
section_title = ''
articles = []
for post in div.findAll('div', attrs={'class':['plainModule', 'departments plainModule']}):
h4=post.find('h3')
if h4 is not None:
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
section_title = self.tag_to_string(h4)
articles = []
self.log('Found section:', section_title)
else:
link=post.find('a',href=True)
article_cat=link.findPrevious('p', attrs={'class':'article-cat'})
url=link['href']+'?c=y&story=fullstory'
description=self.tag_to_string(post.findAll('p')[-1]).strip()
title=self.tag_to_string(link).strip()
if article_cat is not None:
title += u' (%s)'%self.tag_to_string(article_cat).strip()
self.log('\tFound article:', title)
articles.append({'title':title, 'url':url, 'description':description, 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
articles = []
ans = [(key, val) for key, val in feeds.iteritems()]
return ans