Thread: Smithsonian Mag
View Single Post
Old 04-26-2013, 07:53 PM   #2
rainrdx
Connoisseur
rainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy bluerainrdx can differentiate black from dark navy blue
 
Posts: 55
Karma: 13316
Join Date: Jul 2012
Device: iPad
Update: fixed missing articles, better processing of article categories, minor bug fixes.

Code:
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
from collections import OrderedDict

class Smithsonian(BasicNewsRecipe):

    title       = 'Smithsonian Magazine'
    __author__  = 'Rick Shang'

    description = 'This magazine chronicles the arts, environment, sciences and popular culture of the times. It is edited for modern, well-rounded individuals with diverse, general interests. With your order, you become a National Associate Member of the Smithsonian. Membership benefits include your subscription to Smithsonian magazine, a personalized membership card, discounts from the Smithsonian catalog, and more.'
    language = 'en'
    category = 'news'
    encoding = 'UTF-8'
    keep_only_tags = [dict(attrs={'id':['articleTitle', 'subHead', 'byLine', 'articleImage', 'article-text']})]
    remove_tags = [dict(attrs={'class':['related-articles-inpage', 'viewMorePhotos']})]
    no_javascript = True
    no_stylesheets = True

    def parse_index(self):
        #Go to the issue
        soup0 = self.index_to_soup('http://www.smithsonianmag.com/issue/archive/')
        div = soup0.find('div',attrs={'id':'archives'})
        issue = div.find('ul',attrs={'class':'clear-both'})
        current_issue_url = issue.find('a', href=True)['href']
        soup = self.index_to_soup(current_issue_url)

        #Go to the main body
        div = soup.find ('div', attrs={'id':'article-body'})

        #Find date
        date = re.sub('.*\:\W*', "", self.tag_to_string(div.find('h2')).strip())
        self.timefmt = u' [%s]'%date

        #Find cover
        self.cover_url = div.find('img',src=True)['src']

        feeds = OrderedDict()
        section_title = ''
        articles = []
        for post in div.findAll('div', attrs={'class':['plainModule', 'departments plainModule']}):
            h4=post.find('h3')
            if h4 is not None:
                if articles:
                    if section_title not in feeds:
                        feeds[section_title] = []
                    feeds[section_title] += articles
                section_title = self.tag_to_string(h4)
                articles = []
                self.log('Found section:', section_title)
            else:
                link=post.find('a',href=True)
		article_cat=link.findPrevious('p', attrs={'class':'article-cat'})
                url=link['href']+'?c=y&story=fullstory'
                description=self.tag_to_string(post.findAll('p')[-1]).strip()
		title=self.tag_to_string(link).strip()
		if article_cat is not None:
			title += u' (%s)'%self.tag_to_string(article_cat).strip()
                self.log('\tFound article:', title)
                articles.append({'title':title, 'url':url, 'description':description, 'date':''})

        if articles:
	    if section_title not in feeds:
                        feeds[section_title] = []
            feeds[section_title] += articles
	    articles = []
	
        ans = [(key, val) for key, val in feeds.iteritems()]
        return ans
rainrdx is offline   Reply With Quote