New: Roger Ebert Journal

[spedinfargo]

A second Roger Ebert recipe - this a pull of his journal/blog. It is one of the best film blog you'll find on the web (one of the best blogs period) and it has great running conversations - some of the smartest most well-written comments you'll see. It generates a rather large file...

[Updated 2001-11-25 due to a minor change on the web page. Note - this is different than the other Roger Ebert feed I created. This one is his blog]

[spedinfargo]

import re
import urllib2
import time
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup, SoupStrainer

class EbertJournal(BasicNewsRecipe):
    title                 = 'Roger Ebert Journal'
    __author__            = 'Shane Erstad'
    description           = 'Roger Ebert Journal'
    publisher             = 'Chicago Sun Times'
    category              = 'movies'
    oldest_article        = 8
    max_articles_per_feed = 100
    no_stylesheets        = True
    use_embedded_content  = False
    encoding              = 'ISO-8859-1'
    masthead_url          = 'http://rogerebert.suntimes.com/graphics/global/roger.jpg'
    language              = 'en'
    remove_empty_feeds    = False
    PREFIX                  = 'http://blogs.suntimes.com/ebert'
    
    remove_tags_before = dict(id='content')
    remove_tags_after = dict(id='comments-open')

    



    extra_css             = """
                                @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
                                .article_description,body{font-family: Arial,Helvetica,sans1,sans-serif}
                                .color-2{display:block; margin-bottom: 10px; padding: 5px, 10px;
                                border-left: 1px solid #D00000; color: #D00000}
                                img{margin-bottom: 0.8em} """


    conversion_options = {
                          'comment'          : description
                        , 'tags'             : category
                        , 'publisher'        : publisher
                        , 'language'         : language
                        , 'linearize_tables' : True
                        }


    feeds          = [
                        (u'Roger Ebert Journal'   , u'http://blogs.suntimes.com/ebert/' )
                     ]

    preprocess_regexps = [

        (re.compile(r'<span class="vcard author">Roger Ebert</span>', re.DOTALL|re.IGNORECASE),
            lambda m: 'Roger Ebert'),
        
        (re.compile(r'<span class="vcard author">', re.DOTALL|re.IGNORECASE),
            lambda m: '<hr width="80%"><span class="vcard author">'),
        
        (re.compile(r'<blockquote>', re.DOTALL|re.IGNORECASE),
            lambda m: ''),

        (re.compile(r'<a class="a2a_dd".*?</a>', re.DOTALL|re.IGNORECASE),
            lambda m: ''),

        (re.compile(r'<h2 class="comments-open-header">Leave a comment</h2>', re.DOTALL|re.IGNORECASE),
            lambda m: ''),

        (re.compile(r'a title="Reply".*?</a>', re.DOTALL|re.IGNORECASE),
            lambda m: '')
    ]
    

    def parse_index(self):

        totalfeeds = []
        lfeeds = self.get_feeds()
        for feedobj in lfeeds:
            feedtitle, feedurl = feedobj
            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
            articles = []
            soup = self.index_to_soup(feedurl)
            for item in soup.findAll(attrs={'class':['entry-asset asset hentry']}):

                clearimages = item.find(attrs={'class':['mt-image-left']}).replaceWith("")
                bodysection = item.find(attrs={'class':['asset-body']})
                datesection = item.find(attrs={'class':['published']})
                titlesection = item.find(attrs={'class':['asset-name entry-title']})


                self.log(bodysection)

                link = titlesection.find('a')
                url         = link['href']
                title       = self.tag_to_string(link)
                self.log(url)
                self.log(title)
                articles.append({
                                      'title'      :title
                                     ,'date'       :' [' + self.tag_to_string(datesection) + ']'
                                     ,'url'        :url
                                     ,'description':self.tag_to_string(bodysection)
                                    })
            totalfeeds.append((feedtitle, articles))
        return totalfeeds