View Single Post
Old 09-25-2010, 03:19 PM   #10
TonytheBookworm is on a distinguished road
TonytheBookworm's Avatar
Posts: 264
Karma: 62
Join Date: May 2010
Device: kindle 2, kindle 3, Kindle fire
Originally Posted by Starson17 View Post
I believe you, because I saw your problem, too. This is similar, but not the same, as it's not dependent on --test. It's almost like another recipe is running. I'm getting 13 feeds, not 3, They start with "Features," and there is no "Departments" feed.
I have a feeling or recipes are different like Kovid mentioned. Here is the built in recipe I have.

#!/usr/bin/env  python
__license__   = 'GPL v3'
__copyright__ = '2008, Kovid Goyal'
__docformat__ = 'restructuredtext en'

import re
from import BasicNewsRecipe

class ScientificAmerican(BasicNewsRecipe):
    title = u'Scientific American'
    description = u'Popular science. Monthly magazine.'
    __author__ = 'Kovid Goyal'
    language = 'en'
    remove_javascript   = True
    encoding = 'utf-8'

    def print_version(self, url):
        return url + '&print=true'

    def parse_index(self):
        soup = self.index_to_soup('')
        month = self.tag_to_string(soup.find('p',attrs={'id':'articleDek'}))
        self.timefmt = ' [%s]'%(' '.join(month.strip().split()[:2]))
        img = soup.find('img', alt='Scientific American Magazine', src=True)
        if img is not None:
            self.cover_url = img['src']

        feeds = []
        for div in soup.findAll('div', attrs={'class':['primaryCol',
            current_section = None
            for tag in div.findAll(['h2', 'ul']):
                if == 'h2':
                    current_section = self.tag_to_string(tag).strip()
                    self.log('\tFound section:', current_section)
                elif current_section is not None and == 'ul':
                    articles = []
                    for li in tag.findAll('li'):
                        t = li.findAll('a',
                                attrs={'class':lambda x: x != 'thumb'},
                                href=lambda x: x and 'article.cfm' in x)
                        if not t:
                        t = t[-1]
                        title = self.tag_to_string(t)
                        url = t['href']
                        desc = ''
                        p = li.find(attrs={'class':'dek'})
                        if p is not None:
                            desc = self.tag_to_string(p)
                        articles.append({'title':title, 'url':url,
                            'description':desc, 'date':''})
                        self.log('\t\tFound article:', title, '\n\t\tat', url)
                    if articles:
                        feeds.append((current_section, articles))
                    current_section = None
        return feeds

    def postprocess_html(self, soup, first_fetch):
        if soup is not None:
            for span in soup.findAll('span', attrs={'class':'pagination'}):
            if not first_fetch:
                div = soup.find('div', attrs={'class':'headline'})
                if div:

        return soup

    preprocess_regexps = [
        (re.compile(r'Already a Digital subscriber.*Now</a>', re.DOTALL|re.IGNORECASE), lambda match: ''),
        (re.compile(r'If your institution has site license access, enter.*here</a>.', re.DOTALL|re.IGNORECASE), lambda match: ''),
        (re.compile(r'to subscribe to our.*;.*\}', re.DOTALL|re.IGNORECASE), lambda match: ''),
        (re.compile(r'\)\(jQuery\);.*-->', re.DOTALL|re.IGNORECASE), lambda match: ''),
TonytheBookworm is offline   Reply With Quote