Quote:
Originally Posted by Starson17
I believe you, because I saw your problem, too. This is similar, but not the same, as it's not dependent on --test. It's almost like another recipe is running. I'm getting 13 feeds, not 3, They start with "Features," and there is no "Departments" feed.
|
I have a feeling or recipes are different like Kovid mentioned. Here is the built in recipe I have.
Spoiler:
Code:
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'''
sciam.com
'''
import re
from calibre.web.feeds.news import BasicNewsRecipe
class ScientificAmerican(BasicNewsRecipe):
title = u'Scientific American'
description = u'Popular science. Monthly magazine.'
__author__ = 'Kovid Goyal'
language = 'en'
remove_javascript = True
encoding = 'utf-8'
def print_version(self, url):
return url + '&print=true'
def parse_index(self):
soup = self.index_to_soup('http://www.scientificamerican.com/sciammag/')
month = self.tag_to_string(soup.find('p',attrs={'id':'articleDek'}))
self.timefmt = ' [%s]'%(' '.join(month.strip().split()[:2]))
img = soup.find('img', alt='Scientific American Magazine', src=True)
if img is not None:
self.cover_url = img['src']
feeds = []
for div in soup.findAll('div', attrs={'class':['primaryCol',
'secondaryCol']}):
current_section = None
for tag in div.findAll(['h2', 'ul']):
if tag.name == 'h2':
current_section = self.tag_to_string(tag).strip()
self.log('\tFound section:', current_section)
elif current_section is not None and tag.name == 'ul':
articles = []
for li in tag.findAll('li'):
t = li.findAll('a',
attrs={'class':lambda x: x != 'thumb'},
href=lambda x: x and 'article.cfm' in x)
if not t:
continue
t = t[-1]
title = self.tag_to_string(t)
url = t['href']
desc = ''
p = li.find(attrs={'class':'dek'})
if p is not None:
desc = self.tag_to_string(p)
articles.append({'title':title, 'url':url,
'description':desc, 'date':''})
self.log('\t\tFound article:', title, '\n\t\tat', url)
if articles:
feeds.append((current_section, articles))
current_section = None
return feeds
def postprocess_html(self, soup, first_fetch):
if soup is not None:
for span in soup.findAll('span', attrs={'class':'pagination'}):
span.extract()
if not first_fetch:
div = soup.find('div', attrs={'class':'headline'})
if div:
div.extract()
return soup
preprocess_regexps = [
(re.compile(r'Already a Digital subscriber.*Now</a>', re.DOTALL|re.IGNORECASE), lambda match: ''),
(re.compile(r'If your institution has site license access, enter.*here</a>.', re.DOTALL|re.IGNORECASE), lambda match: ''),
(re.compile(r'to subscribe to our.*;.*\}', re.DOTALL|re.IGNORECASE), lambda match: ''),
(re.compile(r'\)\(jQuery\);.*-->', re.DOTALL|re.IGNORECASE), lambda match: ''),
]