I'm attempting to make my first recipe for this but I'm stuck.
I've used
The Atlantic recipe as a template and below is my code so far.
The "for tag in col.findAll" line is where I think I'm stuck, I can't get it to run the For loop even once no matter what I put in. I printed out "col" and tried all different combinations of "class" names I saw in the output (throwing everything in by the end) but the "Print "START OF TAG"" never gets triggered and I get the error "ValueError: No articles found, aborting".
Any help to proceed would be appreciated.
Code:
#!/usr/bin/env python2
from __future__ import unicode_literals
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
'''
technologyreview.com
'''
import re
from calibre.web.feeds.news import BasicNewsRecipe
class TheAtlantic(BasicNewsRecipe):
title = 'MIT Technology Review'
__author__ = ''
description = ''
INDEX = 'http://www.technologyreview.com/magazine/'
language = 'en'
encoding = 'utf-8'
"""keep_only_tags = [
{'attrs':{'class':['article-header', 'article-body', 'article-magazine']}},
]
remove_tags = [
{'name': ['meta', 'link', 'noscript']},
{'attrs':{'class':['offset-wrapper', 'ad-boxfeatures-wrapper']}},
{'attrs':{'class':lambda x: x and 'article-tools' in x}},
{'src':lambda x:x and 'spotxchange.com' in x},
]
no_stylesheets = True
preprocess_regexps = [
(re.compile(r'<script\b.+?</script>', re.DOTALL), lambda m: ''),
(re.compile(r'^.*<html', re.DOTALL|re.IGNORECASE), lambda m: '<html'),
]
def print_version(self, url):
return url + '?single_page=true'"""
def parse_index(self):
soup = self.index_to_soup(self.INDEX)
col = soup.find(attrs={'class':'view-content'})
current_section, current_articles = None, []
feeds = []
print col
print "END OF COL"
for tag in col.findAll(name=['h2', 'li'], attrs={'class':['col', 'content-block', 'wrapper', ' ', 'image', 'content-block in-this-issue no-border']}):
print "START OF TAG"
print tag
if tag.name == 'h2':
if current_section and current_articles:
feeds.append((current_section, current_articles))
current_section = self.tag_to_string(tag).capitalize()
current_articles = []
self.log('Found section:', current_section)
elif current_section:
a = tag.find('a', href=True)
if a is not None:
title, url = self.tag_to_string(a), a['href']
if title and url:
p = tag.find('p', attrs={'class':'river-dek'})
desc = self.tag_to_string(p) if p is not None else ''
current_articles.append({'title':title, 'url':url, 'description':desc})
self.log('\tArticle:', title, '[%s]' % url)
self.log('\t\t', desc)
if current_section and current_articles:
feeds.append((current_section, current_articles))
return feeds