The Spectator recipe does not work

[Feirpatak]

Hi there!

I would like to use this recipe below, but it doesn't work:

from calibre.web.feeds.recipes import BasicNewsRecipe

class NYTimes(BasicNewsRecipe):

title = 'Spectator Magazine'
__author__ = 'Krittika Goyal'
description = 'Magazine'
timefmt = ' [%d %b, %Y]'
needs_subscription = False
language = 'en'

no_stylesheets = True
#auto_cleanup = True
#auto_cleanup_keep = '//div[@class="thumbnail"]'

keep_only_tags = dict(name='div', attrs={'id':'content'})
remove_tags = [
dict(name='div', attrs={'id':['disqus_thread']}),
##dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}),
##dict(name='form', attrs={'onsubmit':''}),
#dict(name='section', attrs={'id':['article-quote', 'article-navigation']}),
]

#TO GET ARTICLE TOC
def spec_get_index(self):
return self.index_to_soup('http://www.spectator.co.uk/')

# To parse artice toc
def parse_index(self):
parse_soup = self.index_to_soup('http://www.spectator.co.uk/')

feeds = []
feed_title = 'Spectator Magazine Articles'

articles = []
self.log('Found section:', feed_title)
div = parse_soup.find(attrs={'class':'one-col-tax-widget magazine-list columns-1 post-8 taxonomy-category full-width widget section-widget icit-taxonomical-listings'})
for art in div.findAll(name='h2'):
art_info = art.find(name = 'a')
if art_info is None:
continue
art_title = self.tag_to_string(art_info)
url = art_info.get('href')
self.log.info('\tFound article:', art_title, 'at', url)
article = {'title':art_title, 'url':url, 'date':''}
#au = art.find(attrs={'class':'articleAuthors'})
#if au is not None:
#article['author'] = self.tag_to_string(au)
#desc = art.find(attrs={'class':'hover_text'})
#if desc is not None:
#desc = self.tag_to_string(desc)
#if 'author' in article:
#desc = ' by ' + article['author'] + ' ' +desc
#article['description'] = desc
articles.append(article)
if articles:
feeds.append((feed_title, articles))

return feeds

This is what my Calibre told me:

Resolved conversion options
calibre version: 1.48.0
{'asciiize': False,
'author_sort': None,
'authors': None,
'base_font_size': 0,
'book_producer': None,
'change_justification': 'original',
'chapter': None,
'chapter_mark': 'pagebreak',
'comments': None,
'cover': None,
'debug_pipeline': None,
'dehyphenate': True,
'delete_blank_paragraphs': True,
'disable_font_rescaling': False,
'dont_compress': False,
'dont_download_recipe': False,
'duplicate_links_in_toc': False,
'embed_all_fonts': False,
'embed_font_family': None,
'enable_heuristics': False,
'expand_css': False,
'extra_css': None,
'extract_to': None,
'filter_css': None,
'fix_indents': True,
'font_size_mapping': None,
'format_scene_breaks': True,
'html_unwrap_factor': 0.4,
'input_encoding': None,
'input_profile': <calibre.customize.profiles.InputProfile object at 0x01DC1470>,
'insert_blank_line': False,
'insert_blank_line_size': 0.5,
'insert_metadata': False,
'isbn': None,
'italicize_common_cases': True,
'keep_ligatures': False,
'language': None,
'level1_toc': None,
'level2_toc': None,
'level3_toc': None,
'line_height': 0,
'linearize_tables': False,
'lrf': False,
'margin_bottom': 5.0,
'margin_left': 5.0,
'margin_right': 5.0,
'margin_top': 5.0,
'markup_chapter_headings': True,
'max_toc_links': 50,
'minimum_line_height': 120.0,
'mobi_file_type': 'old',
'mobi_ignore_margins': False,
'mobi_keep_original_images': False,
'mobi_toc_at_start': False,
'no_chapters_in_toc': False,
'no_inline_navbars': True,
'no_inline_toc': False,
'output_profile': <calibre.customize.profiles.KindlePaperWhiteOutp ut object at 0x01DC1810>,
'page_breaks_before': None,
'personal_doc': '[PDOC]',
'prefer_author_sort': False,
'prefer_metadata_cover': False,
'pretty_print': False,
'pubdate': None,
'publisher': None,
'rating': None,
'read_metadata_from_opf': None,
'remove_fake_margins': True,
'remove_first_image': False,
'remove_paragraph_spacing': False,
'remove_paragraph_spacing_indent_size': 1.5,
'renumber_headings': True,
'replace_scene_breaks': '',
'search_replace': None,
'series': None,
'series_index': None,
'share_not_sync': False,
'smarten_punctuation': False,
'sr1_replace': '',
'sr1_search': '',
'sr2_replace': '',
'sr2_search': '',
'sr3_replace': '',
'sr3_search': '',
'start_reading_at': None,
'subset_embedded_fonts': False,
'tags': None,
'test': False,
'timestamp': None,
'title': None,
'title_sort': None,
'toc_filter': None,
'toc_threshold': 6,
'toc_title': None,
'unsmarten_punctuation': False,
'unwrap_lines': True,
'use_auto_toc': False,
'verbose': 2}
InputFormatPlugin: Recipe Input running
Using custom recipe
Found section: Spectator Magazine Articles
Python function terminated unexpectedly
'NoneType' object has no attribute 'findAll' (Error Code: 1)
Traceback (most recent call last):
File "site.py", line 132, in main
File "site.py", line 109, in run_entry_point
File "site-packages\calibre\utils\ipc\worker.py", line 195, in main
File "site-packages\calibre\gui2\convert\gui_conversion.py", line 25, in gui_convert
File "site-packages\calibre\ebooks\conversion\plumber.py", line 1040, in run
File "site-packages\calibre\customize\conversion.py", line 241, in __call__
File "site-packages\calibre\ebooks\conversion\plugins\recipe_ input.py", line 117, in convert
File "site-packages\calibre\web\feeds\news.py", line 992, in download
File "site-packages\calibre\web\feeds\news.py", line 1157, in build_index
File "<string>", line 38, in parse_index
AttributeError: 'NoneType' object has no attribute 'findAll'

I would be grateful for your help
Best regards
F.

[kovidgoyal]

Looks like the website has changed, I will look into updating the recipe when I have a moment.

[kovidgoyal]

Should be fine now.

[Feirpatak]

It is fine, indeed!
Thank you very much. You are doing a great job.
All the best!

[Feirpatak]

Hello,
there is a problem with this recipe:

PHP Code:

Resolved conversion options
calibre version: 1.48.0
{'asciiize': False,
 'author_sort': None,
 'authors': None,
 'base_font_size': 0,
 'book_producer': None,
 'change_justification': 'original',
 'chapter': None,
 'chapter_mark': 'pagebreak',
 'comments': None,
 'cover': None,
 'debug_pipeline': None,
 'dehyphenate': True,
 'delete_blank_paragraphs': True,
 'disable_font_rescaling': False,
 'dont_compress': False,
 'dont_download_recipe': False,
 'duplicate_links_in_toc': False,
 'embed_all_fonts': False,
 'embed_font_family': None,
 'enable_heuristics': False,
 'expand_css': False,
 'extra_css': None,
 'extract_to': None,
 'filter_css': None,
 'fix_indents': True,
 'font_size_mapping': None,
 'format_scene_breaks': True,
 'html_unwrap_factor': 0.4,
 'input_encoding': None,
 'input_profile': <calibre.customize.profiles.InputProfile object at 0x01DC0470>,
 'insert_blank_line': False,
 'insert_blank_line_size': 0.5,
 'insert_metadata': False,
 'isbn': None,
 'italicize_common_cases': True,
 'keep_ligatures': False,
 'language': None,
 'level1_toc': None,
 'level2_toc': None,
 'level3_toc': None,
 'line_height': 0,
 'linearize_tables': False,
 'lrf': False,
 'margin_bottom': 5.0,
 'margin_left': 5.0,
 'margin_right': 5.0,
 'margin_top': 5.0,
 'markup_chapter_headings': True,
 'max_toc_links': 50,
 'minimum_line_height': 120.0,
 'mobi_file_type': 'old',
 'mobi_ignore_margins': False,
 'mobi_keep_original_images': False,
 'mobi_toc_at_start': False,
 'no_chapters_in_toc': False,
 'no_inline_navbars': True,
 'no_inline_toc': False,
 'output_profile': <calibre.customize.profiles.KindlePaperWhiteOutput object at 0x01DC0810>,
 'page_breaks_before': None,
 'personal_doc': '[PDOC]',
 'prefer_author_sort': False,
 'prefer_metadata_cover': False,
 'pretty_print': False,
 'pubdate': None,
 'publisher': None,
 'rating': None,
 'read_metadata_from_opf': None,
 'remove_fake_margins': True,
 'remove_first_image': False,
 'remove_paragraph_spacing': False,
 'remove_paragraph_spacing_indent_size': 1.5,
 'renumber_headings': True,
 'replace_scene_breaks': '',
 'search_replace': None,
 'series': None,
 'series_index': None,
 'share_not_sync': False,
 'smarten_punctuation': False,
 'sr1_replace': '',
 'sr1_search': '',
 'sr2_replace': '',
 'sr2_search': '',
 'sr3_replace': '',
 'sr3_search': '',
 'start_reading_at': None,
 'subset_embedded_fonts': False,
 'tags': None,
 'test': False,
 'timestamp': None,
 'title': None,
 'title_sort': None,
 'toc_filter': None,
 'toc_threshold': 6,
 'toc_title': None,
 'unsmarten_punctuation': False,
 'unwrap_lines': True,
 'use_auto_toc': False,
 'verbose': 2}
InputFormatPlugin: Recipe Input running
Using custom recipe
Python function terminated unexpectedly
  'NoneType' object has no attribute '__getitem__' (Error Code: 1)
Traceback (most recent call last):
  File "site.py", line 132, in main
  File "site.py", line 109, in run_entry_point
  File "site-packages\calibre\utils\ipc\worker.py", line 195, in main
  File "site-packages\calibre\gui2\convert\gui_conversion.py", line 25, in gui_convert
  File "site-packages\calibre\ebooks\conversion\plumber.py", line 1040, in run
  File "site-packages\calibre\customize\conversion.py", line 241, in __call__
  File "site-packages\calibre\ebooks\conversion\plugins\recipe_input.py", line 117, in convert
  File "site-packages\calibre\web\feeds\news.py", line 992, in download
  File "site-packages\calibre\web\feeds\news.py", line 1157, in build_index
  File "<string>", line 44, in parse_index
TypeError: 'NoneType' object has no attribute '__getitem__' 


The website address has changed: they added "new" at the beginning (http://new.spectator.co.uk/magazine/)

How to fix it?

Best regards

F.

[kovidgoyal]

https://github.com/kovidgoyal/calibr...6f3cf4c5ae804c

[Feirpatak]

Fixed! Many thanks!

F.

[Feirpatak]

Hi there!

There is a problem with The Spectator recipe (the same as above).
I have got this reply from Calibre:

Fetch news from Spectator Magazine
Conversion options changed from defaults:
verbose: 2
output_profile: 'kindle_pw'
Resolved conversion options
calibre version: 2.79.1
{'asciiize': False,
'author_sort': None,
'authors': None,
'base_font_size': 0,
'book_producer': None,
'change_justification': 'original',
'chapter': None,
'chapter_mark': 'pagebreak',
'comments': None,
'cover': None,
'debug_pipeline': None,
'dehyphenate': True,
'delete_blank_paragraphs': True,
'disable_font_rescaling': False,
'dont_compress': False,
'dont_download_recipe': False,
'duplicate_links_in_toc': False,
'embed_all_fonts': False,
'embed_font_family': None,
'enable_heuristics': False,
'expand_css': False,
'extra_css': None,
'extract_to': None,
'filter_css': None,
'fix_indents': True,
'font_size_mapping': None,
'format_scene_breaks': True,
'html_unwrap_factor': 0.4,
'input_encoding': None,
'input_profile': <calibre.customize.profiles.InputProfile object at 0x038233B0>,
'insert_blank_line': False,
'insert_blank_line_size': 0.5,
'insert_metadata': False,
'isbn': None,
'italicize_common_cases': True,
'keep_ligatures': False,
'language': None,
'level1_toc': None,
'level2_toc': None,
'level3_toc': None,
'line_height': 0,
'linearize_tables': False,
'lrf': False,
'margin_bottom': 5.0,
'margin_left': 5.0,
'margin_right': 5.0,
'margin_top': 5.0,
'markup_chapter_headings': True,
'max_toc_links': 50,
'minimum_line_height': 120.0,
'mobi_file_type': 'old',
'mobi_ignore_margins': False,
'mobi_keep_original_images': False,
'mobi_toc_at_start': False,
'no_chapters_in_toc': False,
'no_inline_navbars': True,
'no_inline_toc': False,
'output_profile': <calibre.customize.profiles.KindlePaperWhiteOutp ut object at 0x03823770>,
'page_breaks_before': None,
'personal_doc': '[PDOC]',
'prefer_author_sort': False,
'prefer_metadata_cover': False,
'pretty_print': False,
'pubdate': None,
'publisher': None,
'rating': None,
'read_metadata_from_opf': None,
'remove_fake_margins': True,
'remove_first_image': False,
'remove_paragraph_spacing': False,
'remove_paragraph_spacing_indent_size': 1.5,
'renumber_headings': True,
'replace_scene_breaks': '',
'search_replace': None,
'series': None,
'series_index': None,
'share_not_sync': False,
'smarten_punctuation': False,
'sr1_replace': '',
'sr1_search': '',
'sr2_replace': '',
'sr2_search': '',
'sr3_replace': '',
'sr3_search': '',
'start_reading_at': None,
'subset_embedded_fonts': False,
'tags': None,
'test': False,
'timestamp': None,
'title': None,
'title_sort': None,
'toc_filter': None,
'toc_threshold': 6,
'toc_title': None,
'transform_css_rules': None,
'unsmarten_punctuation': False,
'unwrap_lines': True,
'use_auto_toc': False,
'verbose': 2}
InputFormatPlugin: Recipe Input running
Using custom recipe
Section: Features
Article: Trump really could crush Isis. But what happens next could be worse
Article: How the stress industry is gaslighting Britain
Article: Britain under Corbyn? Just look at Venezuela
Article: An obituary for the Bullingdon Club, by one of its old boys
Article: Famous people don’t need personal publicists. But they all have them
Article: I’ve got Mick Jagger’s lost memoir
Article: Why do people think it’s OK to order me to dance?
Article: Personalised marshmallows, shoes and pork pies? The cult of personalisation has gone too far
Article: Why Auguste Rodin preferred to sculpt women who couldn’t sit still
Section: The Week
Article: The Labour party has turned into a political bed-blocker
Article: Labour third behind Tories and Ukip among working-class voters, says poll
Article: Hippy Neil took over my life... even my mum thought I was veggie like him
Article: Justin Welby is the victim of some very old-fashioned dirty tricks
Article: 1917: War is no excuse for theft
Article: As lettuce grows scarce, who’s growing our fruit and veg
Article: Ignore Sajid Javid — Britain has bags of room for more houses
Section: Columnists
Article: What Vladimir Putin said to the Eton boys
Article: For the sake of Britain’s constitution, will everyone please shut up?
Article: I went to see Disney World — and saw a dying country
Article: In (conditional) defence of John Bercow
Article: One good thing about bank closures: they could give the Co-op a new job
Section: Books
Article: The importance of being frivolous
Article: Simon Callow tries (again) to get inside Wagner’s head
Article: Was Ezra Pound mad?
Article: How many slaves existed in George V’s Britain?
Article: Nothing happens, but everything changes in Sara Baume’s new novel
Article: Althea Altemus: a single mother in the Roaring Twenties
Article: There’s swash and buckle galore in Dumas’s sequel to The Three Musketeers
Article: Victor Hugo speaks volumes for the outcast and the destitute
Article: How LSD saved my marriage — and my life
Article: How to improve bedside manners
Article: Three’s a crowd in James Lasdun’s The Fall Guy
Article: Books and Arts
Section: Arts
Python function terminated unexpectedly
'NoneType' object has no attribute 'find' (Error Code: 1)
Traceback (most recent call last):
File "site.py", line 132, in main
File "site.py", line 109, in run_entry_point
File "site-packages\calibre\utils\ipc\worker.py", line 195, in main
File "site-packages\calibre\gui2\convert\gui_conversion.py", line 26, in gui_convert
File "site-packages\calibre\ebooks\conversion\plumber.py", line 1074, in run
File "site-packages\calibre\customize\conversion.py", line 245, in __call__
File "site-packages\calibre\ebooks\conversion\plugins\recipe_ input.py", line 119, in convert
File "site-packages\calibre\web\feeds\news.py", line 1007, in download
File "site-packages\calibre\web\feeds\news.py", line 1174, in build_index
File "<string>", line 61, in parse_index
File "<string>", line 39, in parse_spec_section
AttributeError: 'NoneType' object has no attribute 'find'

I would be grateful for your help.
Best regards
Feirpatak