BigOven recipe broken

[kenr276]

BigOven recipe has been broken for awhile, here's the error message:


calibre, version 0.7.54
ERROR: Conversion Error: <b>Failed</b>: Fetch news from BigOven

Fetch news from BigOven
Resolved conversion options
calibre version: 0.7.54
{'asciiize': False,
'author_sort': None,
'authors': None,
'base_font_size': 0,
'book_producer': None,
'change_justification': 'original',
'chapter': None,
'chapter_mark': 'pagebreak',
'comments': None,
'cover': None,
'debug_pipeline': None,
'dehyphenate': True,
'delete_blank_paragraphs': True,
'disable_font_rescaling': False,
'dont_compress': False,
'dont_download_recipe': False,
'enable_heuristics': False,
'extra_css': None,
'fix_indents': True,
'font_size_mapping': None,
'format_scene_breaks': True,
'html_unwrap_factor': 0.4,
'input_encoding': None,
'input_profile': <calibre.customize.profiles.InputProfile object at 0x05143350>,
'insert_blank_line': False,
'insert_metadata': False,
'isbn': None,
'italicize_common_cases': True,
'keep_ligatures': False,
'language': None,
'level1_toc': None,
'level2_toc': None,
'level3_toc': None,
'line_height': 0,
'linearize_tables': False,
'lrf': False,
'margin_bottom': 5.0,
'margin_left': 5.0,
'margin_right': 5.0,
'margin_top': 5.0,
'markup_chapter_headings': True,
'max_toc_links': 50,
'minimum_line_height': 120.0,
'mobi_ignore_margins': False,
'no_chapters_in_toc': False,
'no_inline_navbars': True,
'no_inline_toc': False,
'output_profile': <calibre.customize.profiles.KindleOutput object at 0x05143670>,
'page_breaks_before': None,
'password': 'guest',
'personal_doc': '[PDOC]',
'prefer_author_sort': False,
'prefer_metadata_cover': False,
'pretty_print': False,
'pubdate': None,
'publisher': None,
'rating': None,
'read_metadata_from_opf': None,
'remove_fake_margins': True,
'remove_first_image': False,
'remove_paragraph_spacing': False,
'remove_paragraph_spacing_indent_size': 1.5,
'renumber_headings': True,
'replace_scene_breaks': '',
'rescale_images': False,
'series': None,
'series_index': None,
'smarten_punctuation': False,
'sr1_replace': '',
'sr1_search': '',
'sr2_replace': '',
'sr2_search': '',
'sr3_replace': '',
'sr3_search': '',
'tags': None,
'test': False,
'timestamp': None,
'title': None,
'title_sort': None,
'toc_filter': None,
'toc_threshold': 6,
'toc_title': None,
'unwrap_lines': True,
'use_auto_toc': False,
'username': 'guest',
'verbose': 2}
InputFormatPlugin: Recipe Input running
Failed feed: 4 & 5 Star Rated Recipes
Traceback (most recent call last):
File "site-packages\calibre\web\feeds\news.py", line 1315, in parse_feeds
File "site-packages\mechanize-0.2.4-py2.7.egg\mechanize\_mechanize.py", line 203, in open
File "site-packages\mechanize-0.2.4-py2.7.egg\mechanize\_mechanize.py", line 255, in _mech_open
httperror_seek_wrapper: HTTP Error 404: Feed not found error: FeedBurner cannot locate this feed URI.

Python function terminated unexpectedly
No articles found, aborting (Error Code: 1)
Traceback (most recent call last):
File "site.py", line 103, in main
File "site.py", line 85, in run_entry_point
File "site-packages\calibre\utils\ipc\worker.py", line 119, in main
File "site-packages\calibre\gui2\convert\gui_conversion.py", line 25, in gui_convert
File "site-packages\calibre\ebooks\conversion\plumber.py", line 915, in run
File "site-packages\calibre\customize\conversion.py", line 204, in __call__
File "site-packages\calibre\web\feeds\input.py", line 105, in convert
File "site-packages\calibre\web\feeds\news.py", line 735, in download
File "site-packages\calibre\web\feeds\news.py", line 880, in build_index
ValueError: No articles found, aborting

[Starson17]

Originally Posted by kenr276

BigOven recipe has been broken for awhile, here's the error message:

They changed the site. My wife noticed and I recall working on it. I believe I've got a fix for it done, but I must have forgotten to upload it. I'll check my main system this weekend and try to post it here for you and Kovid (when his eye gets better).

[kenr276]

Thank You

[Starson17]

Originally Posted by Starson17

I'll check my main system this weekend and try to post it here for you and Kovid (when his eye gets better).

Try this. There were some errors in the RSS feed, and I thought they'd eventually fix them. I recall that's why I was waiting. They didn't fix them, so I fixed them here.
Try this:

Spoiler:

from calibre.web.feeds.news import BasicNewsRecipe
import re

class BigOven(BasicNewsRecipe):
    title               = 'BigOven'
    __author__          = 'Starson17'
    description         = 'Recipes for the Foodie in us all. Registration is free. A fake username and password just gives smaller photos.'
    language            = 'en'
    category            = 'news, food, recipes, gourmet'
    publisher           = 'Starson17'
    use_embedded_content= False
    no_stylesheets      = True
    oldest_article      = 24
    remove_javascript   = True
    remove_empty_feeds    = True
    cover_url           = 'http://www.software.com/images/products/BigOven%20Logo_177_216.JPG'
    max_articles_per_feed = 30
    needs_subscription = True

    conversion_options = {'linearize_tables'  : True
                        , 'comment'           : description
                        , 'tags'              : category
                        , 'publisher'         : publisher
                        , 'language'          : language
                        }
    
    def get_browser(self):
        br = BasicNewsRecipe.get_browser()
        if self.username is not None and self.password is not None:
            br.open('http://www.bigoven.com/account/login?ReturnUrl=/')
            br.select_form(nr=1)
            br['Email']  = self.username
            br['Password'] = self.password
            br.submit()
        return br

    remove_attributes = ['style', 'font']

    def get_article_url(self, article):
        url = article.get('feedburner_origlink',article.get('link', None))
        front, middle, end = url.partition('comhttp//www.bigoven.com')
        url = front + 'com' + end
        return url

    keep_only_tags = [dict(name='div', attrs={'id':['nosidebar_main']})]

    remove_tags_after = [dict(name='div', attrs={'class':['display-field']})]
    
    remove_tags =  [dict(name='ul', attrs={'class':['tabs']})]
     
    preprocess_regexps = [
        (re.compile(r'Want detailed nutrition information?', re.DOTALL), lambda match: ''),
        (re.compile('\(You could win \$100 in our ', re.DOTALL), lambda match: ''),
         ]
   
    def preprocess_html(self, soup):
        for tag in soup.findAll(name='a', text=re.compile(r'.*View Metric.*', re.DOTALL)):
            tag.parent.parent.extract()
        for tag in soup.findAll(text=re.compile(r'.*Try BigOven Pro for Free.*', re.DOTALL)):
            tag.extract()
        for tag in soup.findAll(text=re.compile(r'.*Add my photo of this recipe.*', re.DOTALL)):
            tag.parent.extract()
        for tag in soup.findAll(name='a', text=re.compile(r'.*photo contest.*', re.DOTALL)):
            tag.parent.extract()
        for tag in soup.findAll(name='a', text='Remove ads'):
            tag.parent.parent.extract()
        for tag in soup.findAll(name='ol', attrs={'class':['recipe-tags']}):
            tag.parent.extract()
        return soup

    feeds = [(u'Recent Raves', u'http://www.bigoven.com/rss/recentraves'),
                   (u'Recipe Of The Day', u'http://feeds.feedburner.com/bigovencom-RecipeOfTheDay')]

If you see anything that needs fixing, let me know. The site has changed significantly, so I may have missed some cleanup. I was showing someone how to write recipes, so this has a variety of methods of removing junk. It may not be the most efficient in all cases, but it works.

If it seems to work for you, let us know, and I'm sure Kovid will fix the builtin when he's feeling better.

[kenr276]

Thank you for your help, that has got it working and it looks good.

[Starson17]

Originally Posted by Starson17

Try this:

Spoiler:

from calibre.web.feeds.news import BasicNewsRecipe
import re

class BigOven(BasicNewsRecipe):
    title               = 'BigOven'
    __author__          = 'Starson17'
    description         = 'Recipes for the Foodie in us all. Registration is free. A fake username and password just gives smaller photos.'
    language            = 'en'
    category            = 'news, food, recipes, gourmet'
    publisher           = 'Starson17'
    use_embedded_content= False
    no_stylesheets      = True
    oldest_article      = 24
    remove_javascript   = True
    remove_empty_feeds    = True
    cover_url           = 'http://www.software.com/images/products/BigOven%20Logo_177_216.JPG'
    max_articles_per_feed = 30
    needs_subscription = True

    conversion_options = {'linearize_tables'  : True
                        , 'comment'           : description
                        , 'tags'              : category
                        , 'publisher'         : publisher
                        , 'language'          : language
                        }
    
    def get_browser(self):
        br = BasicNewsRecipe.get_browser()
        if self.username is not None and self.password is not None:
            br.open('http://www.bigoven.com/account/login?ReturnUrl=/')
            br.select_form(nr=1)
            br['Email']  = self.username
            br['Password'] = self.password
            br.submit()
        return br

    remove_attributes = ['style', 'font']

    def get_article_url(self, article):
        url = article.get('feedburner_origlink',article.get('link', None))
        front, middle, end = url.partition('comhttp//www.bigoven.com')
        url = front + 'com' + end
        return url

    keep_only_tags = [dict(name='div', attrs={'id':['nosidebar_main']})]

    remove_tags_after = [dict(name='div', attrs={'class':['display-field']})]
    
    remove_tags =  [dict(name='ul', attrs={'class':['tabs']})]
     
    preprocess_regexps = [
        (re.compile(r'Want detailed nutrition information?', re.DOTALL), lambda match: ''),
        (re.compile('\(You could win \$100 in our ', re.DOTALL), lambda match: ''),
         ]
   
    def preprocess_html(self, soup):
        for tag in soup.findAll(name='a', text=re.compile(r'.*View Metric.*', re.DOTALL)):
            tag.parent.parent.extract()
        for tag in soup.findAll(text=re.compile(r'.*Try BigOven Pro for Free.*', re.DOTALL)):
            tag.extract()
        for tag in soup.findAll(text=re.compile(r'.*Add my photo of this recipe.*', re.DOTALL)):
            tag.parent.extract()
        for tag in soup.findAll(name='a', text=re.compile(r'.*photo contest.*', re.DOTALL)):
            tag.parent.extract()
        for tag in soup.findAll(name='a', text='Remove ads'):
            tag.parent.parent.extract()
        for tag in soup.findAll(name='ol', attrs={'class':['recipe-tags']}):
            tag.parent.extract()
        return soup

    feeds = [(u'Recent Raves', u'http://www.bigoven.com/rss/recentraves'),
                   (u'Recipe Of The Day', u'http://feeds.feedburner.com/bigovencom-RecipeOfTheDay')]

Kovid, this is ready, too. (The recipe code is unchanged from the post above, so if it's already been updated, ignore this.)