[luis.nando]

We are again with a broken recipe for the Printed Folha Version. Some API changes in get_browser() may be the culprit. It would be nice if @kovidgoyal would help out on this one as I can't figure out how to correct it.

Below is the current broken code:

from calibre.web.feeds.news import BasicNewsRecipe

import re
import datetime

class FSP(BasicNewsRecipe):

    title      = u'Folha de S\xE3o Paulo'
    __author__ = 'Joao Eduardo Bertacchi'
    description = u'Printed edition contents. UOL subscription required.' + \
                  u' [Conte\xfado completo da edi\xe7\xe3o impressa. Somente para assinantes UOL.]'

    today=datetime.date.today()

    masthead_url = 'http://f.i.uol.com.br/fsp/furniture/images/lgo-fsp-430x50-ffffff.gif'

    language = 'pt_BR'
    no_stylesheets = True
    max_articles_per_feed  = 100
    remove_javascript  = True
    needs_subscription = True


    keep_only_tags = [
        dict(name='div', id='articleNew'), dict(name='table', attrs={'class':'articleGraphic'}),
        dict(name='article', id='news'),
    ]

    publication_type = 'newspaper'
    simultaneous_downloads = 5

    remove_attributes = ['height','width']

    # The following is an attempt to fix the problem with the section names, but whenever new sections are added it can generate accentuation problems still
    section_dict = {'cotidian' : 'cotidiano', 'ilustrad': 'ilustrada',
                    'quadrin': 'quadrinhos' , 'opiniao' : u'opini\xE3o',
                    'ciencia' : u'cincia' , 'saude' : u'sa\xfade',
                    'ribeirao' : u'ribeir\xE3o' , 'equilibrio' : u'equil\xedbrio',
                    'imoveis' : u'im\xf3veis', 'negocios' : u'neg\xf3cios',
                    'veiculos' : u've\xedculos', 'corrida' : 'folha corrida',
                    'turismo':'turismo'}

    # this solves the problem with truncated content in Kindle
    conversion_options = {'linearize_tables' : True}

    extra_css = """
#articleNew { font: 18px Times New Roman,verdana,arial; }
img { background: none !important; float: none; margin: 0px; }
.newstexts { list-style-type: none; height: 20px; margin: 15px 0 10px 0; }
.newstexts.last { border-top: 1px solid #ccc; margin: 5px 0 15px 0; padding-top: 15px; }
.newstexts li { display: inline; padding: 0 5px; }
.newstexts li.prev { float: left; }
.newstexts li.next { float: right; }
.newstexts li span { width: 12px; height: 15px; display: inline-block; }
.newstexts li.prev span { background-position: -818px -46px; }
.newstexts li.next span { background-position: -832px -46px; }
.newstexts li a { font: bold 12px arial, verdana, sans-serif; text-transform: uppercase; color: #999; text-decoration: none !important; }
.newstexts li a:hover { text-decoration: underline !important }
.headerart { font-weight: bold; }
.title { font: bold 39px Times New Roman,verdana,arial; margin-bottom: 15px; margin-top: 10px; }
.creditart, .origin { font: bold 12px arial, verdana, sans-serif; color: #999; margin: 0px; display: block; }
.headerart p, .fine_line p { margin: 0 !important; }
.fine_line { font: bold 18px Times New Roman,verdana,arial; }
.fine_line p { margin-bottom: 18px !important;  }
.fine_line p:first-child { font-weight: normal; font-style: italic; font-size: 20px !important; }
.eye { display: block; width: 317px; border-top: 2px solid #666; padding: 7px 0 7px; border-bottom: 2px solid #666; font-style: italic; font-weight: bold; }
.kicker { font-weight: bold; text-transform: uppercase; font-size: 18px; font-family: Times New Roman,verdana,arial !important; }
.blue { color: #000080; }
.red { color: #F00; }
.blue { color: #000080; }
.green { color: #006400; }
.orange { color: #FFA042; }
.violet { color: #8A2BE2; }
.text_footer { font-size: 15px; }
.title_end { font-size: 23px; font-weight: bold; }
.divisor { text-indent: -9999px; border-bottom: 1px solid #ccc; height: 1px; margin: 0; }
.star { background: none !important; height: 15px; }
.articleGraphic { margin-bottom: 20px; }
"""

    # This is the code for login, here a mini browser is called and id entered
    def get_browser(self):
        br = BasicNewsRecipe.get_browser(self)
        if self.username is not None and self.password is not None:
            br.open('https://acesso.uol.com.br/login.html')
            br.form = br.forms()
            br['user']   = self.username
            br['pass'] = self.password
            br.submit().read()
        return br

    # Parsing the index webpage
    def parse_index(self):

        # In the last version, the index page became simpler:
        INDEX = 'http://www1.folha.uol.com.br/fsp/'
        self.log('--> INDEX set ', INDEX)

        soup = self.index_to_soup(INDEX)

        feeds = []
        articles = []
        section_title = u'Primeira p\xe1gina'

        for post in soup.findAll('a'):
            strpost = str(post)
            if re.match('<a href="http://www1.folha.uol.com.br/.*/"><span.class="', strpost):
                if articles:
                    feeds.append((section_title, articles))
                    self.log()
                    self.log('--> new section found, creating old section feed: ', section_title)
                # section_title = post['name']
                section_title = self.tag_to_string(post)
                if section_title in self.section_dict:
                    section_title = self.section_dict[section_title]
                articles = []
                self.log('--> new section title:   ', section_title)
            elif strpost.startswith('<a href="/fsp/cp'):
                break
            elif strpost.startswith('<a href'):
                url = post['href']
                if url.startswith('http://www1.folha.uol.com.br/'):
                    title = self.tag_to_string(post)
                    self.log()
                    self.log('--> post:  ', post)
                    self.log('--> url:   ', url)
                    self.log('--> title: ', title)
                    articles.append({'title':title, 'url':url})

        if articles:
            feeds.append((section_title, articles))

        del feeds[0]
        return feeds

And the error is:

calibre, version 3.4.0 (linux2, embedded-python: False)
Conversion error: Falha: Fetch news from Folha de São Paulo

Fetch news from Folha de São Paulo
Conversion options changed from defaults:
  output_profile: 'kindle'
  verbose: 2
Resolved conversion options
calibre version: 3.4.0
{'asciiize': False,
 'author_sort': None,
 'authors': None,
 'base_font_size': 0,
 'book_producer': None,
 'change_justification': 'original',
 'chapter': None,
 'chapter_mark': 'pagebreak',
 'comments': None,
 'cover': None,
 'debug_pipeline': None,
 'dehyphenate': True,
 'delete_blank_paragraphs': True,
 'disable_font_rescaling': False,
 'dont_compress': False,
 'dont_download_recipe': False,
 'duplicate_links_in_toc': False,
 'embed_all_fonts': False,
 'embed_font_family': None,
 'enable_heuristics': False,
 'expand_css': False,
 'extra_css': None,
 'extract_to': None,
 'filter_css': None,
 'fix_indents': True,
 'font_size_mapping': None,
 'format_scene_breaks': True,
 'html_unwrap_factor': 0.4,
 'input_encoding': None,
 'input_profile': <calibre.customize.profiles.InputProfile object at 0x7fa8a5e11850>,
 'insert_blank_line': False,
 'insert_blank_line_size': 0.5,
 'insert_metadata': False,
 'isbn': None,
 'italicize_common_cases': True,
 'keep_ligatures': False,
 'language': None,
 'level1_toc': None,
 'level2_toc': None,
 'level3_toc': None,
 'line_height': 0,
 'linearize_tables': False,
 'lrf': False,
 'margin_bottom': 5.0,
 'margin_left': 5.0,
 'margin_right': 5.0,
 'margin_top': 5.0,
 'markup_chapter_headings': True,
 'max_toc_links': 50,
 'minimum_line_height': 120.0,
 'mobi_file_type': 'old',
 'mobi_ignore_margins': False,
 'mobi_keep_original_images': False,
 'mobi_toc_at_start': False,
 'no_chapters_in_toc': False,
 'no_inline_navbars': True,
 'no_inline_toc': False,
 'output_profile': <calibre.customize.profiles.KindleOutput object at 0x7fa8a5e11f10>,
 'page_breaks_before': None,
 'personal_doc': '[PDOC]',
 'prefer_author_sort': False,
 'prefer_metadata_cover': False,
 'pretty_print': False,
 'pubdate': None,
 'publisher': None,
 'rating': None,
 'read_metadata_from_opf': None,
 'remove_fake_margins': True,
 'remove_first_image': False,
 'remove_paragraph_spacing': False,
 'remove_paragraph_spacing_indent_size': 1.5,
 'renumber_headings': True,
 'replace_scene_breaks': '',
 'search_replace': None,
 'series': None,
 'series_index': None,
 'share_not_sync': False,
 'smarten_punctuation': False,
 'sr1_replace': '',
 'sr1_search': '',
 'sr2_replace': '',
 'sr2_search': '',
 'sr3_replace': '',
 'sr3_search': '',
 'start_reading_at': None,
 'subset_embedded_fonts': False,
 'tags': None,
 'test': False,
 'timestamp': None,
 'title': None,
 'title_sort': None,
 'toc_filter': None,
 'toc_threshold': 6,
 'toc_title': None,
 'transform_css_rules': None,
 'unsmarten_punctuation': False,
 'unwrap_lines': True,
 'use_auto_toc': False,
 'verbose': 2}
InputFormatPlugin: Recipe Input running
Using custom recipe
Traceback (most recent call last):
  File "/usr/bin/calibre-parallel", line 20, in <module>
    sys.exit(main())
  File "/usr/lib/calibre/calibre/utils/ipc/worker.py", line 195, in main
    result = func(*args, **kwargs)
  File "/usr/lib/calibre/calibre/gui2/convert/gui_conversion.py", line 26, in gui_convert
    plumber.run()
  File "/usr/lib/calibre/calibre/ebooks/conversion/plumber.py", line 1088, in run
    accelerators, tdir)
  File "/usr/lib/calibre/calibre/customize/conversion.py", line 245, in __call__
    log, accelerators)
  File "/usr/lib/calibre/calibre/ebooks/conversion/plugins/recipe_input.py", line 118, in convert
    ro = recipe(opts, log, self.report_progress)
  File "/usr/lib/calibre/calibre/web/feeds/news.py", line 904, in __init__
    self.browser = self.get_browser()
  File "<string>", line 86, in get_browser
AttributeError: 'list' object has no attribute 'next'

I will get to it in the future, but would appreciate if someone could help out! Best.