Hi Kovid,
thanks for integrating my recipe into Calibre!
In the meantime I have been working on an improved version, please find the new code below.
Changelog:
- added subscription support
- modify formatting of author and date lists instead of using CSS magic
- remove local hyperlinks
- code cleanup
Code:
#!/usr/bin/env python2
# vim:fileencoding=utf-8
from __future__ import unicode_literals
__license__ = 'GPL v3'
__copyright__ = '2016, Aimylios'
'''
handelsblatt.com
'''
import re
from calibre.web.feeds.news import BasicNewsRecipe
class Handelsblatt(BasicNewsRecipe):
title = 'Handelsblatt'
__author__ = 'Aimylios' # based on the work of malfi and Hegi
description = 'RSS-Feeds von Handelsblatt.com'
publisher = 'Verlagsgruppe Handelsblatt GmbH'
publication_type = 'newspaper'
needs_subscription = 'optional'
language = 'de'
encoding = 'utf-8'
oldest_article = 4
max_articles_per_feed = 30
simultaneous_downloads = 20
no_stylesheets = True
remove_javascript = True
remove_empty_feeds = True
ignore_duplicate_articles = {'title', 'url'}
conversion_options = {'smarten_punctuation' : True,
'publisher' : publisher}
# uncomment this to reduce file size
#compress_news_images = True
#compress_news_images_max_size = 16
cover_source = 'https://kaufhaus.handelsblatt.com/downloads/handelsblatt-epaper-p1951.html'
masthead_url = 'http://www.handelsblatt.com/images/logo_handelsblatt/11002806/7-formatOriginal.png'
feeds = [
('Top-Themen', 'http://www.handelsblatt.com/contentexport/feed/top-themen'),
('Politik', 'http://www.handelsblatt.com/contentexport/feed/politik'),
('Unternehmen', 'http://www.handelsblatt.com/contentexport/feed/unternehmen'),
('Finanzen', 'http://www.handelsblatt.com/contentexport/feed/finanzen'),
('Technologie', 'http://www.handelsblatt.com/contentexport/feed/technologie'),
('Panorama', 'http://www.handelsblatt.com/contentexport/feed/panorama'),
('Sport', 'http://www.handelsblatt.com/contentexport/feed/sport')
]
keep_only_tags = [dict(name='div', attrs={'class':['vhb-article-container']})]
remove_tags = [
dict(name='span', attrs={'class':['vhb-colon', 'vhb-label-premium']}),
dict(name='aside', attrs={'class':['vhb-article-element vhb-left',
'vhb-article-element vhb-left vhb-teasergallery',
'vhb-article-element vhb-left vhb-shorttexts']}),
dict(name='article', attrs={'class':['vhb-imagegallery vhb-teaser',
'vhb-teaser vhb-type-video']}),
dict(name='small', attrs={'class':['vhb-credit']}),
dict(name='div', attrs={'class':['white_content', 'fb-post']}),
dict(name='a', attrs={'class':['twitter-follow-button']}),
dict(name='blockquote')
]
preprocess_regexps = [
# Insert ". " after "Place" in <span class="hcf-location-mark">Place</span>
(re.compile(r'(<span class="hcf-location-mark">[^<]+)(</span>)',
re.DOTALL|re.IGNORECASE), lambda match: match.group(1) + '. ' + match.group(2)),
# Insert ": " after "Title" in <em itemtype="text" itemprop="name" class="vhb-title">Title</em>
(re.compile(r'(<em itemtype="text" itemprop="name" class="vhb-title">[^<]+)(</em>)',
re.DOTALL|re.IGNORECASE), lambda match: match.group(1) + ': ' + match.group(2))
]
extra_css = 'h2 {text-align: left} \
h3 {font-size: 1em; text-align: left} \
h4 {font-size: 1em; text-align: left; margin-bottom: 0em} \
em {font-style: normal; font-weight: bold} \
.vhb-subline {font-size: 0.6em; text-transform: uppercase} \
.vhb-teaser-head {margin-top: 1em; margin-bottom: 1em} \
.vhb-caption-wrapper {font-size: 0.6em} \
.hcf-location-mark {font-weight: bold} \
.panel-body p {margin-top: 0em}'
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
if self.username is not None and self.password is not None:
br.open('https://profil.vhb.de/sso/login?service=http://www.handelsblatt.com')
br.select_form(nr=0)
br['username'] = self.username
br['password'] = self.password
br.submit()
return br
def get_cover_url(self):
soup = self.index_to_soup(self.cover_source)
style = soup.find('img', alt='Handelsblatt ePaper', style=True)['style']
self.cover_url = style.partition('(')[-1].rpartition(')')[0]
return self.cover_url
def print_version(self, url):
main, sep, id = url.rpartition('/')
return main + '/v_detail_tab_print/' + id
def preprocess_html(self, soup):
# remove all articles without relevant content (e.g., videos)
article_container = soup.find('div', {'class':'vhb-article-container'})
if article_container is None:
self.abort_article()
else:
# remove all local hyperlinks
for a in soup.findAll('a', {'href':True}):
if a['href'] and a['href'][0] in ['/', '#']:
a.replaceWith(a.renderContents())
return soup
def postprocess_html(self, soup, first_fetch):
# convert lists of author(s) and date(s) into simple text
for cap in soup.findAll('div', {'class':re.compile('.*vhb-article-caption')}):
cap.replaceWith(cap.renderContents())
for row in soup.findAll('div', {'class':'vhb-article-author-row'}):
for ul in row.findAll('ul'):
entry = ''
for li in ul.findAll(lambda tag: tag.name == 'li' and not tag.attrs):
entry = entry + li.renderContents() + ', '
for li in ul.findAll(lambda tag: tag.name == 'li' and tag.attrs):
entry = entry + li.renderContents() + '<br/>'
ul.parent.replaceWith(entry)
# make sure that all figure captions (including the source) are shown
# without linebreaks by using the alternative text given within <img/>
# instead of the original text (which is oddly formatted)
for fig in soup.findAll('figure', {'class':'vhb-image'}):
fig.find('div', {'class':'vhb-caption'}).replaceWith(fig.find('img')['alt'])
return soup