New Musical Express Magazine recipe

[IngeAning]

Hello

Current NME recipe do not work correctly.
Here is mine suggestion to change this.

Best Regards

import re
from calibre.web.feeds.news import BasicNewsRecipe
from calibre import browser
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ebooks.BeautifulSoup import Tag 
class AdvancedUserRecipe1306061239(BasicNewsRecipe):
    title          = u'New Musical Express Magazine'
    description = 'UK Rock & Pop Mag.'
    __author__ = 'Dave Asbury, Inge Aning'
    category = 'Music, Film, Tv'
    publisher = 'Time Inc. (UK) Ltd.'
    '''
    ' updated 11/3/2015 
    ' feeds url
    ' cover and masterhead url
    ' fix for a bug that prevents some pages render
    ' changes to website
    '''

    remove_empty_feeds = True
    encoding  = 'utf-8'
    remove_javascript = True
    no_stylesheets = True
    oldest_article = 7
    max_articles_per_feed = 20
    auto_cleanup = False
    language = 'en'
    compress_news_images = True
    simultaneous_downloads  = 20
    use_embedded_content = False
    recursions = 0
     
    conversion_options = {
                          'comment'   : description
                        , 'tags'      : category
                        , 'publisher' : publisher
                        , 'language'  : language
    }

    feeds          = [
                    (u'NME News',u'http://www.nme.com/rss/news'),
                    (u'Reviews',u'http://www.nme.com/rss/reviews'),
                    (u'Blogs',u'http://www.nme.com/rss/blogs'),
    ]

    keep_only_tags = [
        dict(name='div',attrs={'id':'content'}),
    ]
    
    remove_attributes = [ 'border', 'cellspacing', 'align', 'cellpadding', 'colspan',
                          'valign', 'vspace', 'hspace', 'alt', 'width', 'height' ]

    remove_tags = [
            dict(name='meta'),
            dict(name='span',attrs={'class':'article_info'}),
            dict(name='div',attrs={'class':'breadcrumbs'}),
            dict(name='div',attrs={'class':'mugshot'}),
            dict(name='div',attrs={'class':'header'}),
            dict(name='div',attrs={'class':re.compile('youtube.*',re.IGNORECASE)}),
            dict(name='div',attrs={'class':re.compile('socialbuttons.*',re.IGNORECASE)}),
            dict(name='div',attrs={'class':'clear_both'}),
            dict(name='div',attrs={'class':re.compile('headline.*',re.IGNORECASE)}),
            dict(name='div',attrs={'class':'member-signedout'}),
            dict(name='div',attrs={'class':re.compile('prev_next.*',re.IGNORECASE)}),
            dict(name='div',attrs={'class':re.compile('article_related.*',re.IGNORECASE)}),
            dict(name='div',attrs={'class':re.compile('feature_bar.*',re.IGNORECASE)}),
            dict(name='div',attrs={'class':re.compile('ebay.*',re.IGNORECASE)}),
            dict(name='div',attrs={'id':re.compile('morenews.*',re.IGNORECASE)}),
            dict(name='div',attrs={'id':re.compile('ticketspopup.*',re.IGNORECASE)}),
            dict(name='div',attrs={'id':re.compile('ratemy_logprompt.*',re.IGNORECASE)}),
            dict(name='div',attrs={'id':re.compile('related_artist.*',re.IGNORECASE)}),
            dict(name='img',attrs={'class':re.compile('video_play_large.*',re.IGNORECASE)}),
            dict(name='ul',attrs={'class':re.compile('prev_next.*',re.IGNORECASE)}),
            dict(name='ul',attrs={'class':re.compile('nme_store.*',re.IGNORECASE)}),
            dict(name='p',attrs={'class':re.compile('top',re.IGNORECASE)}),
            dict(name='table',attrs={'class':re.compile('tickets.*',re.IGNORECASE)}),
    ]

    masthead_url   = 'http://default.media.ipcdigital.co.uk/300/000001014/e1ab_orh100000w300/NME-logo.jpg'

    def get_cover_url(self):
        magazine_page_raw = self.index_to_soup('http://www.nme.com/magazine', raw=True)
        magazine_page_raw = re.sub(r'<script\b.+?</script>', '', magazine_page_raw, flags=re.DOTALL|re.IGNORECASE)
        magazine_page_raw = re.sub(r'\!\[if ', '!--[if ', magazine_page_raw, flags=re.DOTALL|re.IGNORECASE)
        magazine_page = self.index_to_soup(magazine_page_raw)
        cov = magazine_page.find('img',attrs={'class':'magcover'})
        
        cov2 = str(cov['src'])

        br = browser()
        br.set_handle_redirect(False)
        try:
            br.open_novisit(cov2)
            cover_url = str(cov2)
        except:
                cover_url = 'http://default.media.ipcdigital.co.uk/300/000001014/e1ab_orh100000w300/NME-logo.jpg'
        return cover_url

    def preprocess_raw_html(self, raw_html, url):
        '''
        Need this for a bug on site that prevents blogg post being parsed correctly
        '''
        raw_html = re.sub(r'\!\[if ', '!--[if ', raw_html, flags=re.DOTALL|re.IGNORECASE)

        return raw_html 

    def preprocess_html(self, soup): 
        youtube_regex = re.compile(r'(?:youtube\.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu\.be/)(?P<id>[^"&?/ ]{11})', re.DOTALL|re.IGNORECASE)
        instagram_regex = re.compile(r'.*?instagram.*?', re.DOTALL|re.IGNORECASE)
        twitter_regex = re.compile(r'.*?twitter.*?', re.DOTALL|re.IGNORECASE)
        visualise_regex = re.compile(r'.*?visualise.*?', re.DOTALL|re.IGNORECASE)
        soundcloud_regex = re.compile(r'(?P<url>.*?(w|api)\.soundcloud.*?com\/(tracks|playlists)\/\d{8,9})', re.DOTALL|re.IGNORECASE)
        dailymotion_regex = re.compile(r'.*?dailymotion.*?', re.DOTALL|re.IGNORECASE)
        spotify_regex = re.compile(r'.*?spotify.*?', re.DOTALL|re.IGNORECASE)
        vine_regex = re.compile(r'.*?vine.*?', re.DOTALL|re.IGNORECASE)
        doubleHtmlEntities = re.compile(ur'(&amp;)(?P<e>[\d\w\#]*;)', re.DOTALL|re.IGNORECASE|re.UNICODE)
        for iframe in soup.findAll('iframe'):
            if iframe.has_key('src') and youtube_regex.search(iframe['src']) is not None :
                pq = Tag(soup, 'blockquote')
                br = Tag(soup, 'br')
                pq.insert(0, '[ YouTube ]')
                pq.insert(1, br)
                m = youtube_regex.search(iframe['src'])
                if m.group('id') is not None:
                    imgTag = Tag(soup, 'img', [('src','http://img.youtube.com/vi/' + m.group('id') + '/0.jpg')])
                    pq.insert(len(pq.contents), imgTag)
                pq.insert(len(pq.contents), iframe['src'])
                iframe.replaceWith(pq)
            elif iframe.has_key('src') and soundcloud_regex.search(iframe['src']) is not None:
                m = soundcloud_regex.search(iframe['src']) 
                pq = Tag(soup, 'blockquote')
                br = Tag(soup, 'br')
                pq.insert(0, '[ SoundCloud ]')
                pq.insert(1, br)
                pq.insert(2, m.group('url'))
                #imgUrl = self.get_soundcloud_pic(iframe['src'])
                iframe.replaceWith(pq)
            elif iframe.has_key('src') and dailymotion_regex.search(iframe['src']) is not None:
                pq = Tag(soup, 'blockquote')
                br = Tag(soup, 'br')
                pq.insert(0, '[ DailyMotion ]')
                pq.insert(1, br)
                imgUrl = self.get_dailymotion_pic(iframe['src'])
                if imgUrl is not None:
                    imgTag = Tag(soup, 'img', [('src',imgUrl)])
                    pq.insert(len(pq.contents), imgTag)
                pq.insert(len(pq.contents), iframe['src'])
                iframe.replaceWith(pq)
            elif iframe.has_key('src') and spotify_regex.search(iframe['src']) is not None:
                pq = Tag(soup, 'blockquote')
                br = Tag(soup, 'br')
                pq.insert(0, '[ Spotify ]')
                pq.insert(1, br)
                imgUrl = self.get_spotify_pic(iframe['src'])
                if imgUrl is not None:
                    imgTag = Tag(soup, 'img', [('src',imgUrl)])
                    pq.insert(len(pq.contents), imgTag)
                pq.insert(len(pq.contents), iframe['src'])
                iframe.replaceWith(pq)
            elif iframe.has_key('src') and vine_regex.search(iframe['src']) is not None:
                pq = Tag(soup, 'blockquote')
                br = Tag(soup, 'br')
                pq.insert(0, '[ Vine ]')
                pq.insert(1, br)
                imgUrl = self.get_vine_pic(iframe['src'])
                if imgUrl is not None:
                    imgTag = Tag(soup, 'img', [('src',imgUrl)])
                    pq.insert(len(pq.contents), imgTag)
                pq.insert(len(pq.contents), iframe['src'])
                iframe.replaceWith(pq)
            elif iframe.has_key('src') and visualise_regex.search(iframe['src']) is not None :
                imgUrl = self.get_visualise_pic(iframe['src'])
                if imgUrl is not None:
                    imgTag = Tag(soup, 'img', [('src',imgUrl)])
                    iframe.replaceWith(imgTag)
        for blockquote in soup.findAll('blockquote'):
            if blockquote.has_key('class') and twitter_regex.search(blockquote['class']) is not None:
                pq = Tag(soup, 'blockquote')
                br = Tag(soup, 'br')
                pq.insert(0, '[ Twitter ]')
                pq.insert(len(pq.contents), br)
                match = re.search("(?P<url>pic\.twitter[^\s<]+)", str(blockquote))
                if match is not None: 
                    img = self.get_twitter_pic(str(match.group("url")))
                    if img is not None:
                        pq.insert(len(pq.contents),img)
                for p in blockquote.findAll(name='p'):
                    x = 0
                    plen = len(p.contents)
                    while True:
                        c = len(pq.contents)
                        if p.contents[x].string is not None:
                            pq.insert(c, re.sub(doubleHtmlEntities, lambda m: '&' + m.group(2), str(p.contents[x].string), re.IGNORECASE | re.UNICODE))
                        else:
                            pq.insert(c, p.contents[x].content)
                        x += 1
                        if x == plen:
                            break
                    br = Tag(soup, 'br')
                    pq.insert(len(pq.contents), br)
                    p.extract()
                if len(blockquote.contents) > 0:
                    x = 0
                    xlen = len(blockquote.contents)
                    while True:
                        c = len(pq.contents)
                        if blockquote.contents[x].string is not None:
                            pq.insert(c, re.sub(doubleHtmlEntities, lambda m: '&' + m.group(2), str(blockquote.contents[x].string), re.IGNORECASE | re.UNICODE))
                        else:
                            pq.insert(c, blockquote.contents[x].content)
                        x += 1
                        if x == xlen:
                            break
                blockquote.replaceWith(pq)
            elif blockquote.has_key('class') and instagram_regex.search(blockquote['class']) is not None:
                pq = Tag(soup, 'blockquote')
                br = Tag(soup, 'br')
                pq.insert(0, '[ Instagram ]')
                pq.insert(1, br)
                a = blockquote.find(name='a',attrs={'href':instagram_regex})
                imgUrl = None
                if a is not None:
                    imgUrl = self.get_instagram_pic(str(a['href']))
                if imgUrl is not None:
                    img = Tag(soup, 'img', [('src',imgUrl)])
                    pq.insert(len(pq.contents),img)
                for p in blockquote.findAll(name='p'):
                    x = 0
                    plen = len(p.contents)
                    while x < plen:
                        c = len(pq.contents)
                        if p.contents[x].string is not None:
                            pq.insert(c, re.sub(doubleHtmlEntities, lambda m: '&' + m.group(2), str(p.contents[x].string), re.IGNORECASE | re.UNICODE))
                        # else:
                            # pq.insert(c, p.contents[x].content)
                        x += 1
                    br = Tag(soup, 'br')
                    c = len(pq.contents)
                    pq.insert(c, br)
                blockquote.replaceWith(pq)
        for alink in soup.findAll('a'): 
            if alink.string is not None: 
                tstr = alink.string 
                alink.replaceWith(tstr) 
            elif alink.img is not None:
                tstr = alink.img
                alink.replaceWith(tstr)
            elif alink.span is not None:
                tstr = alink.span
                alink.replaceWith(tstr)
        return soup

    def get_visualise_pic(self, url):
        returnValue = None
        try:
            raw = self.browser.open(url).read()
        except:
            print '404: ' + url
            return returnValue
        bs = BeautifulSoup(raw)
        imgRaw = bs.find(name='meta', attrs={'property':'og:image'})
        if imgRaw is not None:
            returnValue = str(imgRaw['content'])
        return returnValue

    
    def get_twitter_pic(self, url):
        returnValue = None
        try:
            raw = self.browser.open('https://' + url).read()
        except:
            print '404: ' + url
            return returnValue
        bs = BeautifulSoup(raw)
        refresh = bs.find('meta', {'http-equiv':'refresh'})
        if refresh is not None:
            content = refresh.get('content').partition('=')[2]
            try:
                raw = self.browser.open(content).read()
            except:
                print '404: ' + url
                return returnValue
            bs = BeautifulSoup(raw)
        img = bs.find(name='img',attrs={'alt':re.compile('.*permalink.*',re.IGNORECASE)})
        if img is not None:
            returnValue = img
        return returnValue

    def get_soundcloud_pic(self, url):
        # content loaded via javascript and require an login and/or registered application identification
        # returnValue = None
        # raw = self.browser.open(soundcloudUrl + '&visual=true').read()
        # bs = BeautifulSoup(raw)
        # imgRaw = bs.find(name='div', attrs={'style':re.compile(r'backgroud-image:*?',re.IGNORECASE)})
        # if imgRaw is not None:
            # returnValue = str(imgRaw['style'])
        return None # returnValue

    def get_instagram_pic(self, url):
        returnValue = None
        try:
            raw = self.browser.open(url).read()
        except:
            print '404: ' + url
            return returnValue
        m = re.search('\"display_src\":\"(?P<url>http[s]?:.*?)\"', str(raw))
        if m is not None:
            returnValue = re.sub(r'\\','',m.group("url"), flags=re.DOTALL|re.IGNORECASE)
        return returnValue

    def get_dailymotion_pic(self, url):
        returnValue = None
        try:
            raw = self.browser.open(url).read()
        except:
            print '404: ' + url
            return returnValue
        m = re.search('("thumbnail_url\"\:\")(?P<url>http.*?)(\")', str(raw))
        if m is not None:
            returnValue = re.sub(r'\\','',m.group("url"), flags=re.DOTALL|re.IGNORECASE)
        return returnValue

    def get_spotify_pic(self, url):
        returnValue = None
        try:
            raw = self.browser.open(url).read()
        except ex:
            print '404: ' + url
            return returnValue
        m = re.search('data-ca=\"(?P<url>.*?)\"', str(raw))
        if m is not None:
            returnValue = m.group("url")
        return returnValue

    def get_vine_pic(self, url):
        returnValue = None
        try:
            raw = self.browser.open(url).read()
        except:
            print '404: ' + url
            return returnValue
        m = re.search('"thumbnail.*?src=\"(?P<url>.*?)\"', str(raw))
        if m is not None:
            returnValue = m.group("url")
        return returnValue

    preprocess_regexps = [
        (re.compile(r'<script\b.+?</script>', re.DOTALL|re.IGNORECASE), lambda h1: ''),
        (re.compile(r'<a.* id="buy-tickets-button".*</a>',re.IGNORECASE), lambda h2: ''),
        (re.compile(r'<a.* class="gallery.*</a>',re.IGNORECASE), lambda h2: ''),
    ]

    extra_css = '''
                    h1 h2 {
                        font-family:Arial,Helvetica,sans-serif; 
                        font-weight:bold;font-size:large;
                    }
                    h3 {
                        font-family:Arial,Helvetica,sans-serif; 
                        font-weight:normal;
                        font-size:small;
                        font-style:italic;
                        display:inline;
                    }
                    body {
                        font-family:Helvetica,Arial,sans-serif;
                        font-size:small;
                    }
                    blockquote {
                        font-family:"Courier New", 
                        Courier, monospace;
                        font-size:90%;
                    }
                    img {
                        display:block;
                    }
                    .date{
                        font-style:italic;
                        font-weight:normal;
                    }
                    .article_header>p:not(.date){
                        font-weight:bold;
                    }
                '''