Code:
from calibre.web.feeds.news import BasicNewsRecipe
from calibre import browser
from calibre.constants import config_dir, CONFIG_DIR_MODE
import re
import mechanize
from calibre.utils.magick import Image
#declare global temp file
Feeds_File = config_dir+'\\feeds.txt'
# needed for getting rid of repeat feeds
class AdvancedUserRecipe1306097511(BasicNewsRecipe):
title = u'Cosmopolitan UK'
description = 'Author : D.Asbury : Womens Fashion, beauty and Gossip for women from COSMOPOLITAN -UK'
__author__ = 'Dave Asbury'
#last update 7/10/12 hopefully get current cover from itunes
# greyscale code by Starson
cover_url = 'http://www.cosmopolitan.magazine.co.uk/files/4613/2085/8988/Cosmo_Cover3.jpg'
no_stylesheets = True
oldest_article = 31
max_articles_per_feed = 20
remove_empty_feeds = True
remove_javascript = True
#global variables required for getting rid of duplicate articles
article_already_exists = False
preprocess_regexps = [
(re.compile(r'<!-- Begin tmpl module_competition_offer -->.*?<!-- End tmpl module_competition_offer-->', re.IGNORECASE | re.DOTALL), lambda match: '')]
language = 'en_GB'
masthead_url = 'http://www.cosmopolitan.co.uk/cm/cosmopolitanuk/site_images/header/cosmouk_logo_home.gif'
keep_only_tags = [
dict(attrs={'class' : ['dateAuthor', 'publishDate']}),
dict(name='div',attrs ={'id' : ['main_content']})
]
remove_tags = [
dict(name='div',attrs={'class' : ['blogInfo','viral_toolbar','comment_number','prevEntry nav']}),
dict(name='div',attrs={'class' : 'blog_module_about_the_authors'}),
dict(attrs={'id': ['breadcrumbs','comment','related_links_list','right_rail','content_sec_fb_more','content_sec_mostpopularstories','content-sec_fb_frame_viewfb_bot']}),
dict(attrs={'class' : ['scroller','read_liked_that_header','fb_back_next_area']}),
dict(name='li',attrs={'class' : 'thumb'})
]
feeds = [
(u'Love & Sex', u'http://www.cosmopolitan.co.uk/love-sex/rss/'), (u'Men', u'http://cosmopolitan.co.uk/men/rss/'), (u'Fashion', u'http://cosmopolitan.co.uk/fashion/rss/'), (u'Hair & Beauty', u'http://cosmopolitan.co.uk/beauty-hair/rss/'), (u'LifeStyle', u'http://cosmopolitan.co.uk/lifestyle/rss/'), (u'Cosmo On Campus', u'http://cosmopolitan.co.uk/campus/rss/'), (u'Celebrity Gossip', u'http://cosmopolitan.co.uk/celebrity-gossip/rss/')]
print '@@@@@@@',Feeds_File
def parse_feeds(self):
feeds = BasicNewsRecipe.parse_feeds(self)
print 'create empty file'
print
#open and close empty file - otherwise crashes as you can't append a file that doesn't exist?
read_file=open(Feeds_File,'w+')
read_file.close()
# repeat for all feeds
for feed in feeds:
print 'Feed file = ',Feeds_File
# for each section do
print
print 'Feed section is ',feed.title
# for each artcile in each section check if it's in the feeds file
for article in feed.articles[:]:
article_already_exists = False
print
#open the file and reads lines of text
read_file=open(Feeds_File)
while 1:
line=read_file.readline()
print
print'****'
print 'Value of line:',line
print 'article.title is:',article.title
if str(line) == str(article.title+'\n'):
article_already_exists = True
print 'repeated article'
break
print'*****'
print
# eof reached
if not line: break
read_file.close()
# couldn't find article so write it to file
if article_already_exists == False:
read_file=open(Feeds_File,'a')
read_file.write(article.title+'\n')
read_file.close()
if article_already_exists == True:
article.url ='' # delete the url so won't download
return feeds
def get_cover_url(self):
soup = self.index_to_soup('http://itunes.apple.com/gb/app/cosmopolitan-uk/id461363572?mt=8')
# look for the block containing the sun button and url
cov = soup.find(attrs={'alt' : 'iPhone Screenshot 1'})
cov2 = str(cov['src'])
# cov2=cov2[7:]
print '88888888 ',cov2,' 888888888888'
#cover_url=cov2
#return cover_url
br = mechanize.Browser()
br.set_handle_redirect(False)
try:
br.open_novisit(cov2)
cover_url = cov2
except:
cover_url = 'http://www.cosmopolitan.magazine.co.uk/files/4613/2085/8988/Cosmo_Cover3.jpg'
return cover_url
def postprocess_html(self, soup, first):
#process all the images
for tag in soup.findAll(lambda tag: tag.name.lower()=='img' and tag.has_key('src')):
iurl = tag['src']
img = Image()
img.open(iurl)
if img < 0:
raise RuntimeError('Out of memory')
img.type = "GrayscaleType"
img.save(iurl)
return soup