Recipe for Medical Journal of Australia

[PatStapleton]

I wasn't sure where to post custom recipes, so I'm posting here.

I wrote a recipe for the Medical Journal of Australia, hopefully this is useful to somebody.

Note that I wrote it to use my institution's login so you may need to adjust this for your own use. The parts to adjust are the "#LOGIN" section, and the "#fix url to pickup institution login by appending" line.

Spoiler:

__license__   = 'GPL v3'
__copyright__ = '2011, Pat Stapleton <pat.stapleton at gmail.com>'

from calibre.web.feeds import Feed
from calibre.web.feeds.recipes import BasicNewsRecipe
import re

class MJA(BasicNewsRecipe):
    title          = u'MJA'
    description = u'The Medical Journal of Australia'
    category = u'medical, science, health, Australia'
    __author__            = 'Pat Stapleton'
    oldest_article = 14
    max_articles_per_feed = 100
    auto_cleanup = True
    needs_subscription = True
    language              = 'en_AU'
    remove_empty_feeds    = True
    publication_type      = 'journal'
    publisher            = u'Australian Medical Association'
    #masthead_url = 'http://www.mja.com.au/MJAnav.gif'
    
    feeds          = [(u'MJA', u'http://feeds.feedburner.com/TheMedicalJournalOfAustralia?format=xml')]
    
    #LOGIN
    def get_browser(self):
        br = BasicNewsRecipe.get_browser()
        br.open('http://ipacez.nd.edu.au/login?url=http://www.mja.com.au')
        br.select_form(nr=0)
        br['user'] = self.username
        br['pass'] = self.password
        br.submit()
        return br
    
    def parse_feeds(self):
        # Do the "official" parse_feeds first
	myFeeds = BasicNewsRecipe.parse_feeds(self)
        
        # Loop thru all articles and compile list of sections
	sectionedArticles = {}
	for curfeed in myFeeds:
            for a, curarticle in enumerate(curfeed.articles):
                #MJA articles have the title format '[SECTION] Article Title' so lets grab section
                section_article = re.split(']', curarticle.title)
                sectionTitle = section_article[0].lstrip('[')
                articleTitle = section_article[1].lstrip()
                if sectionTitle not in sectionedArticles:
                    sectionedArticles[sectionTitle] = []
                
                #cleanup article's title (remove ugly section prefix)
                curarticle.title = articleTitle
                
                #fix url to pickup institution login by appending
                curarticle.url = curarticle.url.replace('http://www.mja.com.au', 'http://www.mja.com.au.ipacez.nd.edu.au')
                
                sectionedArticles[sectionTitle].append(curarticle)
        
        #Create our nice list of sectioned feeds to return
        retFeeds = []
        for section in sectionedArticles:
            newSection = Feed()
            newSection.title = section
            newSection.description = self.description
            newSection.articles = sectionedArticles[section]
            newSection.image_url = None
            retFeeds.append(newSection)
        
        return retFeeds

Enjoy!

-Pat

[kovidgoyal]

This is the right place.

[PatStapleton]

Thanks Kovid .

I just started using Calibre yesterday, and I must say this is a great program! Thanks very much for writing such a useful piece of software!


-Pat

[kovidgoyal]

You're welcome

[PatStapleton]

Hi everybody, the site has since changed so I've rewritten this recipe so it works again.

Like last time, I've written it to use my institution's login so for your own use you should modify/remove the #LOGIN section and the line:

"url = 'http://www.mja.com.au.ipacez.nd.edu.au' + section.a.get('href')"

should be changed to:

"url = 'http://www.mja.com.au' + section.a.get('href')"

Spoiler:

__license__   = 'GPL v3'
__copyright__ = '2012, Pat Stapleton <pat.stapleton at gmail.com>'

from calibre.web.feeds import Feed
from calibre.web.feeds.recipes import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from dateutil import parser
import datetime
import time
import re

class MJA(BasicNewsRecipe):
    title          = u'MJA'
    description = u'The Medical Journal of Australia'
    category = u'medical, science, health, Australia'
    __author__            = 'Pat Stapleton'
    oldest_article = 14
    max_articles_per_feed = 1000
    auto_cleanup = True
    needs_subscription = True
    language              = 'en_AU'
    remove_empty_feeds    = True
    publication_type      = 'journal'
    publisher            = u'Australian Medical Association'
    #masthead_url = 'http://www.mja.com.au/MJAnav.gif'
    issue = None
    
    #LOGIN
    def get_browser(self):
        br = BasicNewsRecipe.get_browser()
        br.open('http://ipacez.nd.edu.au/login?url=http://www.mja.com.au')
        br.select_form(nr=0)
        br['user'] = self.username
        br['pass'] = self.password
        br.submit()
        return br
    
    #Get cover image
    def get_cover_url(self):
        retUrl = ''
        rawc = self.index_to_soup('http://www.mja.com.au',True)
        soup = BeautifulSoup(rawc)
        for itimg in soup.findAll('img',src=True):
            if '/cover' in itimg['src']:
                retUrl = 'http://www.mja.com.au/' + itimg['src']
        
        return retUrl

    def parse_index(self):
	#Get link to contents of current issue
	rawc = self.index_to_soup('http://www.mja.com.au',True)
        soup = BeautifulSoup(rawc)
	linkObject = soup.find('div', 'homepage-current-issue-inner')
	link = 'http://www.mja.com.au' + linkObject.contents[0]['href']
	
	feeds = []
	rawc = self.index_to_soup(link, True)
	soup = BeautifulSoup(rawc)

	artCount = 0
	sectionCount = 0
	sectionTitles = soup.findAll('h3')
	sectionTitle = self.tag_to_string(sectionTitles[0])
	articles = []
	for section in soup.findAll(attrs={'class':lambda x: x and 'views-row' in x}):
		artCount = artCount + 1
		classField = section.get('class')
		if(classField.find('views-row-1') != -1): #this is first of new section
			if(articles):
				print(articles)
				feeds.append((sectionTitle, articles))
				articles = []
			if(sectionCount < len(sectionTitles)):
				sectionTitle = self.tag_to_string(sectionTitles[sectionCount])
				sectionCount = sectionCount + 1
			else:
				break
		if(section.p): 
			artTitle = self.tag_to_string(section.p)
		else:
			artTitle = self.tag_to_string(section.a)
		url = 'http://www.mja.com.au.ipacez.nd.edu.au' + section.a.get('href')
		date = ''
		desc = ''
		content = ''
		article = {'title':artTitle, 'url':url, 'date':date, 'description':desc, 'content':content}
		articles.append(article)
	
	if(articles):
		feeds.append((sectionTitle, articles))
		articles = []

	return feeds

Enjoy!

-Pat