[patoliadixit]

Originally Posted by PatStapleton

Ok this is working although a couple of minor bugs which I haven't bothered addressing as I've spent as much time as I'd like to for now:
- "\n" characters appear and I haven't been able to remove them as such
- there can be duplicate articles as they sometimes appear under more than one feed e.g. "Latest" and "Technology" (perhaps just remove the "Latest" feed by commenting out if you prefer)

#!/usr/bin/env python2
# vim:fileencoding=utf-8
from __future__ import unicode_literals, division, absolute_import, print_function
__license__ = 'GPL v3'
__copyright__ = '2020, Pat Stapleton <pat.stapleton at gmail.com>'
'''
Recipe for Knappily
'''
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.web.feeds import Feed
from calibre.ebooks.BeautifulSoup import BeautifulSoup

class Knappily(BasicNewsRecipe):
    title          = 'Knappily'
    language       = 'en'
    __author__     = 'Pat Stapleton'
    description = 'One-stop solution for all the major issues ranging from politics, economy, business, sports to technology and law  to make people “a subject matter expert in 2 minutes”.'
    oldest_article = 7 #days
    max_articles_per_feed = 100
    publication_type = 'digital magazine'

    use_embedded_content = False # if set to true will assume that all the article content is within the feed (i.e. won't try to fetch more data)

    feeds          = [
        ('Latest Knapps', 'https://feeds.feedburner.com/knappily-latestknapps'),
        ('Sports', 'https://feeds.feedburner.com/knappily-sports'),
        ('Politics', 'https://feeds.feedburner.com/knappily-politics'),
        ('World', 'https://feeds.feedburner.com/knappily-world'),
        ('Society', 'https://feeds.feedburner.com/knappily-society'),
        ('Environment', 'https://feeds.feedburner.com/knappily-environment'),
        ('Business', 'https://feeds.feedburner.com/knappily-business'),
        ('Technology', 'https://feeds.feedburner.com/knappily-technology'),
        ('Budget', 'https://feeds.feedburner.com/knappily-budget'),
        ('On This Day', 'https://feeds.feedburner.com/knappily-onthisday'),
        ('Ethics', 'https://feeds.feedburner.com/knappily-ethics'),
        ('This!', 'https://feeds.feedburner.com/knappily-this'),
    ]

    #javascript loads the article data from another url to prevent scraping/parsing
    def get_article_url(self, article):
        url = article['link']
        article_id = url[url.rindex("/")+1:len(url)]
        raw_data_url = "https://services.knappily.com/article?id=" + article_id
        return raw_data_url

    def preprocess_html(self, soup):        
        #run through sections and cleanup raw data
        article_list = soup.body.contents
        if(len(article_list) <= 1): self.abort_article()#skip the strange empty single image articles
        idx = 0
        for article in article_list:
            if("_id" in article): del article_list[idx] #unwanted item
            idx = idx + 1
        
        #cleanup and add intro to beginning of article
        article_title = article_list.pop().rsplit('"title":')[-1] #discard last item, but it also contains the title which we can grab
        article_title = article_title[0:len(article_title)-2]#drop last 2 characters as they are closing curly braces
        intro_section = article_list.pop() #intro is 2nd last in the list, move it to the front
        article_list.insert(0, intro_section)
        heading_tag = soup.new_tag("h2") #now add title to front
        heading_tag.string = article_title
        article_list.insert(0, heading_tag)
        return soup

Thank you very much!!! I spent much time on this recipe but was unable to make it. But you really did it!!!! Thanks again....