# -*- coding: utf-8 -*- from __future__ import absolute_import import logging import re # py2 vs py3 transition from ..six import text_type as unicode from ..six.moves.urllib import parse as urlparse from ..six.moves.urllib.error import HTTPError from .base_adapter import BaseSiteAdapter from bs4 import Comment from ..htmlcleanup import fix_excess_space from .. import exceptions as exceptions logger = logging.getLogger(__name__) HTML_TAGS = ('a', 'abbr', 'acronym', 'address', 'applet', 'area', 'article', 'aside', 'audio', 'b', 'base', 'basefont', 'bdi', 'bdo', 'big', 'blockquote', 'body', 'br', 'button', 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'embed', 'fieldset', 'figcaption', 'figure', 'font', 'footer', 'form', 'frame', 'frameset', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'head', 'header', 'hr', 'html', 'i', 'iframe', 'img', 'input', 'ins', 'kbd', 'label', 'legend', 'li', 'link', 'main', 'map', 'mark', 'menu', 'menuitem', 'meta', 'meter', 'nav', 'noframes', 'noscript', 'object', 'ol', 'optgroup', 'option', 'output', 'p', 'param', 'picture', 'pre', 'progress', 'q', 'rp', 'rt', 'ruby', 's', 'samp', 'script', 'section', 'select', 'small', 'source', 'span', 'strike', 'strong', 'style', 'sub', 'summary', 'sup', 'svg', 'table', 'tbody', 'td', 'template', 'textarea', 'tfoot', 'th', 'thead', 'time', 'title', 'tr', 'track', 'tt', 'u', 'ul', 'var', 'video', 'wbr') def getClass(): ''' Initializing the class ''' return NovelFullSiteAdapter class NovelFullSiteAdapter(BaseSiteAdapter): ''' Adapter for LightNovelGate.com ''' def __init__(self, config, url): BaseSiteAdapter.__init__(self, config, url) self.story.setMetadata('siteabbrev', 'novelfull') self.dateformat = "%Y-%m-%dT%H:%M:%S+00:00" self.is_adult = False self.username = None self.password = None # get storyId from url--url validation guarantees query correct m = re.match(self.getSiteURLPattern(), url) if m: self.story.setMetadata('storyId', m.group('id')) # normalized story URL. self._setURL("http://" + self.getSiteDomain() + "/" + self.story.getMetadata('storyId') + ".html") else: raise exceptions.InvalidStoryURL(url, self.getSiteDomain(), self.getSiteExampleURLs()) @staticmethod def getSiteDomain(): return 'novelfull.com' @staticmethod def removeRddChapterHeading(elements, title): if title: for el in elements: title_suffix = re.search( r'^(\s*Chapter \d+\s*:?-?)?\s*(.*)\s*$', el.get_text().rsplit('\n', 1).pop()) if title_suffix.lastindex >= 2: title_search = re.search( r'{}\s*$'.format(title_suffix.group(2)), title) if title_search: el.decompose() break @classmethod def getSiteExampleURLs(cls): return "http://novelfull.com/astoryname" def getSiteURLPattern(self): return r"http://novelfull\.com/(?P\S+)\.html" def use_pagecache(self): ''' adapters that will work with the page cache need to implement this and change it to True. ''' return True def extractChapterUrlsAndMetadata(self): # fetch the chapter. From that we will get almost all the # metadata and chapter list url = self.url base_url = 'http://' + self.getSiteDomain() + '/' logger.debug("URL: " + url) try: data = self._fetchUrl(url) except HTTPError as e: if e.code == 404: raise exceptions.StoryDoesNotExist('404 error: {}'.format(url)) else: raise e soup = self.make_soup(data) [script.extract() for script in soup.find_all('script')] info_div = soup.select_one('div.info') desc_div = soup.select_one('div.desc-text') # getting first Author try: author_link = info_div.find( 'h3', string='Author:').find_next_sibling("a") author_name = author_link.string author_url = urlparse.urljoin(base_url, author_link['href']) self.story.setMetadata('authorId', author_url.rsplit('/', 1).pop()) self.story.setMetadata('authorUrl', author_url) self.story.setMetadata('author', author_name) except Exception: self.story.setMetadata('authorId', 'unknown') self.story.setMetadata('author', 'Unknown') # get title self.story.setMetadata( 'title', unicode(desc_div.find_previous('h3', class_='title').string).strip()) # getting status status = '' try: status = unicode( info_div.find('h3', string='Status:').find_next_sibling("a").string) except Exception: logger.debug('Status not available') if status.strip().lower() == 'completed': self.story.setMetadata('status', 'Completed') else: self.story.setMetadata('status', 'In-Progress') # getting genres genre_list = [] try: genres = info_div.find('h3', string='Genre:').find_next_siblings("a") for agenre in genres: genre_list.append(agenre.string) except Exception: logger.debug('Genres not available') self.story.extendList('genre', genre_list) # getting description synopsis = '' try: pcount = 0 for para in desc_div.find_all('p', recursive=False): synopsis += para.get_text() + ' ' pcount += 1 if pcount > 10: break except Exception: logger.debug('Synopsis not available') self.setDescription(url, synopsis) # getting cover img = soup.select_one('div.book > img') if img: self.setCoverImage(url, urlparse.urljoin(base_url, img['src'])) # getting chapters paged_soup = soup while paged_soup: chapter_lists = paged_soup.find_all('ul', class_='list-chapter') for alist in chapter_lists: chapters_links = alist.find_all('a') for alink in chapters_links: self.add_chapter(alink.string, urlparse.urljoin(base_url, alink['href'])) next_page = None page_list = paged_soup.find('ul', class_='pagination') if page_list: next_page = page_list.select_one('li.next > a') paged_soup = None if next_page: logger.debug('Getting the next page: {0}'.format( next_page['href'])) try: paged_soup = self.make_soup( self._fetchUrl( urlparse.urljoin(base_url, next_page['href']))) except HTTPError: pass def getChapterText(self, url): data = self._fetchUrl(url) if self.getConfig('fix_excess_space', True): data = fix_excess_space(data) soup = self.make_soup(data) chapter_title = '' try: chapter_title = soup.select_one( 'a.chapter-title').get_text().strip() except Exception: logger.debug('Chapter title not available') story = soup.find('div', id='chapter-content') if not story: raise exceptions.FailedToDownload( "Error downloading Chapter: %s! Missing required element!" % url) try: self.removeRddChapterHeading(story.find_all('p', limit=5), chapter_title) except Exception: logger.debug( 'Redundant chapter heading not present or not removable') # Some comments we will get is invalid. Remove them all. [ comment.extract() for comment in story.find_all( text=lambda text: isinstance(text, Comment)) ] [script.extract() for script in story.find_all('script')] [ end_notes.extract() for end_notes in story.find( 'div', string=re.compile(r'^\s*If you find any errors \(')) ] # assume emote when cloudflare email protection is probably wrong for cfa in story.find_all('a', href='/cdn-cgi/l/email-protection'): try: if cfa.previous_element.string[-1] == '@': cfa.replace_with('_@') except Exception: logger.debug('Protected email replace failed') return self.utf8FromSoup(url, story)