Sorry to keep posting changes. Turns out that this regexp:
Code:
# Remove <br> and replace <br><br> with <p>
(re.compile(r'(.*)<br[^>]*>', re.IGNORECASE),
lambda match: match.group() if re.match('<', match.group(1).lstrip()) or len(match.group(1)) < 40
else match.group(1)),
Was causing most of the problems with lines wrapping in the final ebook, and the regexes I was struggling to create basically just put those back. I suspect based on the logic of length < 40 that this was a simple attempt to stop wrapped lines from being forced to wrap, so the line un-wrapping regexp replaces this function anyway.
I've also modified page link stripping regexp further to completely get rid of the breaks.
Here are the latest changes, and this creates the cleanest looking results yet:
Code:
def sanitize_head(match):
x = match.group(1)
x = _span_pat.sub('', x)
return '<head>\n'+x+'\n</head>'
def wrap_lines(match):
ital = match.group('ital')
if not ital:
return ' '
else:
return ital+' '
def chap_head(match):
chap = match.group('chap')
title = match.group('title')
if not title:
return '<h1>'+chap+'</h1><br/>'
else:
return '<h1>'+chap+'<br/>'+title+'</h1><br/>'
class PreProcessor(object):
PREPROCESS = [
# Some idiotic HTML generators (Frontpage I'm looking at you)
# Put all sorts of crap into <head>. This messes up lxml
(re.compile(r'<head[^>]*>(.*?)</head>', re.IGNORECASE|re.DOTALL),
sanitize_head),
# Convert all entities, since lxml doesn't handle them well
(re.compile(r'&(\S+?);'), convert_entities),
# Remove the <![if/endif tags inserted by everybody's darling, MS Word
(re.compile(r'</{0,1}!\[(end){0,1}if\]{0,1}>', re.IGNORECASE),
lambda match: ''),
]
# Fix pdftohtml markup
PDFTOHTML = [
# Remove <hr> tags
(re.compile(r'<hr[^>]*>', re.IGNORECASE), lambda match: '<br />'),
# Remove page breaks & links
(re.compile(r'<br[^>]*>\s*<br[^>]*>\s*\n?\s*<a name=\d+></a>', re.IGNORECASE), lambda match: '<br>\n'),
# Remove page numbers
(re.compile(r'(?<!\w)\s(\d+)\s?(<br>|<br/>|</p><p>)', re.IGNORECASE), lambda match: ''),
# Replace <br><br> with <p>
(re.compile(r'<br[^>]*>\s*<br[^>]*>', re.IGNORECASE), lambda match: '<p>'),
# un-wrap wrapped lines
(re.compile(r'(?<=.{85}[a-z,IA])\s*(?P<ital></i>)?\s*(<p[^>]*>|<br[^>]*>)?\n\r?\s?(?=(<i>)?\w)', re.UNICODE), wrap_lines),
# Add space before and after italics
(re.compile(r'(?<!“)<i>'), lambda match: ' <i>'),
(re.compile(r'</i>(?=\w)'), lambda match: '</i> '),
# Remove hyphenation
(re.compile(r'-\n\r?'), lambda match: ''),
# Remove gray background
(re.compile(r'<BODY[^<>]+>'), lambda match : '<BODY>'),
# Remove non breaking spaces
(re.compile(ur'\u00a0'), lambda match : ' '),
# Detect Chapters to match default XPATH in GUI
(re.compile(r'(<br[^>]*>)?(</?p[^>]*>)?s*(?P<chap>(Chapter|Epilogue|Prologue|Book|Part)\s*(\d+|\w+)?)(</?p[^>]*>|<br[^>]*>)\n?((?=(<i>)?\s*\w+(\s+\w+)?(</i>)?(<br[^>]*>|</?p[^>]*>))((?P<title>.*)(<br[^>]*>|</?p[^>]*>)))?', re.IGNORECASE), chap_head),
# Have paragraphs show better
(re.compile(r'(?<=.{100})<br[^>]*>\n'), lambda match : '<p>\n'),
]