#!/usr/bin/env python
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai

from __future__ import (unicode_literals, division, absolute_import, print_function)

import re
import mechanize
import time

from calibre.utils.config_base import tweaks
from calibre.utils.date import parse_only_date

from calibre_plugins.overdrive_link.numbers import (value_unit, numeric_rank)
from calibre_plugins.overdrive_link.book import (LibraryBook, InfoBook)
from calibre_plugins.overdrive_link.formats import (
    FORMAT_ADOBE_EPUB, FORMAT_ADOBE_PDF, FORMAT_KINDLE_BOOK, FORMAT_MOBI_EBOOK, FORMAT_OD_MP3,
    FORMAT_OD_MUSIC, FORMAT_OD_READ, FORMAT_NOOK_PERIODICALS,
    FORMAT_OD_WMA, FORMAT_OD_VIDEO, FORMAT_OD_VIDEO_MOBILE, FORMAT_OPEN_EPUB, FORMAT_OPEN_PDF,
    FORMAT_STREAMING_VIDEO, FORMAT_DISNEY_ONLINE_BOOK, FORMAT_MEDIADO_READER, FORMAT_OD_LISTEN)
from calibre_plugins.overdrive_link.recipe_81611_1 import roman_to_int
from calibre_plugins.overdrive_link.library import (SearchableLibrary)
from calibre_plugins.overdrive_link.net import (browse_url, open_url, hostname_from_url, isValidHostname)
from calibre_plugins.overdrive_link.author_prep import normalize_author
from calibre_plugins.overdrive_link.title_prep import normalize_title
from calibre_plugins.overdrive_link.parseweb import (
    LibraryError, must_find, must_findAll, text_only, class_contains, double_quote, valid_isbn,
    is_sign_in_form, set_card_number, set_card_pin, beautiful_soup, beautiful_soup_fix, soup_strainer)
from calibre_plugins.overdrive_link.tweak import TWEAK_SAVE_RESPONSES_ON_ERROR
from calibre_plugins.overdrive_link.language import LANGUAGE_CODE2
from calibre_plugins.overdrive_link.fixups import VOLUMES_WITH_UNIQUE_TITLES

from .python_transition import (IS_PYTHON2)
if IS_PYTHON2:
    from .python_transition import (http, repr, str, urllib)
else:
    import http.cookiejar
    import urllib.parse


__license__ = 'GPL v3'
__copyright__ = '2012-2025, John Howell <jhowell@acm.org>'


CHECK_FORMAT_CONSISTENCY = False

GENERIC_SEARCH_SITE = 'search.overdrive.com'


OVERDRIVE_FORMAT_NUMBERS = {
    # from advanced search form
    "25": FORMAT_OD_WMA,        # No longer supported by OverDrive as of summer 2015
    "30": FORMAT_OD_MUSIC,
    "35": FORMAT_OD_VIDEO,
    "40": FORMAT_OD_VIDEO_MOBILE,
    "50": FORMAT_ADOBE_PDF,
    "302": FORMAT_DISNEY_ONLINE_BOOK,
    "303": FORMAT_MEDIADO_READER,
    "304": FORMAT_NOOK_PERIODICALS,
    "410": FORMAT_ADOBE_EPUB,
    "420": FORMAT_KINDLE_BOOK,
    "425": FORMAT_OD_MP3,
    "450": FORMAT_OPEN_PDF,
    "610": FORMAT_OD_READ,
    "625": FORMAT_OD_LISTEN,
    "635": FORMAT_STREAMING_VIDEO,
    "810": FORMAT_OPEN_EPUB,
    }

OVERDRIVE_FORMAT_NAMES = {
    # common alternate names used at some libraries
    'Downloadable Music': FORMAT_OD_MUSIC,
    'Digital Music': FORMAT_OD_MUSIC,
    'Downloadable Video': FORMAT_OD_VIDEO,
    'Digital Video': FORMAT_OD_VIDEO,
    'EPUB eBook': FORMAT_ADOBE_EPUB,
    'EPUB eBooks': FORMAT_ADOBE_EPUB,
    'PDF eBook': FORMAT_ADOBE_PDF,
    'PDF eBooks': FORMAT_ADOBE_PDF,
    'NOOK® Periodicals': FORMAT_NOOK_PERIODICALS,
    }


AUTHOR_ROLES = {'Author', 'Editor', 'Other'}
# ignore roles: "Narrator", "Author of afterword, colophon, etc.", "Author of introduction, etc.", "Composer",


def fix_title(odtitle, subtitle, edition='', odseries=''):
    # Subtitle is sometimes just a subtitle and sometimes has series information. Try to guess which.
    title = odtitle
    series = ''
    series_index = 0.0

    if subtitle and (subtitle not in ['A Novel', 'A History', 'A Short Story']):
        series_str, sep, series_index_str = subtitle.partition(' Series, Book ')
        if not sep:
            series_str, sep, series_index_str = subtitle.partition(' Series, Books ')
        if not sep:
            series_str, sep, series_index_str = subtitle.partition(' Series, Volume ')
        if not sep:
            series_str, sep, series_index_str = subtitle.partition(' Series, Vol. ')
        if not sep:
            series_str, sep, series_index_str = subtitle.partition(', Book ')

        # 'volume' is sometimes part of title and sometimes series number
        if not sep:
            series_str, sep2, series_index_str = subtitle.partition(', Volume ')
            if not sep2:
                series_str, sep2, series_index_str = subtitle.partition(', Vol. ')
            if sep2 and (series_str in VOLUMES_WITH_UNIQUE_TITLES):
                sep = sep2  # volume is series

        if sep:
            # Subtitle is actually series info
            series = series_str
            series_index_str = series_index_str.partition(' of ')[0]
            try:
                series_index = float(series_index_str.strip())  # Check for numeric
            except ValueError:
                try:
                    series_index = roman_to_int(series_index_str.strip().upper())   # Check for Roman numeral
                except ValueError:
                    # non numeric
                    series = ''
                    series_index = 0.0

        if not series:
            title = '%s: %s' % (odtitle, subtitle)

    if edition:
        edition = re.sub(r'[()]', '', edition)
        edition = re.sub(r' Edition$', '', edition)

        if re.match(r'^[0-9]+$', edition):
            edition_num = int(edition)
            if edition_num >= 1 and edition_num <= 99:
                edition = numeric_rank(edition_num)     # 1-> 1st, 2 -> 2nd, etc.

        #  DGO = Digital original = digital exclusive
        if edition not in ['1st', 'Abridged', 'Unabridged', 'Unabridged Selections', 'ebook', 'Illustrated',
                           'Hemingway Library', 'DGO']:
            split_edition = edition.split()
            if len(split_edition) >= 2 and split_edition[1].lower() == 'anniversary':
                title = '%s (%s Anniversary Edition)' % (title, split_edition[0])
            else:
                title = '%s (%s Edition)' % (title, edition)

    if not series:
        series = odseries

    return (title, series, series_index)


def fix_html(page):
    # eliminate malformed comments causing parse errors

    stripped_page = re.sub(r'<!-[^\-].*?-->', '', page, flags=re.DOTALL)
    #self.log.info('stripped_page: %s' % stripped_page)

    return stripped_page


def check_maintenance(url):
    if hostname_from_url(url).startswith('maintenance'):
        raise LibraryError('Website down for maintenance')


class OverDriveLegacy(SearchableLibrary):
    id = ''
    name = 'OverDrive Legacy'

    ebook_formats_supported = {
        FORMAT_ADOBE_EPUB, FORMAT_OPEN_EPUB, FORMAT_OPEN_PDF,
        FORMAT_KINDLE_BOOK, FORMAT_MOBI_EBOOK, FORMAT_ADOBE_PDF, FORMAT_OD_READ,
        FORMAT_MEDIADO_READER}
    audiobook_formats_supported = {FORMAT_OD_LISTEN, FORMAT_OD_MP3, FORMAT_OD_WMA}
    video_formats_supported = {FORMAT_OD_VIDEO, FORMAT_OD_VIDEO_MOBILE, FORMAT_STREAMING_VIDEO}
    music_formats_supported = {FORMAT_OD_MUSIC, }
    periodical_formats_supported = {FORMAT_NOOK_PERIODICALS, }

    formats_supported = (ebook_formats_supported | audiobook_formats_supported |
                         video_formats_supported | music_formats_supported |
                         periodical_formats_supported)

    #supports_recommendation = True
    sign_in_affects_get_current_availability = True     # sign in needed to detect holds and Advantage titles

    @staticmethod
    def validate_library_id(library_id, migrate=True, config=None):
        if (':' in library_id) or ('/' in library_id):
            library_id = hostname_from_url(library_id)

        library_id = library_id.lower()

        if (not isValidHostname(library_id)) or (library_id == GENERIC_SEARCH_SITE):
            raise ValueError('OverDrive library hostname invalid: "%s"' % library_id)

        if '.' not in library_id:
            raise ValueError('OverDrive library hostname must contain a period: "%s"' % library_id)    # Hostname must contain a period

        return library_id

    @staticmethod
    def validate_book_id(book_id, library_id):
        if not re.match(r'^([0-9a-f]{8})\-([0-9a-f]{4})\-([0-9a-f]{4})\-([0-9a-f]{4})\-([0-9a-f]{12})$', book_id):
            raise ValueError('OverDrive book id must be a UUID: "%s"' % book_id)  # (550e8400-e29b-41d4-a716-446655440000)

        return book_id.lower()

    @staticmethod
    def book_url(library_id, book_id):
        return 'http://%s/ContentDetails.htm?ID=%s' % (library_id, book_id.upper())

    def default_url(self):
        # Default url for request to a library
        return 'http://%s' % self.library_id

    def reset_redirect_url(self):
        # Use the default url for new search request
        self.redirect_url = self.default_url()

    def update_redirect_url(self, url):
        # Requests to overdrive libraries return a redirect to a url that has embedded codes after the host name.
        # In order to avoid having each request result in a redirect we cache the redirect returned by the
        # last request for use by the next one. This is reset to the default for each new search.

        check_maintenance(url)

        new_redirect_url = '/'.join(url.split('/')[:-1])

        if new_redirect_url != self.redirect_url:
            if self.redirect_url != self.default_url() and new_redirect_url != self.default_url():
                # only report if change from non-default to new non-default
                self.log.warn('URL redirected from %s to %s' % (self.redirect_url, new_redirect_url))

            self.redirect_url = new_redirect_url

    def __init__(self):
        self.recommendation_allowed = False
        self.supports_isbn = False
        self.ready_for_search = False
        self.cookiejar = http.cookiejar.CookieJar()
        self.library_search_format_values = set()  # value combinations supported for search at this library

        self.holds_checked = False
        self.holds = {}

        self.library_format_names = {}  # translation to standard format names, key is lower case

        for format in self.formats_supported:
            self.library_format_names[format.lower()] = format
            self.library_format_names[format.lower() + 's'] = format

        for od_format, format in OVERDRIVE_FORMAT_NAMES.items():
            self.library_format_names[od_format.lower()] = format

    def sign_in(self, use_credentials):
        self.cookiejar.clear()
        self.reset_redirect_url()
        self.login_url = None
        original_library_id = self.library_id

        br = mechanize.Browser()
        br.set_cookiejar(self.cookiejar)

        if self.card_number and use_credentials:
            self.signin_required = True

            # ILSType = 1 (card number only), 2 (card number & pin), 3 (card & extra),
            # 4 (Card, PIN, and registration: NewPatronFlag=0|1), 5 (External Authentication)

            MAX_TRIES = 6
            tries = 0

            while True:
                if tries >= MAX_TRIES:
                    self.log.error('Sign in unsuccessful after %d tries.' % tries)
                    break

                if tries > 0:
                    time.sleep(10)  # give the problem time to clear before trying again

                tries += 1

                library_name = self.name if self.branch_id == '' else '%s (branch %s)' % (self.name, self.branch_id)
                self.log.info('Signing in to %s' % library_name)

                url = 'http://%s/BANGAuthenticate.dll?Action=AuthCheck&URL=Default.htm&ForceLoginFlag=0' % self.library_id
                data = browse_url(self.log, br, mechanize.Request(url))

                self.login_url = br.geturl()
                self.log.info('Redirected login url %s' % self.login_url)
                check_maintenance(self.login_url)

                if self.branch_id != '':
                    # For multi-branch libraries select the proper sign in page for the specific branch
                    url = self.login_url.replace('SignIn.htm?', 'SignIn2.htm?branchid=%s&' % self.branch_id, 1)
                    data = browse_url(self.log, br, mechanize.Request(url))

                # Select the Sign in form
                try:
                    br.select_form(predicate=lambda f: 'action' in f.attrs and f.attrs['action'] == 'BANGAuthenticate.dll')
                    # Exception if not found

                    if self.branch_id == '':
                        if 'id' in br.form.attrs and 'ILSForm' in str(br.form.attrs["id"]):
                            raise Exception()  # incomplete form - branch selection needed

                        # The LibraryCardILS field is filled with a shortened form of the branch name for consortia and with 'default' or
                        # a short library name for other libraries. May want to use this instead of branch id in the future if OverDrive
                        # changes the sign in process.
                        control_names = [c.name for c in br.form.controls if c.name]
                        if 'LibraryCardILS' in control_names and \
                                (br.form['LibraryCardILS'] == '' or br.form['LibraryCardILS'] == '@@ILS_NAME@@'):
                            raise Exception()  # incomplete form - branch selection needed

                except Exception:
                    soup = beautiful_soup_fix(data)
                    page_text = text_only(soup)

                    if self.branch_id == '':
                        branches = {}

                        # Search for text in delimiters indicating that the information is not normally displayed
                        if ('login to one of the libraries below' in page_text or
                                'please click the appropriate link' in page_text or
                                'please select your library from the list below' in page_text or
                                "please type your library's name and select it from the list" in page_text):

                            hidden_divs = soup.findAll('div', attrs={'class': re.compile('(skip)|(scriptOFF)')})
                            hidden_divs.extend(soup.findAll('noscript'))
                            for hidden in hidden_divs:
                                entries = hidden.findAll('a', attrs={'href': re.compile('SignIn2')})
                                for entry in entries:
                                    href = entry['href']
                                    query = urllib.parse.urlparse(href).query
                                    branch_ids = urllib.parse.parse_qs(query).get('branchid', [])
                                    if len(branch_ids) == 1:
                                        branches[text_only(entry)] = branch_ids[0]

                        if branches:
                            self.log.warn(('It appears that a branch ID is required for %s. Configure this library'
                                           ' in the plugin with the appropriate branch ID:') % self.library_id)

                            for branch_name in sorted(branches.keys()):
                                self.log.info('%s = %s' % (branch_name, branches[branch_name]))

                            raise LibraryError('Branch ID required for sign in - View log for a list of IDs')

                        continue        # retry

                    else:
                        # check for ILSType = 5 (External Authentication)
                        for anchor in soup.findAll('a'):
                            if 'please click here to sign into this system.' in text_only(anchor):
                                url = anchor['href']
                                self.log.info('Using external authentication url %s' % url)
                                browse_url(self.log, br, mechanize.Request(url))

                                # Select the external sign in form
                                try:
                                    if len([f for f in br.forms()]) == 1:
                                        br.select_form(nr=0)    # only one form on page
                                    else:
                                        br.select_form(predicate=lambda f: is_sign_in_form(f))  # Exception if not found

                                except Exception:
                                    raise LibraryError('Missing sign in form')

                                break
                        else:
                            raise LibraryError('Missing sign in form')

                # User credentials
                set_card_number(self.log, br.form, self.card_number)
                set_card_pin(self.log, br.form, self.card_pin)

                # Login
                try:
                    page = browse_url(self.log, br, None)
                except Exception as e:
                    # browser state is lost on error during submit so must retry from the beginning
                    self.log.warn('Sign in form submit exception %s' % repr(e))
                    continue        # retry sign in

                self.update_redirect_url(br.geturl())
                soup = beautiful_soup_fix(page)

                if 'account information is not valid' in page or 'return to the sign in form' in page or \
                        'return to the login form' in page:
                    # get more details of the failure
                    main_content = soup.find('section', id='mainContent')
                    if main_content:
                        error_text = text_only(main_content).partition(' Sign In ')[0]
                        self.log.info('Error: %s' % error_text)

                        if 'unable to connect to the library authentication server' in error_text:
                            self.log.warn('Library authentication server problem')
                            continue        # retry sign in

                    self.log.error('Sign in failed. Check card number and PIN.')
                    break

                elif 'incorrect.  Please try again' in page:
                    # External Authentication failure
                    self.log.info('Error: %s' % text_only(soup))
                    self.log.error('External authentication sign in failed. Check card number and PIN.')
                    break

                elif not ('Log Out' in page or 'Sign Out' in page):
                    # unknown problem
                    self.log.info('Error: %s' % text_only(soup))
                    self.log.error('Sign in unsuccessful.')
                    break

                else:
                    self.log.info('Sign in successful')
                    self.signed_in = True
                    break

        #self.log.info('Checking capabilities of %s'%self.name)

        results, new_url = self.establish_search_session()
        #self.log.info('Response url: %s' % new_url)

        if ('/SignIn.htm' in new_url) or ('/SignIn2.htm' in new_url) or ('<h1>Protected Development Site</h1>' in results):
            self.signin_required = True
            self.signed_in = False
            self.log.error('Library %s requires sign in to perform searches.' % self.name)
            return

        if '/AdvancedSearch.htm' not in new_url:
            raise LibraryError('Failed to locate advanced search page for library.'
                               ' Configuration update required. See Help for details.')

        if self.library_id != original_library_id:
            self.log.error(('Incorrect library id in configuration! Host name %s is redirected to %s.'
                            ' Configuration change followed by "Check and repair book links" is required. See Help for details.') % (
                    original_library_id, self.library_id))

        # Parse the html results for analysis
        soup = beautiful_soup_fix(results)

        # Looking in advanced search for option to search for recommendable books.
        # looking for: var hazMoreCatalog = 0; -or- = 1;
        scripts = soup.findAll('script')
        for script in scripts:
            #self.log.info('script: %s' % str(script))
            split_script = str(script).split()
            if 'hazMoreCatalog' in split_script:
                i = split_script.index('hazMoreCatalog')
                if split_script[i-1] != 'var':
                    raise LibraryError('No "var" before "hazMoreCatalog"')

                if split_script[i+1] != '=':
                    raise LibraryError('No "=" after "hazMoreCatalog"')

                morecatalog = re.sub('[";]', '', split_script[i+2].lower())   # Strip any quoting from value

                if morecatalog != '0':
                    self.log.info('Search for recommendable books is supported by %s' % self.name)
                    self.recommendation_allowed = True
                else:
                    self.log.info('Search for recommendable books is NOT supported by %s' % self.name)

                break

        else:
            raise LibraryError('Missing hazMoreCatalog on search page')

        self.supports_isbn = soup.find('input', attrs={'name': 'ISBN'}) is not None  # Added to OverDrive 6/19/14
        if not self.supports_isbn:
            self.log.info('Search by ISBN is NOT supported by %s' % self.name)

        format_select = soup.find('select', attrs={'name': 'Format'})
        if format_select:
            for option in format_select.findAll('option'):
                format_text = text_only(option)
                format_value = option['value']

                if len(format_value) > 0:
                    self.library_search_format_values.add(format_value)

                    if tweaks.get(TWEAK_SAVE_RESPONSES_ON_ERROR, False):
                        # warn if testing
                        for format_num in format_value.split(','):
                            if format_num and (format_num not in OVERDRIVE_FORMAT_NUMBERS):
                                self.log.warn('Unexpected format number %s for "%s"' % (format_num, format_text))

                    if ((not format_text.startswith('All ')) and
                            format_value in OVERDRIVE_FORMAT_NUMBERS and
                            format_text.lower() not in self.library_format_names):

                        self.log.info('Format "%s" at %s is actually "%s" (%s)' % (
                            format_text, self.name, OVERDRIVE_FORMAT_NUMBERS[format_value], format_value))
                        self.library_format_names[format_text.lower()] = OVERDRIVE_FORMAT_NUMBERS[format_value]
        else:
            raise LibraryError('Missing Format selection on search page')

        self.ready_for_search = True

    def establish_search_session(self):
        # Open search page to establish a session and set cookies
        self.reset_redirect_url()
        tried_urls = set()
        url = 'http://%s/AdvancedSearch.htm' % self.library_id
        host_redirect_required = False

        while True:
            if url in tried_urls:
                # prevent infinite loops of redirection
                raise LibraryError('Library website has redirection loop.')

            tried_urls.add(url)

            self.log.clear_response()

            response = open_url(self.log, url, cookiejar=self.cookiejar, expect_errors=[404])

            if response.is_httperror_exception:
                #self.log.exception('Establish search session', response)

                if not host_redirect_required:
                    # A 404 (Not Found) error can occur for some sites that are redirected to another host name
                    # when a specific page is requested instead of the site's home page. Check for this.
                    self.log.info('Possible website redirection - retrying using home page')

                    url = 'http://%s/' % self.library_id
                    host_redirect_required = True
                    continue

                raise response

            redirect_url = response.geturl()
            #self.log.info('Redirected to: %s' % redirect_url)

            check_maintenance(redirect_url)

            # URLs have appended after the host name: /session-id/xx/yy/lang/
            # session-id uuid
            # xx=10 for regular site,20 for mobile site?
            # yy=50 for regular site, 42, for mobile??, 45 for children's site
            # lang=en

            # add qualifiers needed for some sites (El Rancho)
            # http://elrancho.lib.overdrive.com/AdvancedSearch.htm without cookies calls up simplified search page (mobile?)
            # http://elrancho.lib.overdrive.com/E5F22191-0672-476B-90B3-D9A87AF65804/10/42/en/AdvancedSearch.htm that loads via js
            # http://elrancho.lib.overdrive.com/E5F22191-0672-476B-90B3-D9A87AF65804/10/45/en/default.htm

            if '/10/42/' in redirect_url and '.libraryreserve.com/' not in redirect_url:
                url = redirect_url.replace('/10/42/', '/10/45/', 1)
                host_redirect_required = False
                continue

            redirect_host = hostname_from_url(redirect_url).lower()

            # Make sure that the host name entered by the user matches the host name returned.
            # If not, future searches will fail because post data will be lost during redirection.

            if redirect_host == GENERIC_SEARCH_SITE:
                raise LibraryError(
                        'Non-working library id in configuration!'
                        ' Host name %s is redirected to generic Overdrive site %s.' % (self.library_id, redirect_host))

            if redirect_host == self.library_id:
                if host_redirect_required:
                    raise LibraryError(
                            'Failed to locate advanced search page for library.'
                            ' Possible incorrect library id or change of library website to OverDrive 2016.'
                            ' Configuration update followed by "Check and repair book links" is required.')

                break  # success

            self.library_id = redirect_host  # fixup for now

            #check for further redirection
            url = 'http://%s/AdvancedSearch.htm' % self.library_id
            host_redirect_required = False
            continue

        self.update_redirect_url(redirect_url)
        return response.data_string, redirect_url

    def find_books(self, books, search_author, search_title, keyword_search):
        '''
        Search OverDrive for books that match an author/title (or subsets thereof) and return the found matches with identifiers.
        Some books will have multiple identifiers due to different editions/publishers

        books = Set of LibraryBooks to be updated
        '''

        if not self.ready_for_search:
            return False

        if len(search_author) < 4 and not search_title:
            return False     # Very short author names return bad results

        #find_recommendable = self.config.check_recommendable and self.recommendation_allowed
        find_recommendable = False

        if keyword_search:
            if find_recommendable:
                return False  # combination not supported for OverDrive

            keywords = [p for p in re.split(r'( |".*?")', search_title) if p.strip()]

        search_ebook = len(self.config.search_formats & self.ebook_formats_supported) > 0
        search_audiobook = len(self.config.search_formats & self.audiobook_formats_supported) > 0
        search_video = len(self.config.search_formats & self.video_formats_supported) > 0
        od_formats = self.config.search_formats & self.formats_supported

        RESULTS_PER_PAGE = 20
        MAX_RETRIES = 10    # For OverDrive errors
        MAX_RESULTS_ALLOWED = 500

        # availabilityoptions=more misses some available books so need to search twice!
        for do_recommendable in [False, True]:
            if do_recommendable and not find_recommendable:
                continue

            page_num = 1
            total_pages = 1
            total_results = 0
            results_processed = 0
            retry_count = 0
            keywords_used = 0
            start_search = True

            #self.reset_redirect_url()

            while (page_num <= total_pages):

                if start_search:
                    # start (or restart) a search

                    page_num = 1
                    total_pages = 1
                    total_results = 0
                    results_processed = 0

                    page_url = ''

                    if keyword_search:
                        # use non-advanced search
                        # BANGSearch.dll?Type=FullText&PerPage=20&URL=SearchResultsList.htm
                        #       Sort=SortBy%3DRelevancy&FullTextField=All&FullTextCriteria=keyword&x=11&y=11

                        url = '%s/BANGSearch.dll?Type=FullText&PerPage=%d&URL=SearchResultsList.htm' % (self.redirect_url, RESULTS_PER_PAGE)

                        data = {}
                        data['Sort'] = 'SortBy=Relevancy'
                        data['FullTextField'] = 'All'
                        data['FullTextCriteria'] = keywords[0]  # only 1st word

                        data = urllib.parse.urlencode(data)

                        keywords_used = 1

                    else:
                        # Search form based on: http://xxx.lib.overdrive.com/AdvancedSearch.htm (10/2013)

                        url = '%s/BANGSearch.dll?URL=SearchResultsList.htm' % self.redirect_url

                        data = {}
                        data['Sort'] = 'SortBy=Relevancy'
                        data['PerPage'] = '%d' % RESULTS_PER_PAGE
                        data['Title'] = double_quote(search_title)     # Double quote for exact match
                        data['Creator'] = double_quote(search_author)  # Double quote for exact match

                        if self.supports_isbn:
                            data['ISBN'] = ''       # Added to OverDrive 6/19/14, not yet present at all sites

                        data['CollDate'] = ''
                        data['Subject'] = ''

                        # Use the least broad of the detected format searches that give all of the desired search formats.
                        # (Some will never be selected since we search for both DRM and open formats simultaneously.)
                        for format_value in sorted(self.library_search_format_values, key=len):
                            covered_formats = set()
                            for format_num in format_value.split(','):
                                if format_num in OVERDRIVE_FORMAT_NUMBERS:
                                    covered_formats.add(OVERDRIVE_FORMAT_NUMBERS[format_num])

                            # special cases
                            if FORMAT_KINDLE_BOOK in covered_formats:
                                covered_formats.add(FORMAT_MOBI_EBOOK)

                            #self.log.info('Test formats: value=%s missing=%s' % (
                            #    format_value, ','.join(list(od_formats - covered_formats))))

                            if len(od_formats - covered_formats) == 0:
                                data['Format'] = format_value   # this format combo covers all that we need
                                break
                        else:
                            data['Format'] = ''     # All formats

                        if self.config.search_language in LANGUAGE_CODE2:
                            data['Language'] = LANGUAGE_CODE2[self.config.search_language]
                        else:
                            data['Language'] = ''

                        data['Publisher'] = ''
                        data['Award'] = ''
                        data['interestlevel'] = ''
                        data['AtosBooklevel'] = ''
                        data['Lexilelevel'] = ''
                        data['GradeLevel'] = ''
                        data['contentmaturity'] = ''

                        if do_recommendable:
                            # available to recommend for library acquisition (even if already in collection)
                            data['availabilityoptions'] = 'more'
                        else:
                            # in collection, Use "availcopies" for only those available for checkout now
                            data['availabilityoptions'] = ''

                        data = urllib.parse.urlencode(data)

                    start_search = False

                elif keyword_search and (keywords_used < len(keywords)):
                    # add keywords by searching within results
                    #BANGSearch.dll?Type=FreeForm&SearchID=13279351s&PerPage=20&URL=SearchResultsList.htm&SortBy=relevancy
                    #SearchResult=13279351s&FreeFormFields=title%2Ccreator%2Ckeyword&FreeFormCriteria=holmes

                    page_query = urllib.parse.parse_qs(urllib.parse.urlparse(page_url).query)

                    if 'SearchID' in page_query:
                        search_id = page_query['SearchID'][0].encode('utf-8')
                    elif 'searchid' in page_query:
                        search_id = page_query['searchid'][0].encode('utf-8')
                    else:
                        raise LibraryError('missing SearchID in page_url: %s' % page_url)

                    query = {}
                    query['Type'] = 'FreeForm'
                    query['SearchID'] = search_id
                    query['PerPage'] = '%d' % RESULTS_PER_PAGE
                    query['URL'] = 'SearchResultsList.htm'
                    query['SortBy'] = 'relevancy'

                    url = '%s/BANGSearch.dll?%s' % (self.redirect_url, urllib.parse.urlencode(query))
                    #match = re.search(r'SearchID=(.*?)&', page_url+'&', flags=re.IGNORECASE)
                    #if not match:
                    #    raise LibraryError('missing SearchID in page_url: %s' % page_url)
                    #
                    #search_id = str(match.group(1))

                    data = {}
                    data['SearchResult'] = search_id
                    data['FreeFormFields'] = 'title,creator,keyword'
                    data['FreeFormCriteria'] = keywords[keywords_used]  # only next word

                    data = urllib.parse.urlencode(data)

                    keywords_used += 1
                    page_num = 1

                else:
                    # subsequent page or retry of first page
                    url = page_url

                    if 'SortOrder=' not in url:
                        url += '&SortOrder=desc'    # required to prevent missed books when results are more than one page

                    if '&Page=' in url:
                        url = re.sub(r'&Page=[0-9]+', '&Page=%d' % page_num, url)
                    else:
                        url += '&Page=%d' % page_num

                    data = None

                response = open_url(self.log, url, data, cookiejar=self.cookiejar,
                                    origin='http://%s' % self.library_id, referer='%s/AdvancedSearch.htm' % self.redirect_url,
                                    addheaders=[('Accept-Language', 'en-US,en;q=0.8')])

                page_url = response.geturl()
                #self.log.info('Returned page url: %s' % page_url)

                self.update_redirect_url(page_url)

                # Parse the html results for analysis
                strainer = soup_strainer(name=re.compile('div|section'), attrs={'id': re.compile('searchResults|mainContent')})
                soup = beautiful_soup(response.data_string, parse_only=strainer)

                main_content = soup.find('section', id='mainContent')
                if main_content:
                    main_content_text = text_only(main_content)
                    if main_content_text.find('No results were found for your search.') >= 0:
                        break

                    if main_content_text.find('we are unable to display the page requested.') >= 0:
                        self.log.warn('Unable to display the page requested')

                        self.establish_search_session()

                        retry_count += 1
                        if retry_count > MAX_RETRIES:
                            raise LibraryError('Bad search results from OverDrive - retry later')

                        start_search = True     # Retry search from the beginning
                        continue

                # There appears to be a problem at OverDrive causing results to sometimes not adhere to the search criteria
                # causing excessive extraneous results to be returned. This can happen on a page-by-page basis in the
                # response. Retrying can eventually get it to return the correct filter criteria for the search.

                #faceted_search = soup.find('div', id='facetedSearch02')
                # This the the part of the results page that allows restrictions to be applied after-the-fact

                # This the the part of the results page that allows restrictions to be applied after-the-fact
                faceted_search = must_find(soup, 'div', attrs={'id': re.compile('facetedSearch')})

                if not re.search('var nFacet[0-9]+ = "1";', str(faceted_search)):
                    # This is disabled on the page. Assume bad results.
                    #self.log.info('facetedSearch: %s' % str(faceted_search))
                    self.log.warn('Search filters not applied. Reloading response page for retry.')

                    retry_count += 1
                    if retry_count > MAX_RETRIES:
                        raise LibraryError('Bad search results - retry later')

                    continue        # retry page

                retry_count = 0

                if keyword_search and (keywords_used < len(keywords)):
                    continue        # restrict search with additional keywords

                search_results = must_find(soup, 'div', attrs={'id': 'searchResults'})
                result_table = must_findAll(search_results, 'div', attrs=class_contains('row'), recursive=True)

                # determinate total number of pages of results
                for result_row in result_table:
                    if result_row.get('id', '') in ['resultsPagingTop', 'resultsPagingBtm']:
                        paging_title_count_div = result_row.find('div', id='pagingTitleCount')
                        if paging_title_count_div:
                            # EG: "Number of Results1-20 of 346"
                            paging_title_count_text = re.sub(r'([0-9]+)', r' \1 ', text_only(paging_title_count_div))
                            paging_title_count_list = paging_title_count_text.split()

                            if (len(paging_title_count_list) < 8 or paging_title_count_list[0] != 'Number' or
                                    paging_title_count_list[1] != 'of' or paging_title_count_list[2] != 'Results' or
                                    paging_title_count_list[4] != '-' or paging_title_count_list[6] != 'of'):
                                raise LibraryError('Unexpected pagingTitleCount %s' % paging_title_count_text)

                            first_result = int(paging_title_count_list[3])
                            if first_result != results_processed + 1:
                                raise LibraryError('Unexpected first result %d instead of %d' % (first_result, results_processed + 1))

                            new_total_results = int(paging_title_count_list[7])
                            if total_results and (new_total_results != total_results):
                                raise LibraryError('Total results changed from %d to %d' % (total_results, new_total_results))

                            total_results = new_total_results
                            total_pages = ((total_results - 1) // RESULTS_PER_PAGE) + 1  # floor division

                            self.log.info('Response: page %d of %d. %d total results' % (page_num, total_pages, total_results))

                            if total_results > MAX_RESULTS_ALLOWED:
                                return True

                            break
                else:
                    raise LibraryError('Missing pagingTitleCount')

                for result_row in result_table:
                    #self.log.info('Result result_row: %s' % str(result_row))

                    if result_row.get('id', '') in [
                            'resultsPagingTop', 'resultsPagingTopMobile', 'resultsPagingBtm', 'resultsPagingBtmMobile']:
                        # Ignore rows that are not actual results
                        continue

                    result_list = result_row.find('ul')
                    if not result_list:
                        # Ignore rows that are not actual results
                        continue

                    result_list_items = result_list.findAll('li', attrs={'class': re.compile('search-result')}, recursive=False)

                    for result_list_item in result_list_items:
                        # Start of book info
                        crid = ''
                        authors = []
                        title = ''
                        subtitle = ''
                        available = None
                        fmtclass = 'Unknown format'
                        desired_format = False

                        anchor = must_find(result_list_item, 'a')

                        if not re.match('CRID', anchor['name']):
                            raise LibraryError('Unexpected result item anchor: %s' % anchor['name'])

                        crid = anchor['name'][4:].lower()
                        if not re.match(r'^([0-9a-f]{8})\-([0-9a-f]{4})\-([0-9a-f]{4})\-([0-9a-f]{4})\-([0-9a-f]{12})$', crid):
                            # Crid format incorrect (550e8400-e29b-41d4-a716-446655440000)
                            raise LibraryError('CRID is not a UUID: "%s"' % crid)

                        # Search for results in javascript: var szLibOwned = "1"; | "0" | "always available"
                        script = must_find(result_list_item, 'script')

                        split_script = str(script).split()
                        if 'szLibOwned' not in split_script:
                            raise LibraryError('Missing szLibOwned')

                        i = split_script.index('szLibOwned')
                        if split_script[i-1] != 'var':
                            raise LibraryError('No "var" before "szLibOwned"')

                        if split_script[i+1] != '=':
                            raise LibraryError('No "=" after "szLibOwned"')

                        libowned = re.sub('[";]', '', split_script[i+2].lower())   # Strip quoting from value

                        if libowned == '0':
                            available = False
                        elif libowned[:6] == 'always' or re.match('^[0-9]+$', libowned):
                            available = True
                        else:
                            raise LibraryError('Unexpected szLibOwned = %s' % libowned)

                        cover_div = must_find(result_list_item, 'div', attrs={'class': 'coverID'})

                        # as of 5/29/2014
                        fmtclass = cover_div.get('data-fmtclass')
                        if fmtclass:
                            if fmtclass == 'eBook':
                                if search_ebook:
                                    desired_format = True
                            elif fmtclass == 'Audiobook':
                                if search_audiobook:
                                    desired_format = True
                            elif fmtclass == 'Video':
                                if search_video:
                                    desired_format = True
                            elif fmtclass == 'Music':
                                pass
                            else:
                                raise LibraryError('Unknown data-fmtclass %s' % fmtclass)

                        # prior to 5/29/2014
                        for cover_image in cover_div.findAll('img'):
                            #self.log.info('Cover image: %s' % str(cover_image))

                            fmtclass = cover_image.get('data-fmtid')
                            if fmtclass:
                                if fmtclass == 'formateBook':
                                    if search_ebook:
                                        desired_format = True
                                elif fmtclass == 'formatAudiobook':
                                    if search_audiobook:
                                        desired_format = True
                                else:
                                    raise LibraryError('Unknown data-fmtid %s' % fmtclass)

                        title_div = result_list_item.find('div', attrs={'class': 'trunc-title-line-list'})
                        if title_div:
                            title = text_only(title_div)

                        subtitle_div = result_list_item.find('div', attrs={'class': 'trunc-subtitle-line-list'})
                        if subtitle_div:
                            subtitle = text_only(subtitle_div)

                        author_div = result_list_item.find('div', attrs={'class': 'trunc-author-line-list'})
                        if author_div:
                            author = normalize_author(text_only(author_div), unreverse=False)
                            authors.append(author)  # Seems only the first author is shown in results list.

                            # Creator id - possible future use
                            #<div class="trunc-author-line-list" title="Alexander McCall Smith" alt="Alexander McCall Smith">
                            #<a href="http://pplc.lib.overdrive.com/EF74DAEA-ED08-43BD-AC34-37057DDD70F0/10/50/en/BANGSearch.dll?
                            #Type=Creator&ID=238694&PerPage=20&URL=SearchResultsList.htm"
                            #title="Search for other content by Alexander McCall Smith">Alexander McCall Smith</a></div>

                        series_div = result_list_item.find('div', attrs={'class': 'trunc-series-line-list'})
                        if series_div:
                            series = text_only(series_div)

                            # Series id - possible future use
                            #<div class="trunc-series-line-list">Series:&nbsp;
                            #<a href="http://pplc.lib.overdrive.com/EF74DAEA-ED08-43BD-AC34-37057DDD70F0/10/50/en/BANGSearch.dll?
                            #Type=Series&ID={501A51EF-10D9-4778-9FF3-61D35B3C8C00}&SortBy=CollDate&PerPage=20&URL=SearchResultsList.htm"
                            #class="seriesColbox">The No. 1 Ladies' Detective Agency</a></div>

                        # The language restriction appears to work so make book language = search language

                        title, series, series_index = fix_title(title, subtitle)
                        title = normalize_title(title)

                        lbook = LibraryBook(
                                authors=authors, title=title,
                                available=available, recommendable=(do_recommendable and not available),
                                lib=self, book_id=crid, series=series, series_index=series_index,
                                language=self.config.search_language, search_author=search_author)

                        if not desired_format:
                            self.log.info('Ignoring %s: %s' % (fmtclass, repr(lbook)))
                        elif not (available or do_recommendable):
                            self.log.info('Ignoring unavailable %s: %s' % (fmtclass, repr(lbook)))
                        else:
                            self.log.info('Found %s: %s' % (fmtclass, repr(lbook)))
                            books.add(lbook)

                        results_processed += 1

                page_num += 1

            if results_processed != total_results:
                raise LibraryError('Expected %s but found %d' % (value_unit(total_results, 'result'), results_processed))

        return False

    def get_overdrive_page(self, query, secure=False, use_strainer=True):
        redirect_url = self.login_url.rpartition('/')[0] if secure else self.redirect_url
        response = open_url(self.log, '%s/%s' % (redirect_url, query), cookiejar=self.cookiejar)

        if not secure:
            page_url = response.geturl()
            #self.log.info('Returned page url: %s' % page_url)

            self.update_redirect_url(page_url)

        html = fix_html(response.data_string)

        # Quick and dirty fix for pages with unterminated CDATA sections
        num_cdata = html.count("<![CDATA[")
        num_cdata_term = html.count("]]>")
        if num_cdata > num_cdata_term:
            # eliminate possible unterminated CDATA
            self.log.info('Page has %s but only %s' % (
                    value_unit(num_cdata, 'CDATA section'), value_unit(num_cdata_term, 'terminator')))

            html = html.replace("<![CDATA[", "").replace("]]>", "")

        if use_strainer:
            strainer = soup_strainer(name=re.compile('section'), attrs={'id': re.compile('mainContent')})
        else:
            strainer = None

        return beautiful_soup(html, parse_only=strainer)

    def get_book_page(self, book_id, query='ContentDetails.htm', use_strainer=True):
        return self.get_overdrive_page('%s?ID=%s' % (query, book_id.upper()), use_strainer=use_strainer)

    def get_book_info(self, book_id, cache):
        authors = []
        title = ''
        subtitle = ''
        edition = ''
        publisher = ''
        pubdate = None
        isbn = ''
        formats = set()

        soup = self.get_book_page(book_id)
        main_content = must_find(soup, 'section', attrs={'id': 'mainContent'})

        title_div = main_content.find('div', attrs={'id': 'detailsTitle'})
        if title_div:
            title = text_only(title_div)

        subtitle_div = main_content.find('div', attrs={'id': 'subtitleDetails'})
        if subtitle_div:
            subtitle = text_only(subtitle_div)

        edition_div = main_content.find('div', attrs={'id': 'editionExpand'})
        if edition_div:
            edition_items = edition_div.findAll('li')
            edition = ' '.join([text_only(edition_item) for edition_item in edition_items])

        creator_divs = main_content.findAll('div', attrs={'id': re.compile('creatorDetails|creatorSubDetails')})
        for creator_div in creator_divs:
            creator_list = creator_div.findAll('li')

            if not creator_list:
                creator_list = [creator_div]

            for creator_li in creator_list:
                author = normalize_author(text_only(creator_li), unreverse=False)
                #self.log.info('normalize author: %s -> %s' % (text_only(creator_li), author))

                if author and (author not in authors):
                    authors.append(author)
                    #self.log.info('  author=%s'%author)

        formats_div = main_content.find('div', attrs={'id': 'formatsAtDownload'})
        if formats_div:
            format_list_items = formats_div.findAll('li')
            for format_list_item in format_list_items:
                od_format = text_only(format_list_item)

                format = self.library_format_names.get(od_format.lower(), None)
                if format is not None:
                    formats.add(format)
                else:
                    self.log.warn('Unknown book format: %s' % od_format)

        format_info_div = main_content.find('div', attrs={'id': 'formatInfoExpand'})
        if format_info_div:
            publisher_div = format_info_div.find('div', attrs={'class': re.compile('publisher-info')})
            if publisher_div:
                publisher = text_only(publisher_div)
                #self.log.info('  publisher=%s'%publisher)

            div_lefts = format_info_div.findAll('div', attrs={'class': re.compile('titleInfoLeft')})
            div_rights = format_info_div.findAll('div', attrs={'class': re.compile('titleInfoRight')})
            if len(div_lefts) != len(div_rights):
                raise LibraryError('L/R mismatch: %d titleInfoLeft, %d titleInfoRight' % (len(div_lefts), len(div_rights)))

            for div_left, div_right in zip(div_lefts, div_rights):
                key = text_only(div_left)
                if key == 'ISBN:':
                    isbn = text_only(div_right)
                elif key == 'Release date:':
                    pubdate = parse_only_date(text_only(div_right), assume_utc=True)

            split_page = str(format_info_div).split()

            if not publisher:
                # Search for results in javascript: var thePublisher = "Random House Publishing Group";
                if 'thePublisher' in split_page:
                    i = split_page.index('thePublisher')
                    if split_page[i-1] == 'var' and split_page[i+1] == '=':
                        publisher = re.sub(r'[";]', '', split_page[i+2])   # Strip quoting from value
                        #self.log.info('  publisher2=%s'%publisher)

            if not isbn:
                # Search for results in javascript: var szISBN = "9781436201667";
                if 'szISBN' in split_page:
                    i = split_page.index('szISBN')
                    if split_page[i-1] == 'var' and split_page[i+1] == '=':
                        isbn = valid_isbn(re.sub(r'[";]', '', split_page[i+2]))   # Strip quoting from value
                        #self.log.info('  isbn2=%s'%isbn)

        publisher = re.sub(r'\[Start\]', '', publisher).strip()    # Handle: Night Shade Books[Start]

        title, series, series_index = fix_title(title, subtitle, edition)
        title = normalize_title(title)

        # only cache available books since format info not always correct for recommendable
        # (Works because get_book_info only called for recommendable library if book not available at any.)
        cache_allowed = False
        lib_copies_div = main_content.find('div', attrs={'class': 'row details-lib-copies'})
        if lib_copies_div:
            lib_copies_txt = text_only(lib_copies_div).partition(':')[2]    # number or 'always available'
            if lib_copies_txt != '0':
                cache_allowed = True

        return InfoBook(authors=authors, title=title, series=series, series_index=series_index, isbn=isbn,
                        publisher=publisher, pubdate=pubdate, formats=formats,
                        lib=self, book_id=book_id, cache_allowed=cache_allowed)

    def get_current_book_availability(self, book_id):
        self.check_current_holds()

        #consortium & advantage: copies owned, copies available, patrons on hold???
        #general: release date (for pre-release titles), on hold by user, checked out by user, always available y/n???

        library_copies = 1
        available_copies = 0
        have_checked_out = False
        number_waiting_overall = None
        hold_position_overall = None
        release_date = None
        is_on_hold = book_id in self.holds

        if is_on_hold:
            self.log.info('Using availability info from %s holds list for %s' % (self.name, book_id))
            hold_position_overall, release_date = self.holds[book_id]

        soup = self.get_book_page(book_id)
        main_content = must_find(soup, 'section', attrs={'id': 'mainContent'})

        if "This title is not available." in text_only(main_content):
            self.log.info("This title is not available")
            return False

        lib_owned = self.od_js_value(soup, 'szLibOwned')
        if lib_owned == "0":
            self.log.info("This title is not owned")
            return False

        available_copies = available_number(must_find(main_content, 'div', attrs={'id': 'availableCopies'}))
        library_copies = available_number(must_find(main_content, 'div', attrs={'id': 'libCopies'}))

        if not main_content.find('div', attrs={'id': 'detailsWishBtn'}):
            read_button_details = must_find(main_content, 'div', attrs={'id': 'readButtonDetails'})
            details_title_button = must_find(read_button_details, 'a', attrs=class_contains('details-title-button'))
            button_text = text_only(details_title_button)

            have_checked_out = details_title_button.get('data-checkedout', '0') != '0' or button_text == 'Go to Bookshelf'

            can_borrow_now = details_title_button.get('id', '') == 'borrowButton' or button_text == 'Borrow'
            if can_borrow_now and available_copies == 0:
                self.log.warn('"Borrow" with %s available copies' % available_copies)
                available_copies = 1

            if button_text == 'Place a Hold' and available_copies != 0:
                self.log.warn('"Place a Hold" with %s available copies' % available_copies)
                available_copies = 0

        # "Go to Holds" label is optionally written by javascript and not easily obtained.

        num_waiting_str = self.od_js_value(main_content, 'deNumWaiting')
        number_waiting_overall = 0 if num_waiting_str is None else int(num_waiting_str)

        # deHoldRatio is number waiting per copy as shown on the page "2 people waiting per copy"

        if release_date is None:
            pre_release_str = self.od_js_value(main_content, 'g_bFutureOnSaleDate')
            pre_release = False if pre_release_str is None else int(pre_release_str) != 0

            if pre_release:
                format_info_div = main_content.find('div', attrs={'id': 'formatInfoExpand'})
                if format_info_div:
                    div_lefts = format_info_div.findAll('div', attrs={'class': re.compile('titleInfoLeft')})
                    div_rights = format_info_div.findAll('div', attrs={'class': re.compile('titleInfoRight')})
                    if len(div_lefts) != len(div_rights):
                        raise LibraryError('L/R mismatch: %d titleInfoLeft, %d titleInfoRight' % (len(div_lefts), len(div_rights)))

                    for div_left, div_right in zip(div_lefts, div_rights):
                        key = text_only(div_left)
                        if key == 'Release date:':
                            release_date = parse_only_date(text_only(div_right), assume_utc=True)

                # try to get release date from pop-up if not available on page
                if release_date is None:
                    release_date = self.get_pre_release_date(book_id)

        # estimate availability
        wait_weeks = self.calculate_wait_weeks(
                library_copies=library_copies, available_copies=available_copies,
                number_waiting_overall=number_waiting_overall, release_date=release_date,
                have_checked_out=have_checked_out, hold_position_overall=hold_position_overall)

        return (wait_weeks, is_on_hold, book_id)

    def check_current_holds(self):
        if self.holds_checked or not self.signed_in:
            return

        self.holds_checked = True

        try:
            soup = self.get_overdrive_page('MyAccount.htm?PerPage=40#myAccount2', secure=True)   # tab 2 is holds

            holds_tab = must_find(soup, 'li', attrs={'id': 'myAccount2Tab'})

            for hold in holds_tab.findAll('li', attrs=class_contains('hold-title-contain')):
                # hold items all have id="hold@title.Metadata.ReserveID"
                # No special handling for suspended holds!

                book_id = hold['data-holdcrid']

                hold_position_overall = int(hold['data-holdposwait'])

                release_date = None

                for span in hold.findAll('span', recursive=True):
                    span_text = text_only(span)
                    if span_text.startswith('This title will be available'):
                        release_date = parse_only_date(span_text.rpartition(' of: ')[2], assume_utc=True)
                        break

                    if span_text.startswith('*Pre-Release Title'):
                        release_date = self.get_pre_release_date(book_id)
                        break

                # save for later use
                self.log.info('Found hold at %s: book_id=%s position_overall=%s release_date=%s' % (
                            self.name, book_id, hold_position_overall, str(release_date)[:10]))

                self.holds[book_id] = (hold_position_overall, release_date)

        except Exception as e:
            self.log.exception('', e)

    def get_pre_release_date(self, book_id):
        pre_release_soup = self.get_book_page(book_id, query='ContentDetails-StreetDate.htm', use_strainer=False)
        noscript = pre_release_soup.find('noscript')
        if noscript:
            return parse_only_date(text_only(noscript), assume_utc=True)

        red_text = pre_release_soup.find('font', attrs={'color': 'red'})
        if red_text:
            return parse_only_date(text_only(red_text), assume_utc=True)

        raise LibraryError('Cannot find pre-release available date for %s' % book_id)

    def od_js_value(self, soup, var):
        for script in soup.findAll('script'):
            #self.log.info('script: %s' % str(script))
            split_script = str(script).split()

            if var in split_script:
                i = split_script.index(var)
                if split_script[i-1] != 'var':
                    continue

                if split_script[i+1] != '=':
                    continue

                return re.sub(r'[";]', '', split_script[i+2].lower())   # Strip any quoting from value

        return None


def available_number(element):
    element_text = text_only(element)
    if element_text.lower() == 'always available':
        return True

    return int(element_text)
