#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai

from __future__ import (unicode_literals, division, absolute_import, print_function)

import bz2
import json
import mechanize
import os
import re
import time
import traceback

from calibre.constants import (config_dir, DEBUG)
from calibre.utils.date import parse_only_date
from calibre.utils.config import JSONConfig
from calibre.utils.config_base import tweaks

from calibre_plugins.overdrive_link.json import js_value
from calibre_plugins.overdrive_link.numbers import value_unit
from calibre_plugins.overdrive_link.book import (InfoBook, LibraryBook)
from calibre_plugins.overdrive_link.formats import (
    FORMAT_ADOBE_EPUB, FORMAT_ADOBE_PDF, FORMAT_KINDLE_BOOK, FORMAT_MOBI_EBOOK, FORMAT_OD_MP3,
    FORMAT_OD_MUSIC, FORMAT_OD_READ, FORMAT_OD_READ_ALONG, FORMAT_OD_MAGAZINE, FORMAT_NOOK_PERIODICALS,
    FORMAT_OD_WMA, FORMAT_OD_VIDEO, FORMAT_OD_VIDEO_MOBILE, FORMAT_OPEN_EPUB, FORMAT_OPEN_PDF,
    FORMAT_STREAMING_VIDEO, FORMAT_MEDIADO_READER, FORMAT_OD_LISTEN)
from calibre_plugins.overdrive_link.language import LANGUAGE_CODE2
from calibre_plugins.overdrive_link.library import SearchableLibrary
from calibre_plugins.overdrive_link.log import html_to_text
from calibre_plugins.overdrive_link.net import (browse_url, open_url, hostname_from_url, isValidHostname)
from calibre_plugins.overdrive_link.author_prep import normalize_author
from calibre_plugins.overdrive_link.title_prep import normalize_title
from calibre_plugins.overdrive_link.overdrive_legacy import (OverDriveLegacy, fix_title)
from calibre_plugins.overdrive_link.overdrive_sites import (known_new_overdrive_sites, equivalent_overdrive_sites)
from calibre_plugins.overdrive_link.parseweb import (
    beautiful_soup_fix, LibraryError, must_find, must_findAll,
    text_only, class_contains, parse_entities, is_sign_in_form, set_card_number, set_card_pin)
from calibre_plugins.overdrive_link.tweak import TWEAK_REMOVE_RESERVE_IDS

from .python_transition import (IS_PYTHON2)
if IS_PYTHON2:
    from .python_transition import (http, repr, str, urllib)
else:
    import http.cookiejar
    import urllib.parse


__license__ = 'GPL v3'
__copyright__ = '2012-2025, John Howell <jhowell@acm.org>'


GENERIC_SEARCH_SITES = {'search.overdrive.com', 'www.overdrive.com'}

OVERDRIVE_SIGNIN_BRANCH_ID = 'overdrive'

OVERDRIVE_BOOKS_FILE = 'overdrive_book_ids.bz2'

OVERDRIVE_FORMAT_IDS = {
    'ebook-epub-adobe': FORMAT_ADOBE_EPUB,
    'ebook-kindle': FORMAT_KINDLE_BOOK,
    'ebook-media-do': FORMAT_MEDIADO_READER,
    'audiobook-mp3': FORMAT_OD_MP3,
    'periodicals-nook': FORMAT_NOOK_PERIODICALS,
    'ebook-epub-open': FORMAT_OPEN_EPUB,
    'ebook-pdf-open': FORMAT_OPEN_PDF,
    'audiobook-overdrive': FORMAT_OD_LISTEN,
    'ebook-overdrive': FORMAT_OD_READ,
    'ebook-pdf-adobe': FORMAT_ADOBE_PDF,
    'video-streaming': FORMAT_STREAMING_VIDEO,
    'od-narration': FORMAT_OD_READ_ALONG,
    'magazine-overdrive': FORMAT_OD_MAGAZINE,
    'ebook-overdrive-readalong': FORMAT_OD_READ_ALONG,
    'ebook-overdrive-provisional': False,   # "OverDrive Read (Provisional)" for not yet published book - ignored
    }

preview_new_overdrive_sites = {}
new_site_equivalents = []


#--------------------------------
# using a json file to keep track of which libraries have been upgraded is not the safest approach due
# to file contention, but it is only temporary.

ConfigStoreLocation = 'plugins/Overdrive Link Temp'
NewOverDriveLibraries = 'NewOverDriveLibraries'

new_overdrive_sites = None
new_book_id_from_legacy = None

book_id_of_reserve_id = None
reserve_id_of_book_id = {}


def load_book_ids(log):
    global book_id_of_reserve_id

    if book_id_of_reserve_id is None:
        book_id_of_reserve_id = {}      # try to load translation only once per run of calibre
        filename = os.path.join(config_dir, 'plugins', OVERDRIVE_BOOKS_FILE)

        if os.path.isfile(filename):
            with bz2.BZ2File(filename, 'rb') as of:
                book_id_of_reserve_id = json.load(of)

            for reserve_id, book_id in book_id_of_reserve_id.items():
                reserve_id_of_book_id[book_id] = reserve_id

            if log is not None:
                log.info("Loaded %d OverDrive book IDs from %s" % (len(book_id_of_reserve_id), filename))
            elif DEBUG:
                print("Loaded %d OverDrive book IDs from %s" % (len(book_id_of_reserve_id), filename))


def save_book_ids(log):
    filename = os.path.join(config_dir, 'plugins', OVERDRIVE_BOOKS_FILE)

    with bz2.BZ2File(filename, 'wb') as of:
        json_data = json.dumps(book_id_of_reserve_id, sort_keys=True, indent=0, separators=(',', ':'))
        of.write(json_data if IS_PYTHON2 else json_data.encode("ascii"))

    log.info("Saved %d OverDrive book IDs to %s" % (len(book_id_of_reserve_id), filename))


def is_new_overdrive_site(library_id):
    global new_overdrive_sites

    #print('is_new_site %s' % library_id)

    if library_id in known_new_overdrive_sites:
        return True

    if new_overdrive_sites is None:
        #print('load persistent NewOverDriveLibraries')

        try:
            persistent = JSONConfig(ConfigStoreLocation)    # load at first need
            persistent.defaults[NewOverDriveLibraries] = []
            new_overdrive_sites = set(persistent[NewOverDriveLibraries])
        except Exception:
            traceback.print_exc()
            return True

    return library_id in new_overdrive_sites


def set_new_overdrive_site(library_id, is_new):
    global new_overdrive_sites

    if bool(is_new_overdrive_site(library_id)) != bool(is_new):
        if is_new:
            new_overdrive_sites.add(library_id)
        else:
            new_overdrive_sites.remove(library_id)

        #print('save persistent NewOverDriveLibraries')

        try:
            persistent = JSONConfig(ConfigStoreLocation)
            persistent[NewOverDriveLibraries] = sorted(list(new_overdrive_sites))   # save on change
        except Exception:
            traceback.print_exc()

#--------------------------------


def paired_library_id(library_id):
    if '.lib.overdrive.' in library_id:
        return library_id.replace('.lib.overdrive.', '.overdrive.')
    elif '.overdrive.' in library_id:
        return library_id.replace('.overdrive.', '.lib.overdrive.')
    elif '.lib.libraryreserve.' in library_id:
        return library_id.replace('.lib.libraryreserve.', '.libraryreserve.')
    elif '.libraryreserve.' in library_id:
        return library_id.replace('.libraryreserve.', '.lib.libraryreserve.')
    return None


def add_new_site_equivalent(s, e):
    if paired_library_id(s) != e:
        for se in new_site_equivalents:
            if s in se:
                se.add(e)
                break

            if e in se:
                se.add(s)
                break
        else:
            new_site_equivalents.append({s, e})


class OverDrive(SearchableLibrary):
    id = ''
    name = 'OverDrive'

    ebook_formats_supported = {
        FORMAT_ADOBE_EPUB, FORMAT_OPEN_EPUB, FORMAT_OPEN_PDF,
        FORMAT_KINDLE_BOOK, FORMAT_MOBI_EBOOK, FORMAT_ADOBE_PDF, FORMAT_OD_READ,
        FORMAT_MEDIADO_READER}
    audiobook_formats_supported = {FORMAT_OD_LISTEN, FORMAT_OD_MP3, FORMAT_OD_WMA, FORMAT_OD_READ_ALONG}
    video_formats_supported = {FORMAT_OD_VIDEO, FORMAT_OD_VIDEO_MOBILE, FORMAT_STREAMING_VIDEO}
    music_formats_supported = {FORMAT_OD_MUSIC, }
    periodical_formats_supported = {FORMAT_NOOK_PERIODICALS, }

    formats_supported = (ebook_formats_supported | audiobook_formats_supported |
                         video_formats_supported | music_formats_supported |
                         periodical_formats_supported)

    #supports_recommendation = True              # detected per library
    sign_in_affects_get_current_availability = True     # sign in needed to detect holds and Advantage titles

    @staticmethod
    def validate_library_id(library_id, migrate=True, config=None):
        orig_library_id = library_id

        if (':' in library_id) or ('/' in library_id):
            library_id = hostname_from_url(library_id)

        library_id = library_id.lower()

        if (not isValidHostname(library_id)) or (library_id in GENERIC_SEARCH_SITES):
            raise ValueError('OverDrive library hostname invalid: "%s"' % orig_library_id)

        if '.' not in library_id:
            raise ValueError('OverDrive library hostname must contain a period: "%s"' % orig_library_id)

        # detect cases where an alternate form of the same library id has already been configured and use it if so
        if (config is not None) and (config.library(OverDrive.id, library_id) is None):
            alt_library_ids = set()

            for lib_id in [library_id, paired_library_id(library_id)]:
                if lib_id is not None:
                    alt_library_ids.add(lib_id)

                    if lib_id in equivalent_overdrive_sites:
                        for equiv in equivalent_overdrive_sites[lib_id]:
                            alt_library_ids.add(equiv)
                            alt_library_ids.add(paired_library_id(equiv))

            for lib_id in alt_library_ids:
                if (lib_id is not None) and (lib_id != library_id) and (config.library(OverDrive.id, lib_id) is not None):
                    library_id = lib_id
                    break

        return library_id

    @staticmethod
    def validate_book_id(book_id, library_id):
        # update known books to new id
        if is_new_overdrive_site(library_id):
            load_book_ids(None)
            book_id = book_id_of_reserve_id.get(book_id, book_id)

        if tweaks.get(TWEAK_REMOVE_RESERVE_IDS, False):
            if not re.match(r'^[0-9]+$', book_id):
                raise ValueError('OverDrive book id must be a numeric: "%s"' % book_id)
        else:
            if not (re.match(r'^[0-9]+$', book_id) or
                    re.match(r'^([0-9a-f]{8})\-([0-9a-f]{4})\-([0-9a-f]{4})\-([0-9a-f]{4})\-([0-9a-f]{12})$', book_id)):
                raise ValueError('OverDrive book id must be a numeric or a UUID: "%s"' % book_id)

        return book_id

    @staticmethod
    def book_url(library_id, book_id):
        # temporary check to determine version of an OverDrive library site

        if len(book_id) <= 10:
            set_new_overdrive_site(library_id, True)

        if is_new_overdrive_site(library_id):
            return OverDrive.new_book_url(library_id, book_id)

        return OverDriveLegacy.book_url(library_id, book_id)

    @staticmethod
    def new_book_url(library_id, book_id):
        return 'https://%s/media/%s' % (library_id, book_id.upper())    # works for old or new style ID at new style library

    def __init__(self):
        self.recommendation_allowed = False
        self.signin_required = False
        self.signed_in = False
        self.is_legacy = False
        self.legacy = None
        self.ready_for_search = False
        self.cookiejar = http.cookiejar.CookieJar()
        self.holds_checked = False
        self.holds = {}
        self.loans = {}
        self.media_data_cache = {}

    def sign_in(self, use_credentials, allow_redirect=False, search_params=None, new_site_preview=False):
        self.cookiejar.clear()
        tried_ids = set()

        while True:
            if self.library_id in tried_ids:
                raise LibraryError(
                        'Incorrect library id in configuration!'
                        ' Host name %s is redirected to itself. Configuration update required.' % self.library_id)

            tried_ids.add(self.library_id)

            if new_site_preview:
                # temp - set cookie to enable preview of new site format
                self.cookiejar.set_cookie(http.cookiejar.Cookie(
                            version=0, name='secretClub', value='1', port=None, port_specified=False,
                            domain=self.library_id, domain_specified=True, domain_initial_dot=False,
                            path="/", path_specified=True, secure=False, expires=None, discard=False,
                            comment=None, comment_url=None, rest=None))

            self.cookiejar.set_cookie(http.cookiejar.Cookie(
                        version=0, name='hideLibbyIntercept', value='1', port=None, port_specified=False,
                        domain=self.library_id, domain_specified=True, domain_initial_dot=False,
                        path="/", path_specified=True, secure=False, expires=None, discard=False,
                        comment=None, comment_url=None, rest=None))

            #br = mechanize.Browser()
            #br.set_cookiejar(self.cookiejar)
            #self.browse_od_url(br, request=mechanize.Request('https://expired-rsa-dv.ssl.com/'))
            #self.browse_od_url(br, request=mechanize.Request('https://expired-ecc-ev.ssl.com/'))

            try_url = 'https://%s/' % self.library_id
            response = self.open_od_url(try_url)
            soup = beautiful_soup_fix(response.data_string)

            redirect_url = response.geturl()
            if redirect_url != try_url:
                self.log.info('Redirected to %s' % redirect_url)

            redirect_host = hostname_from_url(redirect_url).lower()

            if redirect_host in GENERIC_SEARCH_SITES:
                raise LibraryError(
                        'Non-working library id in configuration!'
                        ' Host name %s is redirected to the generic Overdrive site (%s).' % (self.library_id, redirect_host))

            if ((redirect_url.startswith('https:') and redirect_host.endswith('libraryreserve.com')) or
                    re.search(r'/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/', redirect_url, flags=re.IGNORECASE) or
                    soup.find('section', id='mainContent')):
                # temporary check needed until all OverDrive sites have been upgraded to new format
                self.log.info('Legacy OverDrive site layout detected for %s' % self.name)
                set_new_overdrive_site(self.library_id, False)
                self.is_legacy = True

                thunder_pop = soup.find('div', id='thunderPop')
                if thunder_pop:
                    link = thunder_pop.find('a', attrs={'href': re.compile(r'^https://')})
                    if link:
                        # Pop up message link to new style website
                        self.log.info('New preview site: %s' % link['href'])
                        preview_new_overdrive_sites[link['href'].replace('/preview', '')] = self.name
                        new_site = hostname_from_url(link['href'])
                        add_new_site_equivalent(self.library_id, new_site)

                # create an old-style library instance to use
                self.legacy = OverDriveLegacy()
                self.legacy.enabled = self.enabled
                self.legacy.provider = OverDriveLegacy
                self.legacy.provider_id = self.provider_id
                self.legacy.library_id = self.library_id
                self.legacy.branch_id = self.branch_id
                self.legacy.name = self.name
                self.legacy.card_number = self.card_number
                self.legacy.card_pin = self.card_pin
                self.legacy.log = self.log
                self.legacy.config = self.config

                self.legacy.signin_required = False
                self.legacy.signed_in = False
                self.legacy.did_sign_in = True

                try:
                    self.legacy.sign_in(use_credentials)
                except Exception:
                    self.signin_required = self.legacy.signin_required
                    self.signed_in = self.legacy.signed_in
                    self.ready_for_search = self.legacy.ready_for_search
                    self.recommendation_allowed = self.legacy.recommendation_allowed
                    raise

                self.signin_required = self.legacy.signin_required
                self.signed_in = self.legacy.signed_in
                self.ready_for_search = self.legacy.ready_for_search
                self.recommendation_allowed = self.legacy.recommendation_allowed
                return

            elif (redirect_host != self.library_id) and ('/login' not in redirect_url) and ('.auth.overdrive.com/' not in redirect_url):
                add_new_site_equivalent(self.library_id, redirect_host)

                msg = ('Incorrect library id in configuration! Host name %s is redirected to %s.'
                       ' Configuration change followed by "Check and repair book links" is required. See Help for details.') % (
                            self.library_id, redirect_host)

                if not allow_redirect:
                    raise LibraryError(msg)

                self.log.error(msg)
                self.library_id = redirect_host
                continue

            break

        toaster = soup.find('div', attrs={'class': 'js-toaster'}) or soup.find('div', attrs={'class': 'toaster'})
        if not toaster:
            raise LibraryError('Unknown OverDrive library website layout. Cannot access.')

        set_new_overdrive_site(self.library_id, True)
        self.log.info('new OverDrive site layout detected for %s' % self.name)

        if self.card_number and use_credentials:
            self.signin_required = True

            data = {"forward": "/"}     # ?openAccountMenu=true

            response = self.open_od_url('https://%s/account/sign-in?%s' % (self.library_id, urllib.parse.urlencode(data)))

            login_url = response.geturl()
            soup = beautiful_soup_fix(response.data_string)

            if "<h1>Something went wrong</h1>" in response.data_string:
                msg = 'Something went wrong during sign in'
                alert = soup.find('div', attrs=class_contains('alert-box'))
                if alert:
                    msg += ': ' + text_only(alert)

                raise LibraryError(msg)

            model_script = soup.find('script', attrs={'id': 'model'})
            if model_script is not None:
                self.signin_pre_2019(soup, login_url)
            else:
                # login change 08/2019
                if self.branch_id == OVERDRIVE_SIGNIN_BRANCH_ID:
                    raise LibraryError("Overdrive login using email is not currently supported, use library-specific login (card and pin) instead")

                login_forms = self.get_login_forms(soup)
                ils_name_of_branch = {}
                local_ils_names = set()
                report_forms = False

                for form in login_forms['forms']:
                    display_name = form.get('displayName', '').strip()
                    form_type = form['type']

                    if form_type == "Local":
                        # visitorHomeWebsiteId may by old-style branch id, seems unused
                        local_ils_names.add(form['ilsName'])
                        ils_name_of_branch[display_name] = form['ilsName']
                    elif form_type == "External":
                        ils_name_of_branch[display_name] = '[UNSUPPORTED EXTERNAL SIGN IN]'
                    elif form_type == "EmailRegistration":
                        ils_name_of_branch[display_name] = '[UNSUPPORTED EMAIL REGISTRATION SIGN IN]'
                    else:
                        self.log.warn("Unknown login form type %s for %s" % (form_type, self.library_id))
                        report_forms = True

                if report_forms or len(local_ils_names) == 0:
                    self.log.info('window.OverDrive.loginForms: %s' % json.dumps(login_forms, sort_keys=True, indent=4, separators=(',', ': ')))

                if len(local_ils_names) == 1:
                    self.branch_id = list(local_ils_names)[0]

                elif self.branch_id not in local_ils_names:
                    self.log.warn('A branch ID is required for %s. Configure this library'
                                  ' in the plugin with the appropriate branch ID. (See the log for a list.)' % self.library_id)

                    self.log.info("")
                    self.log.info("[For this branch] = [Use this branch ID]")

                    for branch, ils_name in sorted(ils_name_of_branch.items()):
                        self.log.info('%s = %s' % (branch, ils_name))

                    self.log.info("")

                for form in login_forms['forms']:
                    if form['type'] == "Local" and self.branch_id == form['ilsName']:
                        data = {}
                        data['authType'] = form['type']
                        data['ilsName'] = form['ilsName']

                        local = form['local']
                        if local.get('username', {}).get('enabled', False):
                            data['username'] = self.card_number
                        if local.get('password', {}).get('enabled', False):
                            data['password'] = self.card_pin

                        break
                else:
                    raise LibraryError("Branch ID '%s' not found in login forms. (See the log for list of the branches found.)" % self.branch_id)

                login_url = 'https://%s/account/signInOzone?%s' % (self.library_id, urllib.parse.urlencode({"forwardUrl": "/"}))
                response = self.open_od_url(login_url, data=urllib.parse.urlencode(data))

                login_url = response.geturl()
                soup = beautiful_soup_fix(response.data_string)

                if "ozone" in login_url.lower() or soup.find('div', attrs={'class': 'SignInPage'}) is not None:
                    raise LibraryError(self.get_login_error_message(soup))

                self.log.info('Sign in successful')
                self.signed_in = True

        elif (
                '/signin' in redirect_url.lower() or '/login' in redirect_url or
                '.auth.overdrive.com/' in redirect_url or "ozone" in redirect_url.lower()):
            self.signin_required = True
            self.log.error('Library %s requires sign in to perform searches.' % self.name)
            return

        # check advanced search form for options at this library
        response = self.open_od_url('https://%s/advanced-search' % self.library_id)
        soup = beautiful_soup_fix(response.data_string)
        search_form = must_find(soup, 'form', attrs={'id': 'advanced-search-form'})

        availability = must_find(search_form, 'select', attrs={'name': 'availability'})
        if availability.find('option', attrs={'data-parameter-name': 'showAlsoRecommendable'}):
            self.log.info('Search for recommendable books is supported by %s' % self.name)
            self.recommendation_allowed = True

        if search_params is not None:
            # collect data for site inventory

            for input in search_form.findAll('input', recursive=True):
                input_name = input.get('name', 'unknown')
                search_params[input_name].add(self.library_id)

            for select in search_form.findAll('select', recursive=True):
                select_name = select.get('name', 'unknown')

                for option in select.findAll('option', recursive=True):
                    option_value = option.get('value', '')
                    data_param = option.get('data-parameter-name', '')
                    option_label = text_only(option)
                    search_params['%s/%s/%s/%s' % (select_name, option_value, data_param, option_label)].add(self.library_id)

        self.ready_for_search = True

    def signin_pre_2019(self, soup, login_url):
        model_script = soup.find('script', attrs={'id': 'model'})
        model1 = js_value(self.log, html_to_text(model_script.string), "")
        #self.log.info('model1: %s' % json.dumps(model1, sort_keys=True, indent=4, separators=(',', ': ')))

        if self.branch_id == OVERDRIVE_SIGNIN_BRANCH_ID:
            # OverDrive authentication
            self.log.info('Signing in to OverDrive for %s' % self.name)

            data = {}
            data["librarylogin"] = "false"
            data["logintype"] = "odaccount"
            data[model1["tokenName"]] = model1["tokenValue"]    # data["idsrv.xsrf"] = "..."
            data["username"] = self.card_number
            data["password"] = self.card_pin
            data["CaptchaBypass"] = model1["captchaBypass"]

            response = self.open_od_url(login_url, data=urllib.parse.urlencode(data))

        else:
            # Library authentication
            if "libraryViewModel" in model1:
                model2 = model1["libraryViewModel"]
            else:
                self.log.info('libraryViewModel not present on sign-in page')
                response = self.open_od_url(model1["libraryAuthUrl"])
                login_url = response.geturl()

                soup = beautiful_soup_fix(response.data_string)

                model_script = must_find(soup, 'script', attrs={'id': 'model'})
                model2 = js_value(self.log, html_to_text(model_script.string), "")
                #self.log.info('model2: %s' % json.dumps(model2, sort_keys=True, indent=4, separators=(',', ': ')))

            login_forms = model2["loginForms"]

            if model2["displayLoginFormSelector"]:
                # branch selection required

                for login_form in login_forms:
                    if str(login_form["id"]) == self.branch_id:
                        self.log.info("Signing in to %s branch ID %s (%s)" % (self.name, self.branch_id, login_form["displayName"]))
                        break
                else:
                    self.log.warn('It appears that a branch ID is required for %s. Configure this library'
                                  ' in the plugin with the appropriate branch ID:' % self.library_id)

                    for login_form in login_forms:
                        self.log.info('%s = %d' % (login_form["displayName"], login_form["id"]))

                    if self.branch_id:
                        raise LibraryError("Failed to find branch ID %s - View log for a list of valid IDs" % self.branch_id)

                    raise LibraryError('Branch ID required for sign in - View log for a list of valid IDs')

            elif len(login_forms) != 1:
                raise LibraryError('Single OverDrive site has %d login forms' % len(login_forms))

            else:
                login_form = login_forms[0]     # single branch
                self.log.info('Signing in to %s (%s)' % (self.name, login_form["displayName"]))

            if login_form.get("isUnsupported", False):
                #self.log.info('login_form: %s' % json.dumps(login_form, sort_keys=True, indent=4, separators=(',', ': ')))
                raise LibraryError('Branch "%s" is unsupported' % login_form["displayName"])

            if login_form.get("isRedirect", False):
                #self.log.info('login_form: %s' % json.dumps(login_form, sort_keys=True, indent=4, separators=(',', ': ')))
                raise LibraryError('Branch "%s" is redirect' % login_form["displayName"])

            data = {}
            data[model2["tokenName"]] = model2["tokenValue"]    # data["idsrv.xsrf"] = "..."
            data["librarylogin"] = "true"
            data["logintype"] = "library"
            data["websiteId"] = model2["websiteId"]
            data["serverData"] = ""
            data["signIn"] = model2["signIn"]
            data["ilsName"] = login_form["ilsName"]
            data["formId"] = login_form["formId"]

            if not login_form.get("isExternal", False):
                data["username"] = self.card_number
                data["password"] = self.card_pin if login_form["requiresPin"] else "85c64ecf-2480-461a-891a-460fcc40eacb"
                data["CaptchaBypass"] = model2["captchaBypass"]

                login_url = urllib.parse.urljoin(login_url, model2["loginUrl"])

                response = self.open_od_url(login_url, data=urllib.parse.urlencode(data), referer="https://%s/" % self.library_id)

            else:
                self.log.info('Using external authentication')
                login_url = urllib.parse.urljoin(login_url, login_form["externalTarget"])
                response = self.open_od_url(login_url, data=urllib.parse.urlencode(data), referer="https://%s/" % self.library_id)

                soup = beautiful_soup_fix(response.data_string)
                a = must_find(soup, 'a')
                if 'follow the link' not in text_only(a):
                    raise LibraryError('Missing external authentication sign in link')

                login_url = urllib.parse.urljoin(login_url, a.get('href'))
                br = mechanize.Browser()
                br.set_cookiejar(self.cookiejar)
                self.browse_od_url(br, request=mechanize.Request(login_url))

                # Select the external sign in form
                try:
                    if len([f for f in br.forms()]) == 1:
                        br.select_form(nr=0)    # only one form on page
                    else:
                        br.select_form(predicate=lambda f: is_sign_in_form(f))  # Exception if not found

                except Exception:
                    raise LibraryError('Missing sign in form')

                # User credentials
                set_card_number(self.log, br.form, self.card_number)
                set_card_pin(self.log, br.form, self.card_pin)

                # Login
                response = br
                page = response.data_string = self.browse_od_url(br)
                login_url = response.geturl()

                if ('incorrect.  Please try again' in page) or ('overdrive.com' not in login_url):
                    # External Authentication failure
                    self.log.info('response: %s' % page)    # TEMP???
                    soup = beautiful_soup_fix(page)
                    self.log.info('Error from %s: %s' % (login_url, text_only(soup)))
                    raise LibraryError('External authentication sign in failed. Check card number and PIN.')

        login_url = response.geturl()
        page = response.data_string
        soup = beautiful_soup_fix(page)

        if "<title>Submit this form</title>" not in page:
            self.log.info("login url: %s" % login_url)

            model_script = soup.find('script', attrs={'id': 'model'})
            if model_script:
                model3 = js_value(self.log, html_to_text(model_script.string), "")
                #self.log.info('model3: %s' % json.dumps(model3, sort_keys=True, indent=4, separators=(',', ': ')))

                if "errorMessage" in model3 or "libraryLoginErrorText" in model3:
                    raise LibraryError('Sign in error: %s: %s' % (
                            model3.get("libraryLoginErrorText", ""), model3.get("errorMessage", "")))

            self.log.info("response: %s" % page)
            raise LibraryError('Sign in failure: Missing final form')

        form = must_find(soup, 'form')
        data = {}
        for input in must_findAll(form, 'input'):
            data[input.get('name')] = input.get('value')

        response = self.open_od_url(form.get('action'), data=urllib.parse.urlencode(data), referer=login_url)

        login_url = response.geturl()

        if "openAccountMenu" not in login_url:
            raise LibraryError('Sign in failure: Final login url=%s' % login_url)

        self.log.info('Sign in successful')
        self.signed_in = True

    def find_books(self, books, search_author, search_title, keyword_search):
        if self.legacy:
            return self.legacy.find_books(books, search_author, search_title, keyword_search)

        if not self.ready_for_search:
            return False

        if len(search_author) < 4 and not search_title:
            return False    # very short author names return bad results

        #find_recommendable = self.config.check_recommendable and self.recommendation_allowed
        find_recommendable = False

        if keyword_search and find_recommendable:
            return False    # combination not supported for OverDrive

        RESULTS_PER_PAGE = 24
        MAX_RESULTS_ALLOWED = 500
        MAX_RETRIES = 2

        # showAlsoRecommendable=true misses some available books so need to search twice!
        for do_recommendable in [False, True]:
            if do_recommendable and not find_recommendable:
                continue

            page_num = 1
            total_pages = 1
            total_results = 0
            results_processed = 0
            retries = 0

            while (page_num <= total_pages):
                data = {}

                # These require ' ' to be replaced by '+' in urlencode for proper operation
                if search_title:
                    data['query'] = ' '.join(search_title.lower().split())                 # defaults to 'AND'
                if search_author:
                    data['creator'] = ' AND '.join(search_author.lower().split())     # defaults to 'OR'

                if LANGUAGE_CODE2.get(self.config.search_language):
                    data['language'] = LANGUAGE_CODE2[self.config.search_language]

                if len((self.config.search_formats - self.ebook_formats_supported) & self.formats_supported) == 0:
                    data['mediaType'] = 'ebook'         # only e-books
                elif len((self.config.search_formats - self.audiobook_formats_supported) & self.formats_supported) == 0:
                    data['mediaType'] = 'audiobook'     # only audiobooks

                # showOnlyAvailable=true
                # showOnlyPrerelease=true

                if do_recommendable:
                    data['showAlsoRecommendable'] = 'true'

                data['sortBy'] = 'relevancy'

                if page_num != 1:
                    data['page'] = "%d" % page_num

                response = self.open_od_url('https://%s/search?%s' % (self.library_id, urllib.parse.urlencode(data)))

                soup = beautiful_soup_fix(response.data_string)

                error_container = soup.find('div', attrs=class_contains('error-container'))
                if error_container:
                    msg = text_only(error_container)
                    if "You may be able to fix it by simply refreshing the page" in msg and retries < MAX_RETRIES:
                        self.log.info("retrying on library error: %s" % msg)
                        retries += 1
                        time.sleep(60)
                        continue

                    raise LibraryError(msg)

                # <h1 class="text-center Results-noResultsHeading" id="noresults" tabindex="0">We couldn't find any matches for ...</h1>
                # <h2 class="header-small text-center" id="noresults" tabindex="0">We couldn't find any matches for ...</h2>
                no_results = soup.find('h1', attrs={'id': 'noresults'}) or soup.find('h2', attrs={'id': 'noresults'})
                if no_results:
                    break

                # <h1 class="search-text">Showing 97-120 of 125 results for ...</h2>
                search_text = soup.find('h1', attrs=class_contains('search-text'))
                if not search_text:
                    search_text = soup.find('h2', attrs=class_contains('search-text'))
                    if not search_text:
                        search_text = must_find(soup, 'span', attrs=class_contains('search-text'))

                search_info = text_only(search_text).split()

                if search_info[0] != "Showing" or search_info[2] != "of" or search_info[4] != "results":
                    raise LibraryError('Unexpected search-text %s' % text_only(search_text))

                first_result = int(search_info[1].partition("-")[0])
                new_total_results = int(search_info[3].replace(",", ""))

                if first_result != results_processed + 1:
                    raise LibraryError('Unexpected first result %d instead of %d' % (first_result, results_processed + 1))

                if total_results and (new_total_results != total_results):
                    raise LibraryError('Total results changed from %d to %d' % (total_results, new_total_results))

                total_results = new_total_results
                total_pages = ((total_results - 1) // RESULTS_PER_PAGE) + 1  # floor division

                self.log.info('Response: page %d of %d. %d total results' % (page_num, total_pages, total_results))

                if total_results > MAX_RESULTS_ALLOWED:
                    return True

                for media_data in self.get_media_items2(soup).values():
                    lbook, desired_format = self.get_library_book_from_media(media_data, do_recommendable, search_author)

                    if not desired_format:
                        self.log.info('Ignoring: %s' % repr(lbook))
                    if not (lbook.available or do_recommendable):
                        self.log.info('Ignoring unavailable: %s' % repr(lbook))
                    else:
                        self.log.info('Found: %s' % repr(lbook))
                        books.add(lbook)

                    results_processed += 1

                page_num += 1

            if results_processed != total_results:
                raise LibraryError('Expected %s but found %d' % (value_unit(total_results, 'result'), results_processed))

        return False

    def get_library_book_from_media(self, data, do_recommendable, search_author):
        book_id = parse_entities(data["id"])

        title = parse_entities(data.get("title", ""))
        subtitle = parse_entities(data.get("subtitle", ""))
        edition = parse_entities(data.get("edition", ""))
        series = parse_entities(data.get("series", ""))

        authors = []
        if "firstCreatorName" in data:
            authors.append(normalize_author(parse_entities(data["firstCreatorName"]), unreverse=False))

        # formats not present in search results
        desired_format = False
        book_type = data.get("type", {}).get("id", "ebook")
        if book_type == 'ebook':
            if len(self.config.search_formats & self.ebook_formats_supported) > 0:
                desired_format = True
        elif book_type == 'audiobook':
            if len(self.config.search_formats & self.audiobook_formats_supported) > 0:
                desired_format = True
        elif book_type == 'video':
            if len(self.config.search_formats & self.video_formats_supported) > 0:
                desired_format = True
        elif book_type != 'music':
            raise LibraryError('Unknown book type %s' % book_type)

        language = ''
        for lang in data.get("languages", []):
            language = lang["name"]
            break   # take first

        title, series, series_index = fix_title(title, subtitle, edition, series)
        title = normalize_title(title)

        is_owned = (data.get("availabilityType", "") == "always" or data.get("isOwned", False) or data.get("ownedCopies", 0) > 0)

        reserve_id = data.get("reserveId", "")
        save_book_id_pair(self.log, book_id, reserve_id)

        return (LibraryBook(
                authors=authors, title=title, lib=self, book_id=book_id,
                series=series, series_index=series_index, language=language,
                available=is_owned, recommendable=(do_recommendable and not is_owned),
                search_author=search_author), desired_format)

    def get_book_info(self, book_id, cache):
        if self.legacy:
            return self.legacy.get_book_info(book_id, cache)

        if book_id not in self.media_data_cache:
            response = self.open_od_url(self.new_book_url(self.library_id, book_id), expect_errors=[404])

            if response.is_httperror_exception:
                self.log.info('Book does not exist at %s' % self.name)
                return None

            media_items = self.get_media_items(response.data_string)

            if len(media_items) == 0 and "We no longer offer OverDrive video" in response.data_string:
                return None     # 09/2022, https://brooklyn.overdrive.com/media/9166334
            elif len(media_items) == 1:
                self.media_data_cache[book_id] = list(media_items.values())[0]
            else:
                self.log.error("Found %d mediaItems (expected 1)" % len(media_items))
        else:
            self.log.info('found %s in media data cache' % book_id)

        data = self.media_data_cache[book_id]

        cache_allowed = True
        title = parse_entities(data.get("title", ""))
        subtitle = parse_entities(data.get("subtitle", ""))
        edition = parse_entities(data.get("edition", ""))
        series = parse_entities(data.get("series", ""))

        authors = []
        for creator in data.get("creators", []):
            author = normalize_author(parse_entities(creator.get("name", "")), unreverse=False)
            if author and (author not in authors):
                authors.append(author)

        formats = set()
        pubdate = None
        isbn = ''
        for format in data.get("formats", []):
            format_id = format.get("id", "")
            standard_format = OVERDRIVE_FORMAT_IDS.get(format_id)
            if standard_format:
                formats.add(standard_format)
            elif standard_format is None:   # allow False for unsupported/ignored format
                self.log.warn('Unknown book format: "%s" (%s)' % (format.get("name", ""), format_id))

            on_sale_date = format.get("onSaleDateUtc")
            if on_sale_date:
                pubdate = parse_only_date(on_sale_date, assume_utc=True)

            for ident in format.get("identifiers", []):
                if ident["type"] == "ISBN":
                    isbn = ident["value"]

        if len(formats) == 0 and data.get("isPreReleaseTitle", False):
            # format unknown for pre-release book so assume most common based on type
            book_type = data.get("type", {}).get("id", "ebook")
            self.log.info("Pre-release %s with no formats" % book_type)

            if book_type == "ebook":
                formats.add(FORMAT_ADOBE_EPUB)
                formats.add(FORMAT_OD_READ)
            elif book_type == "audiobook":
                formats.add(FORMAT_OD_MP3)

            cache_allowed = False   # do not cache assumed formats

        language = ''
        for lang in data.get("languages", []):
            language = lang["name"]
            break   # take first

        publisher = parse_entities(data.get("publisher", {}).get("name", ""))
        publisher = re.sub(r'\[Start\]', '', publisher).strip()    # Handle: Night Shade Books[Start]

        #odtitle,odsubtitle,odseries = title,subtitle,series

        title, series, series_index = fix_title(title, subtitle, edition, series)
        title = normalize_title(title)

        #if series != odseries:
        #    self.log.error('series: id=%s title="%s" series="%s" odtitle="%s" odsubtitle="%s" edition="%s" oseries="%s"' % (
        #            book_id, title, series, odtitle, odsubtitle, edition, odseries))

        if data.get("availabilityType", "") != "always" and data.get("ownedCopies", 0) == 0:
            cache_allowed = False   # formats may be wrong for non-owned books, do not cache

        id = data["id"]
        reserve_id = data.get("reserveId", "")
        save_book_id_pair(self.log, id, reserve_id)

        return InfoBook(authors=authors, title=title, series=series, series_index=series_index, isbn=isbn,
                        language=language, publisher=publisher, pubdate=pubdate, formats=formats,
                        lib=self, book_id=book_id, cache_allowed=cache_allowed)

    def get_current_book_availability(self, book_id):
        if self.legacy:
            return self.legacy.get_current_book_availability(book_id)

        self.check_current_holds()

        #consortium & advantage: copies owned, copies available, patrons on hold???
        #general: release date (for pre-release titles), on hold by user, checked out by user, always available y/n???

        is_on_hold = book_id in self.holds

        if is_on_hold:
            self.log.info('Using availability info from %s holds list for %s at %s' % (self.name, book_id, self.name))
            hold_position_overall = self.holds[book_id]["holdListPosition"]

            #"holdsCount":2, "holdsRatio":2 (people waiting per copy), "ownedCopies":1, "placedDate":"2016-10-19T16:20:45.253Z"
            #"isPreReleaseTitle":false, "estimatedReleaseDate":"2013-09-24T04:00:00Z"
        else:
            hold_position_overall = None

        have_checked_out = (book_id in self.loans)

        if book_id not in self.media_data_cache:
            response = self.open_od_url(self.new_book_url(self.library_id, book_id), expect_errors=[404])

            if response.is_httperror_exception:
                self.log.info('Book does not exist at %s' % self.name)
                return False

            media_items = self.get_media_items(response.data_string)

            if len(media_items) != 1:
                self.log.error("Found %d mediaItems (expected 1)" % len(media_items))

            self.media_data_cache[book_id] = list(media_items.values())[0]
        else:
            self.log.info('found %s in media data cache' % book_id)

        data = self.media_data_cache[book_id]

        #self.log.info("data: %s" % str(data))

        availability_type = data.get("availabilityType", "")
        if not availability_type:
            self.log.info("Book is unavailable at %s" % self.name)
            return False

        if availability_type not in ["normal", "always"]:
            self.log.error('Unexpected availabilityType "%s"' % (availability_type))
            return False

        if not (data.get("isOwned", False) or data.get("isHoldable", False)):
            self.log.info("Book is not owned by %s" % self.name)
            return False

        always_available = (availability_type == "always")
        library_copies = True if always_available else data.get("ownedCopies", 0)
        available_copies = True if always_available else data.get("availableCopies", 0)
        # number_waiting_per_copy = data.get("holdsRatio", 0)
        number_waiting_overall = data.get("holdsCount", None)
        release_date = parse_only_date(data.get("estimatedReleaseDate"), assume_utc=True) if data.get("isPreReleaseTitle", False) else None

        # estimate availability
        wait_weeks = self.calculate_wait_weeks(
                library_copies=library_copies, available_copies=available_copies,
                number_waiting_overall=number_waiting_overall, release_date=release_date,
                have_checked_out=have_checked_out, hold_position_overall=hold_position_overall)

        new_id = data["id"]
        reserve_id = data.get("reserveId", "")
        save_book_id_pair(self.log, new_id, reserve_id)

        if "-" in book_id:
            if book_id != reserve_id:
                self.log.error("Expected reserveID '%s' found '%s'" % (book_id, reserve_id))

            self.log.info("Replacing book id '%s' with '%s' at %s" % (book_id, new_id, self.name))
            book_id = new_id
        else:
            if book_id != new_id:
                self.log.error("Expected id '%d' found '%s'" % (book_id, new_id))

        return (wait_weeks, is_on_hold, book_id)

    def check_current_holds(self):
        # possible future use: Given a list of book ids, returns those that fit in the indicated category for the user.
        # POST pplc.overdrive.com/media/info, [2518103,2232646,2306309,2237099,2106188,1745835,1860614,2170149,2385936,1766962,1766779,2144376]
        # {"onCheckouts":[],"onHolds":[],"onWishlist":[],"starRatings":{},"recommendedToLibrary":[]}

        if self.legacy:
            return self.legacy.check_current_holds()

        if self.holds_checked or not self.signed_in:
            return

        self.holds_checked = True

        try:
            # get hold info
            response = self.open_od_url('https://%s/account/holds' % self.library_id)

            for data in self.get_media_items(response.data_string).values():
                #self.log.info("data: %s" % str(data))
                self.holds[data["id"]] = data

        except Exception as e:
            self.log.exception('', e)

        if self.holds:
            self.log.info('Holds: %s' % ', '.join(self.holds.keys()))

        try:
            # get loan info
            response = self.open_od_url('https://%s/account/loans' % self.library_id)

            for data in self.get_media_items(response.data_string).values():
                #self.log.info("data: %s" % str(data))
                self.loans[data["id"]] = data

        except Exception as e:
            self.log.exception('', e)

        if self.loans:
            self.log.info('Loans: %s' % ', '.join(self.loans.keys()))

    def get_media_items(self, response_data):
        soup = beautiful_soup_fix(response_data)

        error_container = soup.find('div', attrs=class_contains('error-container'))
        if error_container:
            raise LibraryError(text_only(error_container))

        return self.get_media_items2(soup)

    def get_media_items2(self, soup):
        for script in soup.findAll('script'):
            if "window.OverDrive.mediaItems =" in str(script):
                break

        else:
            raise LibraryError('Missing window.OverDrive.mediaItems')

        return js_value(self.log, str(script).replace('\x0a', ''), "window.OverDrive.mediaItems =")

    def get_login_forms(self, soup):
        for script in soup.findAll('script'):
            if "window.OverDrive.loginForms =" in str(script):
                break

        else:
            raise LibraryError('Missing window.OverDrive.loginForms')

        return js_value(self.log, str(script), "window.OverDrive.loginForms =")

    def get_login_error_message(self, soup):
        for script in soup.findAll('script'):
            if "window.OverDrive.errorMessage =" in str(script):
                break

        else:
            return "Unknown error during sign in. Check card number and pin."

        return js_value(self.log, str(script), "window.OverDrive.errorMessage =")

    def open_od_url(self, url, **kwargs):
        kwargs['cookiejar'] = self.cookiejar
        kwargs['addheaders'] = [("Accept-Language", "en-US,en;q=0.9")]
        return open_url(self.log, url, **kwargs)

    def browse_od_url(self, br, **kwargs):
        kwargs['addheaders'] = [("Accept-Language", "en-US,en;q=0.9")]
        return browse_url(self.log, br, **kwargs)


def save_book_id_pair(log, book_id, reserve_id):
    if (not (book_id and reserve_id)) or reserve_id.startswith("00000000-0000-0000-0000"):
        return

    load_book_ids(log)

    if reserve_id in book_id_of_reserve_id and book_id_of_reserve_id[reserve_id] != book_id:
        log.warn("Reserve ID %s matches multiple book IDs: %s and %s" % (reserve_id, book_id_of_reserve_id[reserve_id], book_id))

    if book_id in reserve_id_of_book_id and reserve_id_of_book_id[book_id] != reserve_id:
        log.warn("Book ID %s matches multiple reserve IDs: %s and %s" % (book_id, reserve_id_of_book_id[book_id], reserve_id))

    book_id_of_reserve_id[reserve_id] = book_id
    reserve_id_of_book_id[book_id] = reserve_id
