#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import, print_function)

import functools
import re
import traceback
from PyQt5.Qt import (
    QMenu, QToolButton, QProgressDialog, QDialog, QHBoxLayout, QVBoxLayout, QTextBrowser,
    QDialogButtonBox, QPushButton, QIcon, QSize, Qt)

from calibre.gui2 import (error_dialog, question_dialog, Dispatcher)
from calibre.gui2.actions import InterfaceAction
from calibre.constants import numeric_version
from calibre.ebooks.metadata.book.base import Metadata
from calibre.utils.date import (now, UNDEFINED_DATE)
from calibre.utils.config_base import tweaks

from calibre_plugins.overdrive_link.book import CalibreBook
from calibre_plugins.overdrive_link.formats import (
    ALL_READABLE_FORMATS, ALL_LISTENABLE_FORMATS,
    AZW3_FILE_TYPE, MOBI_FILE_TYPE, EPUB_FILE_TYPE)
from calibre_plugins.overdrive_link.link import (
    ODLinkSet, IDENT_NEEDED_LINK, IDENT_AVAILABLE_LINK, IDENT_RECOMMENDABLE_LINK, IDENT_PURCHASABLE_LINK,
    IDENT_ISBN, IDENT_AMAZON, FIELD_WAIT_WEEKS, FIELD_WAIT_WEEKS_DATE, FIELD_ET_STATUS, FIELD_ET_DATE,
    linksets_str, SEP_EBOOK, SEP_AUDIOBOOK,
    LINK_AVAILABLE, LINK_RECOMMENDABLE, LINK_PURCHASABLE, LINK_NEEDED,
    LINK_FORMAT_AUDIOBOOK, BOOK_DISCOVERED, browse_to_url, linkset_menu)
from calibre_plugins.overdrive_link.jobs import (start_search_jobs, SPECIAL_JOBS)
from calibre_plugins.overdrive_link.config import (
    DEFAULT_RELEASE_NOTE_VERSION, set_defaults, PLUGIN_ICONS, HELP_FILE, ReleaseNoteVersion,
    IncrementalSearchSequence, plugin_config, config_set_library_group)
from calibre_plugins.overdrive_link.discovered import (
    have_discovered_books, get_discovered_books, new_books_discovered, DiscoveredBooksDialog)
from calibre_plugins.overdrive_link.numbers import value_unit
from calibre_plugins.overdrive_link.release_notes import (RELEASE_NOTES, LAST_RELEASE_NOTE_VERSION)
from calibre_plugins.overdrive_link.log import (html_escape, html_to_text)
from calibre_plugins.overdrive_link.tweak import TWEAK_LIBRARY_GROUPS
from calibre_plugins.overdrive_link import (ActionOverdriveLink, get_resources, get_icons)

from .python_transition import (IS_PYTHON2)
if IS_PYTHON2:
    from .python_transition import (str)

try:
    from calibre_plugins.overdrive_link_debug.config import DEBUG_MODE
    from calibre_plugins.overdrive_link_debug.formats import get_levels_have
except ImportError:
    DEBUG_MODE = False
    get_levels_have = None


__license__ = 'GPL v3'
__copyright__ = '2012-2025, John Howell <jhowell@acm.org>'


def search_escape(str):
    # escape a string for inclusion in a calibre search (" and \)
    if re.search('[ "()]', str):
        return '"' + str.replace("\\", "\\\\").replace('"', '\\"') + '"'

    return str


def re_escape(str):
    # like re.escape, but only escapes re special characters instead of all non-alphanumerics
    return ''.join(("\\" + c if c in '.^$*+?{}\\[]|()' else c) for c in str)


class OverdriveLinkAction(InterfaceAction):
    name = ActionOverdriveLink.name
    type = ActionOverdriveLink.type

    # Create our top-level menu/toolbar action (text, icon_path, tooltip, keyboard shortcut)
    action_spec = ('Overdrive', None, 'Link books to lending libraries', None)
    popup_type = QToolButton.InstantPopup
    dont_add_to = frozenset(['menubar-device', 'context-menu-device'])
    action_type = 'current'

    def genesis(self):
        # load configuration
        self.config = set_defaults()

        # Load resources and icons
        self.help_text = get_resources(HELP_FILE).decode('utf-8')
        self.icons = get_icons(PLUGIN_ICONS, self.name) if numeric_version >= (5, 99, 3) else get_icons(PLUGIN_ICONS)

        # Initialize the gui menu for this plugin
        self.create_default_menu_actions()
        self.gui.keyboard.finalize()

        self.menu = QMenu(self.gui)
        self.set_default_menu()
        self.menu.aboutToShow.connect(self.set_customized_menu)
        self.menu.aboutToHide.connect(self.set_default_menu)

        # Assign our menu to this action and an icon
        self.qaction.setMenu(self.menu)

        self.main_icon = self.icons[PLUGIN_ICONS[0]]
        self.qaction.setIcon(self.main_icon)

        if not self.config.disable_metadata_plugin:
            self.create_metadata_plugin()

    def create_metadata_plugin(self):
        '''
        Total hack to add a metadata plugin as if it had been loaded separately.
        This plugin is used to show links in the book details panel.
        '''
        try:
            from calibre_plugins.overdrive_link.metadata import OverdriveLinkMetadata
            from calibre.customize.ui import _initialized_plugins
            metadata_plugin = OverdriveLinkMetadata(self.plugin_path, self)
            _initialized_plugins.append(metadata_plugin)
        except Exception:
            # Just continue on, in case a change to calibre breaks the above hack
            traceback.print_exc()

    def create_default_menu_actions(self):
        self.unlink_icon = self.icons['images/unlink.png']
        self.link_icon = self.icons['images/link.png']

        self.default_menu = QMenu(self.gui)
        m = self.default_menu

        # Actions that operate on the current selection

        self.search_book_action = self.create_menu_action(
            m, 'OverdriveLinkSearchBooks',
            'Search for selected books', 'search.png',
            description='Search for selected books at configured lending libraries',
            triggered=functools.partial(self.search, False, False, None))

        self.search_author_action = self.create_menu_action(
            m, 'OverdriveLinkSearchAuthors', 'Search for books by selected authors', 'search.png',
            description='Discover additional books by selected authors, missing from your calibre library',
            triggered=functools.partial(self.search, True, False, None))

        self.discover_by_keywords_action = self.create_menu_action(
            m, 'OverdriveLinkSearchKeywords', 'Discover books by keyword', 'search.png',
            description='Discover books matching the first set of configured keywords',
            triggered=functools.partial(self.search, True, False, 0))    # not selection-based, but fits better here

        self.browse_book_action = self.create_menu_action(
            m, 'OverdriveLinkBrowseBook', 'Browse selected book at lending library', 'store.png',
            description='Browse to the web page for the selected book at the highest priority lending library',
            triggered=self.browse_to_primary_link)

        self.get_availability_action = self.create_menu_action(
            m, 'OverdriveLinkCheckAvailable', 'Check current availability of selected books', 'search.png',
            description='Check if selected books are available now and estimate wait if not',
            triggered=functools.partial(self.get_availability, False))

        self.unlink_book_action = self.create_menu_action(
            m, 'OverdriveLinkUnlinkBook', 'Unlink books from enabled libraries', self.unlink_icon,
            description='Unlink selected book from enabled libraries',
            triggered=functools.partial(self.unlink_books, False))

        self.unlink_all_book_action = self.create_menu_action(
            m, 'OverdriveLinkAllUnlinkBook', 'Unlink books from all libraries', self.unlink_icon,
            description='Unlink selected book from all libraries (remove all links from book)',
            triggered=functools.partial(self.unlink_books, True))

        if DEBUG_MODE:
            self.search_incremental_action = self.create_menu_action(
                m, 'OverdriveLinkSearchIncremental', 'Search incrementally by authors', 'search.png',
                description='Search for books by the next incremental group of authors',
                triggered=functools.partial(self.search, True, True, None))

            self.get_needed_availability_action = self.create_menu_action(
                m, 'OverdriveLinkCheckNeededAvailable', 'Check current availability of selected needed books', 'search.png',
                description='Check if selected needed books are available now and estimate wait if not',
                triggered=functools.partial(self.get_availability, True))

            self.unlink_needed_action = self.create_menu_action(
                m, 'OverdriveLinkUnlinkNeeded', 'Remove "needed" links of selected books', self.unlink_icon,
                description='Remove any "needed" links for currently selected books',
                triggered=self.unlink_needed)

        m.addSeparator()

        # Actions that are not selection-based

        self.display_new_release_notes_action = self.create_menu_action(
            m, 'OverdriveLinkNewReleaseNotes',
            'Important release notes!', 'rating.png',
            description='Display important notes about this plugin release',
            triggered=self.display_new_release_notes)

        self.select_all_action = self.create_menu_action(
            m, 'OverdriveLinkSelectAll', 'Select all available books', self.link_icon,
            description='Select all books containing an available library book link',
            triggered=self.show_available_books)

        self.select_recommend_action = self.create_menu_action(
            m, 'OverdriveLinkSelectRecommend', 'Select all recommendable books', self.link_icon,
            description='Select all books containing only recommendable book links',
            triggered=self.show_recommendable_books)

        self.select_purchase_action = self.create_menu_action(
            m, 'OverdriveLinkSelectPurchase', 'Select all purchasable books', self.link_icon,
            description='Select all books containing only purchasable book links',
            triggered=self.show_purchasable_books)

        self.select_need_action = self.create_menu_action(
            m, 'OverdriveLinkSelectNeed', 'Select all needed books', self.link_icon,
            description='Select all books containing a needed library book link',
            triggered=self.show_needed_books)

        self.select_all_audiobook_action = self.create_menu_action(
            m, 'OverdriveLinkSelectAllAudio', 'Select all available audiobooks', self.link_icon,
            description='Select all books containing an available library audiobook link',
            triggered=self.show_available_audiobooks)

        self.select_recommend_audiobook_action = self.create_menu_action(
            m, 'OverdriveLinkSelectRecommendAudio', 'Select all recommendable audiobooks', self.link_icon,
            description='Select all books containing only recommendable audiobook links',
            triggered=self.show_recommendable_audiobooks)

        self.select_purchase_audiobook_action = self.create_menu_action(
            m, 'OverdriveLinkSelectPurchaseAudio', 'Select all purchasable audiobooks', self.link_icon,
            description='Select all books containing only purchasable audiobook links',
            triggered=self.show_purchasable_audiobooks)

        self.select_need_audiobook_action = self.create_menu_action(
            m, 'OverdriveLinkSelectNeedAudio', 'Select all needed audiobooks', self.link_icon,
            description='Select all books containing a needed library audiobook link',
            triggered=self.show_needed_audiobooks)

        self.manage_results_action = self.create_menu_action(
            m, 'OverdriveLinkManageResults', 'Manage discovered books', 'merge_books.png',
            description='Manage books found through "Search by selected authors", missing from calibre library',
            triggered=self.handle_discovered_books)

        self.gutenberg_index_action = self.create_menu_action(
            m, 'OverdriveLinkBuildGutenbergIndex', 'Build index for Project Gutenberg', 'config.png',
            description='Download catalog from Project Gutenberg and build an index for searching',
            triggered=self.gutenberg_index)

        self.check_action = self.create_menu_action(
            m, 'OverdriveLinkCheckLinks', 'Check and repair book links', 'config.png',
            description='Check existing book links for problems and repair them',
            triggered=self.check_all_links)

        self.customize_action = self.create_menu_action(
            m, 'OverdriveLinkCustomize', 'Customize plugin', 'config.png',
            description='Configure the plugin\'s settings',
            triggered=self.show_configuration)

        self.help_action = self.create_menu_action(
            m, 'OverdriveLinkHelp', 'Help', 'help.png',
            description='Display help for this plugin',
            triggered=self.show_help)

        # Actions that are temporary and error messages (Not default actions)

        self.temp_menu = QMenu(self.gui)
        m = self.temp_menu

        self.customize_error_action = self.add_menu_action(
            m, '(Customize this plugin to enable search)', QIcon(I('dialog_error.png')),
            'Use "Customize plugin" to configure lending libraries', enabled=False)

        self.link_error_action = self.add_menu_action(
            m, '(Selected book not linked)', QIcon(I('dialog_error.png')),
            'There are no links for the selected book', enabled=False)

        self.select_error_action = self.add_menu_action(
            m, '(Select books to enable search)', QIcon(I('dialog_error.png')),
            'Search is performed based on selected books', enabled=False)

    def set_default_menu(self):
        # Copy actions from the default menu to the current menu
        self.menu.clear()

        for a in QMenu.actions(self.default_menu):
            self.menu.addAction(a)

    def set_customized_menu(self):
        # Build menu on the fly based on the number of books selected and actual links
        # Save actions in a temp menu since clearing this menu on aboutToHide sometimes causes the program
        # to crash in QT if the action is deleted just after being triggered

        m = self.menu
        m.clear()

        if self.have_new_release_notes():
            m.addAction(self.display_new_release_notes_action)
            return

        db = self.gui.current_db
        if self.gui.current_view() is self.gui.library_view and self.gui.library_view.selectionModel().hasSelection():
            num_books_selected = len(self.gui.library_view.selectionModel().selectedRows())
        else:
            num_books_selected = 0

        if num_books_selected == 1:
            identifiers = db.new_api.get_proxy_metadata(self.gui.library_view.get_selected_ids()[0]).identifiers

        if self.config.any_enabled() and not self.config.configuration_error:
            if num_books_selected:
                # allow search for books
                m.addAction(self.search_book_action)
                m.addAction(self.search_author_action)

            if DEBUG_MODE:
                m.addAction(self.search_incremental_action)

            if self.config.discover_keywords:
                sm = QMenu(m)
                self.add_menu_action(
                    m, 'Discover books by keyword', QIcon(I('search.png')),
                    'Discover books based on configured keywords', submenu=sm)

                for i, keywords in enumerate(self.config.discover_keywords):
                    self.temp_menu.addAction(
                        self.add_menu_action(
                            sm, keywords, QIcon(I('search.png')),
                            'Discover books matching keywords ' + keywords,
                            triggered=functools.partial(self.search, True, False, i)))

        else:
            m.addAction(self.customize_error_action)

        if num_books_selected:
            m.addSeparator()
            show_linked_book_menu_items = False

            if num_books_selected == 1:
                # Only follow links for a single book
                # Open browser for access to book at lending library
                if linkset_menu(identifiers, m, self.config, save=self.temp_menu):
                    m.addSeparator()
                    show_linked_book_menu_items = True
                else:
                    m.addAction(self.link_error_action)
            else:
                show_linked_book_menu_items = True  # multiple books selected, some may be linked

            if show_linked_book_menu_items and not self.config.configuration_error:
                m.addAction(self.get_availability_action)

                if DEBUG_MODE:
                    m.addAction(self.get_needed_availability_action)

                sm = QMenu(m)
                sm.addAction(self.unlink_all_book_action)
                sm.addAction(self.unlink_book_action)

                if DEBUG_MODE:
                    sm.addAction(self.unlink_needed_action)

                if num_books_selected == 1:
                    sm.addSeparator()
                    linkset_menu(identifiers, sm, self.config, save=self.temp_menu,
                                 unlink_action=self.remove_single_link, icon=self.unlink_icon)

                self.add_menu_action(m, 'Unlink selected books', self.unlink_icon,
                                     'Remove links from books', submenu=sm)

        else:
            m.addAction(self.select_error_action)

        m.addSeparator()

        sm = QMenu(m)

        for is_audio, type, sep in [(False, 'book', SEP_EBOOK), (True, 'audiobook', SEP_AUDIOBOOK)]:
            for ident, cat, all_ebook_action, all_audiobook_action in [
                    (IDENT_AVAILABLE_LINK, 'Available',
                     self.select_all_action, self.select_all_audiobook_action),
                    (IDENT_RECOMMENDABLE_LINK, 'Recommendable',
                     self.select_recommend_action, self.select_recommend_audiobook_action),
                    (IDENT_PURCHASABLE_LINK, 'Purchasable',
                     self.select_purchase_action, self.select_purchase_audiobook_action),
                    (IDENT_NEEDED_LINK, 'Needed',
                     self.select_need_action, self.select_need_audiobook_action),
                    ]:

                if ident == IDENT_NEEDED_LINK and not DEBUG_MODE:
                    continue

                ssm = QMenu(sm)
                ssm.addAction(all_audiobook_action if is_audio else all_ebook_action)

                for library in sorted(self.config.libraries, key=lambda lib: lib.name):

                    if ident == IDENT_RECOMMENDABLE_LINK and not library.provider.supports_recommendation:
                        continue

                    if ident == IDENT_PURCHASABLE_LINK and not library.provider.supports_purchase(library.library_id):
                        continue

                    if ((is_audio and library.provider.formats_supported.isdisjoint(ALL_LISTENABLE_FORMATS)) or
                            ((not is_audio) and library.provider.formats_supported.isdisjoint(ALL_READABLE_FORMATS))):
                        continue

                    self.temp_menu.addAction(self.add_menu_action(
                        ssm, 'Only at ' + library.name, self.link_icon,
                        'Select books with %s:%s%s' % (ident, sep, library.full_library_id),
                        triggered=functools.partial(self.show_linked, ident, sep, library.full_library_id)))

                self.add_menu_action(sm, '%s %ss' % (cat, type), self.link_icon, '', submenu=ssm)

            sm.addSeparator()

        self.add_menu_action(
            m, 'Select previously linked books', self.link_icon,
            'Actions to be taken for books linked previously', submenu=sm)

        m.addSeparator()
        if have_discovered_books(db):
            m.addAction(self.manage_results_action)

        if self.config.have_project_gutenberg:
            m.addAction(self.gutenberg_index_action)

        m.addAction(self.check_action)

        if (TWEAK_LIBRARY_GROUPS in tweaks) and self.config.libraries and (not self.config.configuration_error):
            sm = QMenu(m)
            self.add_menu_action(
                m, 'Set library group', QIcon(I('config.png')),
                'Enable a specific group of libraries in the configuration', submenu=sm)

            for group_name, group_libs in sorted(tweaks.get(TWEAK_LIBRARY_GROUPS).items()):
                lib_names = set()
                # handle library definitions that include others
                for lib_name in group_libs:
                    if lib_name.startswith('*'):
                        lib_names.update(set(tweaks.get(TWEAK_LIBRARY_GROUPS)[lib_name[1:]]))
                    elif not lib_name.startswith('+'):
                        lib_names.add(lib_name)

                extra = lib_names - self.config.library_names
                for lib_name in extra:
                    lib_names.remove(lib_name)
                    lib_names.add(lib_name + '???')

                self.temp_menu.addAction(
                    self.add_menu_action(
                        sm, group_name,
                        QIcon(I('ok.png')) if lib_names == self.config.enabled_library_names else None,
                        ', '.join(sorted(list(lib_names))), triggered=functools.partial(self.set_library_group, lib_names)))

        m.addAction(self.customize_action)
        m.addAction(self.help_action)

    def show_available_books(self):
        self.show_linked(IDENT_AVAILABLE_LINK, SEP_EBOOK)

    def show_recommendable_books(self):
        self.show_linked(IDENT_RECOMMENDABLE_LINK, SEP_EBOOK)

    def show_purchasable_books(self):
        self.show_linked(IDENT_PURCHASABLE_LINK, SEP_EBOOK)

    def show_needed_books(self):
        self.show_linked(IDENT_NEEDED_LINK, SEP_EBOOK)

    def show_available_audiobooks(self):
        self.show_linked(IDENT_AVAILABLE_LINK, SEP_AUDIOBOOK)

    def show_recommendable_audiobooks(self):
        self.show_linked(IDENT_RECOMMENDABLE_LINK, SEP_AUDIOBOOK)

    def show_purchasable_audiobooks(self):
        self.show_linked(IDENT_PURCHASABLE_LINK, SEP_AUDIOBOOK)

    def show_needed_audiobooks(self):
        self.show_linked(IDENT_NEEDED_LINK, SEP_AUDIOBOOK)

    def show_linked(self, link_type, book_type, full_library_id=''):
        # produce a search string that will find books of the right type, linked to the supplied library
        # always use regular expression search so that punctuation characters are not ignores in calibre 5.42+

        ident_re = '%s%s(&|$)' % (re_escape(book_type), re_escape(full_library_id)) if full_library_id else re_escape(book_type)
        search = 'identifiers:%s' % search_escape('=%s:~%s' % (link_type, ident_re))

        if link_type == IDENT_RECOMMENDABLE_LINK:
            search += ' and not identifiers:%s' % search_escape('=%s:~%s' % (IDENT_AVAILABLE_LINK, re_escape(book_type)))

        self.gui.search.set_search_string(search)

    def browse_to_primary_link(self):
        selected_ids = self.gui.library_view.get_selected_ids()
        if not selected_ids:
            error_dialog(self.gui, 'No Books Selected', 'Select a book to browse at lending library', show=True)
            return

        if len(selected_ids) > 1:
            error_dialog(
                self.gui, 'Too many Books Selected', 'Select only a single book to browse at lending library',
                show=True)
            return

        odid = self.gui.current_db.new_api.get_proxy_metadata(selected_ids[0]).identifiers.get(IDENT_AVAILABLE_LINK)
        if not odid:
            error_dialog(
                self.gui, 'Selected book not linked', 'There are no lending library links for the selected book',
                show=True)
            return

        # Open browser for access to book at lending library
        browse_to_url(ODLinkSet(odid, config=self.config).primary_odlink().url())

    def search(self, discover_books, incremental, kw_index):
        db = self.gui.current_db
        selected_ids = db.new_api.all_book_ids() if incremental else self.gui.library_view.get_selected_ids()

        if kw_index is not None:
            if len(self.config.discover_keywords) == 0:
                error_dialog(
                    self.gui, 'No keywords have been configured for book discovery.',
                    'Use "Customize plugin" to configure keywords for search.', show=True)
                return

            keywords = self.config.discover_keywords[kw_index]
            kw_special = keywords.partition(' ')[0]
            special = kw_special in SPECIAL_JOBS
            discover_books = (not special) or (kw_special == 'get_new_overdrive_book_ids')
            if not special:
                selected_ids = []
        else:
            keywords = None
            special = False

            if not selected_ids:
                error_dialog(self.gui, 'No books selected', 'Select books to enable search', show=True)
                return

        if special:
            collect_et_status = (keywords == 'inventory_amazon_et') and self.lib_has_custom_column(FIELD_ET_STATUS)
        else:
            collect_et_status = False
            if not self.config.any_enabled():
                error_dialog(
                    self.gui, 'No enabled lending Library',
                    'Use "Customize plugin" to configure and enable lending libraries',
                    show=True)
                return

            if not self.config.search_formats:
                error_dialog(
                    self.gui, 'No formats have been enabled for search.',
                    'Use "Customize plugin" to configure formats for search.', show=True)
                return

        if not self.config.disable_confirm_dlgs:
            if not keywords:
                q = '<p>Search lending library collection %s %s for %s %s' % (
                    'and recommendable' if self.config.check_recommendable else '',
                    'incrementally' if incremental else '',
                    'books by the same author(s) as' if discover_books else '',
                    value_unit(len(selected_ids), 'selected book'))

            elif special:
                q = '<p>%s' % SPECIAL_JOBS[kw_special].name

            else:
                q = '<p>Discover books in lending library collection %s with keywords %s' % (
                    'and recommendable' if self.config.check_recommendable else '',
                    keywords)

            q += '?<p>(This will run as a background job.)'

            if not question_dialog(self.gui, 'Start search', q):
                return

        ignore_ids = set()

        if discover_books:
            # Need info on all books in the calibre library so that those already known are not found again
            all_calibre_books = self.collect_book_metadata(
                    db.new_api.all_book_ids(), ignore_ids=ignore_ids,
                    collect_et_status=collect_et_status,
                    collect_wait_weeks_field=IDENT_NEEDED_LINK if DEBUG_MODE else IDENT_AVAILABLE_LINK)

            # Since checking for more discovered books, provide those known so far
            orig_discovered_books, orig_discovered_books_version = get_discovered_books(db)

        else:
            all_calibre_books = self.collect_book_metadata(
                    selected_ids, ignore_ids=ignore_ids,
                    collect_et_status=collect_et_status,
                    collect_wait_weeks_field=IDENT_NEEDED_LINK if DEBUG_MODE else IDENT_AVAILABLE_LINK)

            orig_discovered_books = []  # unused for this search type
            orig_discovered_books_version = 0

        if (all_calibre_books is None) and not special:
            return

        if ignore_ids:
            selected_ids = [id for id in selected_ids if id not in ignore_ids]    # drop books to be ignored during search

        if not keywords:
            if not selected_ids:
                error_dialog(self.gui, 'Only ignored books selected', 'Select non-ignored books to enable search', show=True)
                return

            self.gui.status_bar.show_message(
                'Searching lending libraries for ' + value_unit(len(selected_ids), 'book'))
        elif special:
            self.gui.status_bar.show_message(SPECIAL_JOBS[kw_special].name)
        else:
            self.gui.status_bar.show_message('Discovering books with keywords ' + keywords)

        if incremental:
            incremental_sequence = plugin_config[IncrementalSearchSequence]
            plugin_config[IncrementalSearchSequence] = incremental_sequence + 1
        else:
            incremental_sequence = None

        # Start a background job as separate thread in this process to do the actual searches
        # Pass all required configuration so that changes made during the run of the job will not cause problems
        start_search_jobs(
                self, orig_discovered_books_version, self.config, all_calibre_books, selected_ids,
                discover_books, orig_discovered_books, keywords, incremental_sequence, Dispatcher(self._search_complete))

    def get_availability(self, check_needed):
        selected_ids = self.gui.library_view.get_selected_ids()
        if not selected_ids:
            error_dialog(self.gui, 'No Books Selected', 'Select books to enable availability checking', show=True)
            return

        if not self.config.disable_confirm_dlgs:
            q = '<p>Check %s for current availability to borrow or expected wait%s?<p>(This will run as a background job.)' % (
                value_unit(len(selected_ids), 'selected %s book' % ('needed' if check_needed else 'linked')),
                ' and save results in a custom column' if self.lib_has_custom_column(FIELD_WAIT_WEEKS) else '')

            if not question_dialog(self.gui, 'Start check', q):
                return

        selected_books = self.collect_book_metadata(
                selected_ids,
                collect_wait_weeks_field=IDENT_NEEDED_LINK if check_needed else IDENT_AVAILABLE_LINK)  # read database
        if selected_books is None:
            return

        # Start a background job as separate thread in this process to do the actual checks
        start_search_jobs(self, 0, self.config, selected_books, selected_ids, False, [],
                          'get_current_availability_', None, Dispatcher(self._search_complete))

    def unlink_books(self, all_links):
        selected_ids = self.gui.library_view.get_selected_ids()
        if not selected_ids:
            error_dialog(self.gui, 'No Books Selected', 'Select books to enable unlinking', show=True)
            return

        selected_books = self.collect_book_metadata(selected_ids)   # Gather book data from the database
        if selected_books is None:
            return

        if not all_links:
            for book in selected_books:
                book.preserve_disabled_links(self.config)  # Keep any links for disabled libraries

        self._report_search_completion((0, selected_books, [], [], [], ['Unlink books']), None)

    def remove_single_link(self, ltype, odlink):
        selected_ids = self.gui.library_view.get_selected_ids()
        if not selected_ids:
            error_dialog(self.gui, 'No Book Selected', 'Select a book to enable unlinking', show=True)
            return

        selected_books = self.collect_book_metadata(selected_ids)   # Gather book data from the database
        if len(selected_books) != 1:
            error_dialog(self.gui, 'Multiple Books Selected', 'Select a single book to remove one link', show=True)
            return

        book = selected_books[0]
        book.preserve_orig_links()
        book.remove_single_link(ltype, odlink)

        self._report_search_completion((0, selected_books, [], [], [], ['Remove single link from a book']), None)

    def unlink_needed(self):
        selected_ids = self.gui.library_view.get_selected_ids()
        if not selected_ids:
            error_dialog(self.gui, 'No Books Selected', 'Select books to enable unlinking', show=True)
            return

        selected_books = self.collect_book_metadata(selected_ids)   # Gather book data from the database
        if selected_books is None:
            return

        for book in selected_books:
            book.preserve_orig_links()
            book.odnid = ''     # remove needed links

        self._report_search_completion((0, selected_books, [], [], [], ['Unlink needed']), None)

    def check_all_links(self):
        db = self.gui.current_db

        all_calibre_books = self.collect_book_metadata(db.new_api.all_book_ids())
        if all_calibre_books is None:
            return

        all_discovered_books = get_discovered_books(db)[0]

        total_books = len(all_calibre_books) + len(all_discovered_books)

        progress = QProgressDialog('Checking book links for errors', 'Cancel', 0, total_books, self.gui)
        progress.setWindowTitle(self.name)
        progress.setWindowFlags(progress.windowFlags() & (~Qt.WindowContextHelpButtonHint))
        progress.setMinimumWidth(400)
        progress.setMinimumDuration(500)    # Show progress quickly
        progress.setModal(True)
        progress.setValue(0)

        errors = []

        for i, book in enumerate(all_calibre_books):
            book.migrate_orig_links_to_current(self.config)
            book.check_links(errors, self.config)
            book.remove_outdated_links()

            progress.setValue(i + 1)        # calls QApplication.processEvents() for the modal dialog
            if progress.wasCanceled():
                return

        for i, book in enumerate(all_discovered_books):
            book.is_newly_discovered = False
            book.move_current_links_to_orig()
            book.migrate_orig_links_to_current(self.config)
            book.check_links(errors, self.config)
            book.remove_outdated_links()

            progress.setValue(i+len(all_calibre_books))    # calls QApplication.processEvents() for the modal dialog
            if progress.wasCanceled():
                return

        progress.reset()

        self._report_search_completion((None, all_calibre_books, all_discovered_books, errors, [], ['Check links']), None)

    def lib_has_custom_column(self, column_name):
        #print('custom columns: %s' % ', '.join(self.gui.current_db.custom_column_label_map.keys()))
        return column_name[1:] in self.gui.current_db.custom_column_label_map

    def collect_book_metadata(self, ids, ignore_ids=None, collect_wait_weeks_field=None,
                              collect_et_status=False, collect_epub_data=False):

        '''
        Collect and prepare book data for processing by the background job while displaying progress to the user
        '''
        progress = QProgressDialog('Collecting calibre book data', 'Cancel', 0, len(ids), self.gui)
        progress.setWindowTitle(self.name)
        progress.setWindowFlags(progress.windowFlags() & (~Qt.WindowContextHelpButtonHint))
        progress.setMinimumWidth(400)
        progress.setMinimumDuration(500)   # Show progress quickly
        progress.setModal(True)
        progress.setValue(0)

        if collect_wait_weeks_field and not self.lib_has_custom_column(FIELD_WAIT_WEEKS):
            collect_wait_weeks_field = None

        db = self.gui.current_db
        books = []

        for i, id in enumerate(ids):
            mi = db.new_api.get_proxy_metadata(id)

            levels_have, is_fxl = get_levels_have(mi.tags, ignore_ids) if get_levels_have is not None else ([], False)

            if collect_wait_weeks_field:
                wait_weeks = db.new_api.field_for(FIELD_WAIT_WEEKS, id, default_value='')
                if isinstance(wait_weeks, tuple):
                    wait_weeks = ', '.join(sorted(wait_weeks))  # convert tag-like values into single string
            else:
                wait_weeks = ''

            if collect_et_status:
                et_asin = mi.identifiers.get(IDENT_AMAZON, '')

                for odlink in (ODLinkSet(mi.identifiers.get(IDENT_AVAILABLE_LINK, '')) |
                               ODLinkSet(mi.identifiers.get(IDENT_PURCHASABLE_LINK, ''))).amazon().sorted_odlinks():
                    if odlink.amazon_ident() == IDENT_AMAZON:
                        et_asin = odlink.book_id
                        break

                et_status = db.new_api.field_for(FIELD_ET_STATUS, id)
            else:
                et_asin = ''
                et_status = None

            epub_file_name = None
            if collect_epub_data:
                b_formats = self.book_formats(id)
                # choose highest priority desired format that this book has (if any)
                for input_format in [AZW3_FILE_TYPE, EPUB_FILE_TYPE, MOBI_FILE_TYPE]:       # add more as needed
                    if input_format in b_formats:
                        epub_file_name = db.new_api.format(id, input_format, as_file=False, as_path=True, preserve_filename=True)
                        break

            books.append(CalibreBook(
                id=id, authors=[self.remove_author_suffix(a) for a in mi.authors],
                title=mi.title, series=mi.series, series_index=mi.series_index,
                isbn=mi.identifiers.get(IDENT_ISBN, ''),
                orig_odid=mi.identifiers.get(IDENT_AVAILABLE_LINK, ''),
                orig_odrid=mi.identifiers.get(IDENT_RECOMMENDABLE_LINK, ''),
                orig_odpid=mi.identifiers.get(IDENT_PURCHASABLE_LINK, ''),
                orig_odnid=mi.identifiers.get(IDENT_NEEDED_LINK, ''),
                orig_wait_weeks=wait_weeks,
                wait_weeks_field=collect_wait_weeks_field,
                orig_et_asin=et_asin,
                orig_et_status=et_status,
                epub_file_name=epub_file_name,
                levels_have=levels_have,
                is_fxl=is_fxl,
                last_modified=mi.last_modified))

            progress.setValue(i + 1)    # calls QApplication.processEvents() for the modal dialog
            if progress.wasCanceled():
                return None

        progress.reset()
        return books

    def remove_author_suffix(self, author):
        # remove suffix such as " [editor]"
        if author.endswith(']'):
            return author.partition('[')[0].strip()

        return author

    def book_formats(self, book_id):
        formats = self.gui.current_db.formats(book_id, index_is_id=True) or ''
        return set([x.lower().strip() for x in formats.split(',')])

    def _search_complete(self, job):
        '''
        Called upon completion of a search operation in a search background job
        '''
        if job.failed:
            self.gui.job_exception(job, dialog_title='%s search failed!' % self.name)
            return      # Note: unable to track book ids that were in use for this job!

        self.gui.status_bar.show_message('%s search completed' % self.name, 3000)

        if not hasattr(job, 'html_details'):
            job.html_details = job.details  # Details from ParallelJob are actually html.

        self._report_search_completion(job.result, job.html_details)

    def _report_search_completion(self, result, log):
        '''
        Handle search completion. Can also handle completion of check for ids to remove.
        '''

        def add_book_details(detail, book_message_list):
            for book, message, is_discovered in sorted(book_message_list, key=lambda book_message: book_message[0]):
                detail.append(html_escape(''.join(
                    (BOOK_DISCOVERED if is_discovered else '', str(book), ' (%s)' % message if message else ''))))

        def add_optional_key():
            if not self.key_used:
                keys = []
                keys.append(('Discovered book', BOOK_DISCOVERED))
                keys.append(('Available at', LINK_AVAILABLE))
                keys.append(('Recommendable at', LINK_RECOMMENDABLE))
                keys.append(('Purchasable at', LINK_PURCHASABLE))

                if DEBUG_MODE:
                    keys.append(('Needed', LINK_NEEDED))

                keys.append(('Audiobook', LINK_FORMAT_AUDIOBOOK))

                details.append('Key: %s' % ', '.join(['%s=%s' % (k2, k1) for k1, k2 in keys]))
                self.key_used = True

        def report_book_changes(book, is_discovered):
            # Compare the individual links to find those added and removed
            idlinks = book.get_link_identifiers(self.config)
            orig_idlinks = book.get_link_identifiers(self.config, orig=True)

            added_idlinks = {}
            removed_idlinks = {}
            any_added = False
            any_removed = False
            any_other_change = False

            if not is_discovered:
                if book.orig_wait_weeks != book.wait_weeks:
                    wait_weeks_changed.append((book, book.wait_weeks or 'Not available', False))
                    if self.lib_has_custom_column(FIELD_WAIT_WEEKS):
                        any_other_change = True

                if book.orig_et_status is not book.et_status:
                    et_status_changed.append((book, str(book.et_status), False))
                    any_other_change = True

                if book.orig_et_asin != book.et_asin:
                    et_asin_changed.append((book, book.et_asin, False))
                    any_other_change = True

                if book.inventory:
                    book_inventory.append((book, book.inventory, False))

            for ltype in book.idltypes:
                added_idlinks[ltype] = idlinks[ltype] - orig_idlinks[ltype]
                removed_idlinks[ltype] = orig_idlinks[ltype] - idlinks[ltype]

                any_added = any_added or len(added_idlinks[ltype]) > 0
                any_removed = any_removed or len(removed_idlinks[ltype]) > 0

            if any_added:
                added_to_library.append((book, linksets_str(added_idlinks), is_discovered))

            if any_removed:
                removed_from_library.append((book, linksets_str(removed_idlinks), is_discovered))

            if (any_added or any_removed or any_other_change):
                if is_discovered:
                    changed_discovered_books.append(book)
                else:
                    changed_books.append(book)

        orig_discovered_books_version, books, discovered_books, search_errors, search_warnings, search_summaries = result

        changed_books = []
        changed_discovered_books = []

        # Digest and summarize the results by category. The same book may be in multiple categories.
        # Each of these lists holds (book, message, discovered-t/f) tuples.
        discovered_at_library = []
        added_to_library = []
        removed_from_library = []
        wait_weeks_changed = []
        et_status_changed = []
        et_asin_changed = []
        book_inventory = []

        for book in discovered_books:
            if book.is_newly_discovered:
                discovered_at_library.append((book, book.links_str(self.config), True))
                changed_discovered_books.append(book)
            else:
                report_book_changes(book, True)

        for book in books:
            if (book.orig_odid or book.odid or
                    book.orig_odrid or book.odrid or
                    book.orig_odpid or book.odpid or
                    book.orig_odnid or book.odnid or
                    book.orig_wait_weeks or book.wait_weeks or
                    (book.orig_et_status is not None) or (book.et_status is not None) or
                    book.orig_et_asin or book.et_asin or
                    book.inventory):
                report_book_changes(book, False)

        # Show summary of books by category
        main_msg = 'Search complete'
        message = []
        details = []
        self.key_used = False

        if search_summaries:
            main_msg = search_summaries[0]

            if len(books) == 0 and len(discovered_books) == 0:
                details.extend(search_summaries)

        if search_errors:
            message.append('%s in searching' % value_unit(len(search_errors), 'error'))
            add_optional_key()
            details.append('<span style="color:red">---------------- ERRORS WHILE SEARCHING  ----------------</span>')
            details.extend([html_escape(e) for e in search_errors])

        if search_warnings:
            message.append('%s in searching' % value_unit(len(search_warnings), 'warning'))
            if self.config.report_warnings:
                add_optional_key()
                details.append('<span style="color:blue">--------------- WARNINGS WHILE SEARCHING  ---------------</span>')
                details.extend([html_escape(e) for e in search_warnings])

        if search_errors or search_warnings:
            message.append('(click "View log" for details)')

        if discovered_at_library:
            message.append('%s' % value_unit(len(discovered_at_library), 'discovered book'))
            add_optional_key()
            details.append('<span style="color:green">-------- DISCOVERED BOOKS (MISSING FROM CALIBRE) --------')
            details.append('(Choose "Manage discovered books" from the plugin menu to view these.)</span>')
            add_book_details(details, discovered_at_library)

        if added_to_library:
            message.append('%s to be added' % value_unit(len(added_to_library), 'book/library link'))
            add_optional_key()
            details.append('<span style="color:green">------ NEWLY FOUND BOOK/LIBRARY LINKS (TO BE ADDED) ------</span>')
            add_book_details(details, added_to_library)

        if removed_from_library:
            message.append('%s to be removed' % value_unit(len(removed_from_library), 'book/library link'))
            add_optional_key()
            details.append('<span style="color:green">------- REMOVED BOOK/LIBRARY LINKS (TO BE UNLINKED) ------</span>')
            add_book_details(details, removed_from_library)

        if wait_weeks_changed:
            message.append('%s to current availability' % value_unit(len(wait_weeks_changed), 'change'))
            add_optional_key()
            details.append('<span style="color:green">------------- CHANGES TO CURRENT AVAILABILITY -------------</span>')
            if not self.lib_has_custom_column(FIELD_WAIT_WEEKS):
                details.append('<span style="color:red">(Will not be saved due to missing %s custom column)</span>' % FIELD_WAIT_WEEKS)

            add_book_details(details, wait_weeks_changed)

        if et_asin_changed:
            message.append('%s to ET ASIN' % value_unit(len(et_asin_changed), 'change'))
            add_optional_key()
            details.append('<span style="color:green">----------- CHANGES TO ENHANCED TYPESETTING ASIN -----------</span>')
            add_book_details(details, et_asin_changed)

        if et_status_changed:
            message.append('%s to ET status' % value_unit(len(et_status_changed), 'change'))
            add_optional_key()
            details.append('<span style="color:green">---------- CHANGES TO ENHANCED TYPESETTING STATUS ----------</span>')
            add_book_details(details, et_status_changed)

        if book_inventory:
            message.append(value_unit(len(book_inventory), 'EPUB inventory'))
            add_optional_key()
            details.append('<span style="color:green">------------------------ INVENTORY ------------------------</span>')
            add_book_details(details, book_inventory)

        if not message:
            message.append('No changes.')

        message_str = self.name + ' found: ' + '. '.join(message)

        html_details = '\n\n'.join(details)
        details_str = html_to_text(html_details)
        full_log = '\n\n<table bgcolor="darkBlue" width="80%" height="4" align="center"><tr><td></td></tr></table>\n\n'.join(
                [s for s in [message_str, html_details, log] if s])

        log_msg = self.name + ' Log'
        main_msg = self.name + ': ' + main_msg

        proceed_param = (orig_discovered_books_version, changed_books, changed_discovered_books)

        # print('show_det=%s' % self.config.show_details_at_completion)

        if self.config.disable_job_completion_dlgs:
            self._proceed_with_update(proceed_param)

        elif changed_books or changed_discovered_books:
            message_str += '<p>Proceed with updating?'

            if changed_discovered_books:
                self.gui.proceed_question(
                    self._proceed_with_update, proceed_param, full_log,
                    log_msg, main_msg, message_str,
                    det_msg=details_str, show_copy_button=True,
                    action_callback=self._proceed_with_update_no_discovered,
                    action_label='Non-* only', action_icon=QIcon(I('ok.png')), focus_action=False,
                    show_det=self.config.show_details_at_completion,
                    cancel_callback=self._proceed_do_nothing, icon=self.main_icon)
            else:
                self.gui.proceed_question(
                    self._proceed_with_update, proceed_param, full_log,
                    log_msg, main_msg, message_str,
                    det_msg=details_str, show_copy_button=True,
                    show_det=self.config.show_details_at_completion,
                    cancel_callback=self._proceed_do_nothing, icon=self.main_icon)

        else:
            message_str += '<p>No changes will be made.'

            self.gui.proceed_question(
                self._proceed_do_nothing, proceed_param, full_log, log_msg, main_msg, message_str,
                det_msg=details_str, show_copy_button=True,
                show_det=self.config.show_details_at_completion, show_ok=True,
                cancel_callback=self._proceed_do_nothing,
                icon=self.main_icon)

    def _proceed_do_nothing(self, proceed_param):
        self._proceed_with_update((0, [], []), save_discovered_books=False)

    def _proceed_with_update_no_discovered(self, proceed_param):
        self._proceed_with_update(proceed_param, save_discovered_books=False)

    def _proceed_with_update(self, proceed_param, save_discovered_books=True):
        # any uncaught exceptions will prevent further proceed question dialogs, so deal with everything here
        try:
            orig_discovered_books_version, changed_books, changed_discovered_books = proceed_param

            if changed_discovered_books and save_discovered_books:
                new_books_discovered(self.gui.current_db, changed_discovered_books,
                                     orig_discovered_books_version, self.gui, self.config)

            if changed_books:
                db = self.gui.current_db
                modified = []

                with db.new_api.read_lock:
                    for book in changed_books:
                        if not db.new_api.has_id(book.id):
                            modified.append('%s (Removed)' % str(book))
                            book.id = None
                        else:
                            db_last_modified = db.new_api.get_proxy_metadata(book.id).last_modified
                            if db_last_modified > book.last_modified:
                                modified.append('%s (Modified)' % str(book))

                apply_changes = True
                if modified and not self.config.disable_job_completion_dlgs:
                    if not question_dialog(
                            self.gui, 'Some books changed',
                            '<p>The metadata for some books in your library has changed since you started the search. If you'
                            ' proceed, some of those changes may be overwritten.<p>Click "Show details" to see the list of'
                            ' changed books.<p>Do you want to proceed?',
                            det_msg='\n'.join(sorted(modified))):
                        apply_changes = False

                if apply_changes:
                    # Apply the changes to the database while displaying a progress dialog
                    progress = QProgressDialog('Applying changes to book metadata', 'Cancel', 0, len(changed_books), self.gui)
                    progress.setWindowTitle(self.name)
                    progress.setWindowFlags(progress.windowFlags() & (~Qt.WindowContextHelpButtonHint))
                    progress.setMinimumWidth(400)
                    progress.setMinimumDuration(30000 if self.config.disable_job_completion_dlgs else 100)   # Show progress quickly
                    progress.setModal(True)
                    progress.setValue(0)

                    updated_ids = []
                    marked_ids = {}

                    with db.new_api.write_lock:
                        for i, book in enumerate(changed_books):
                            if book.id is not None:
                                updated = False

                                if book.links_have_changed() or (book.et_asin != book.orig_et_asin):
                                    mi = db.new_api.get_metadata(book.id, get_user_categories=False)  # get existing identifiers for book

                                    book.remove_outdated_links()

                                    for ltype in book.idltypes:
                                        mi.set_identifier(ltype, book.get_updated_link_identifier(ltype, mi.identifiers.get(ltype, '')))

                                    if self.config.update_amazon_idents:
                                        for odlink in (
                                                ODLinkSet(book.odid) | ODLinkSet(book.odrid) |
                                                ODLinkSet(book.odpid) | ODLinkSet(book.odnid)).amazon().sorted_odlinks():
                                            mi.set_identifier(odlink.amazon_ident(), odlink.book_id)

                                    if book.et_asin != book.orig_et_asin:
                                        mi.set_identifier(IDENT_AMAZON, book.et_asin)

                                    db.new_api.set_field('identifiers', {book.id: mi.identifiers})  # replace all identifiers
                                    updated = True

                                if book.wait_weeks != book.orig_wait_weeks and self.lib_has_custom_column(FIELD_WAIT_WEEKS):
                                    db.new_api.set_field(FIELD_WAIT_WEEKS, {book.id: book.wait_weeks})

                                    if self.lib_has_custom_column(FIELD_WAIT_WEEKS_DATE):
                                        db.new_api.set_field(FIELD_WAIT_WEEKS_DATE, {book.id: now()})

                                    updated = True

                                if book.et_status != book.orig_et_status and self.lib_has_custom_column(FIELD_ET_STATUS):
                                    db.new_api.set_field(FIELD_ET_STATUS, {book.id: book.et_status})
                                    updated = True

                                if book.et_status is True and self.lib_has_custom_column(FIELD_ET_DATE):
                                    orig_et_date = db.new_api.field_for(FIELD_ET_DATE, book.id, default_value='')
                                    if not orig_et_date:
                                        db.new_api.set_field(FIELD_ET_DATE, {book.id: now()})
                                        updated = True

                                if updated:
                                    updated_ids.append(book.id)

                                    if self.config.mark_changed_books:
                                        marked_ids[book.id] = 'updated'

                            progress.setValue(i + 1)    # calls QApplication.processEvents() for the modal dialog

                    progress.reset()

                    if marked_ids:
                        db.set_marked_ids(marked_ids)  # Mark results

                    # Update the gui view to reflect changes to the database
                    if updated_ids:
                        self.gui.library_view.model().refresh_ids(updated_ids)
                        current = self.gui.library_view.currentIndex()
                        self.gui.library_view.model().current_changed(current, current)
                        self.gui.tags_view.recount()

        except Exception as e:
            traceback.print_exc()
            error_dialog(self.gui, 'Unhandled exception', repr(e), det_msg=traceback.format_exc(), show=True)

    def gutenberg_index(self):
        if not self.config.disable_confirm_dlgs:
            q = '<p>Download Project Gutenberg catalog and build search index?<p>(This will run as a background job.)'

            if not question_dialog(self.gui, 'Start download', q):
                return

        self.gui.status_bar.show_message('Building Project Gutenberg search index')

        # Start a background job
        start_search_jobs(self, 0, self.config, [], [], True, [], 'build_gutenberg_index_', None, Dispatcher(self._search_complete))

    def handle_discovered_books(self):
        db = self.gui.current_db
        if not have_discovered_books(db):
            error_dialog(
                self.gui, 'No Discovered Books',
                'No additional books were found using "Search for books by selected authors".',
                show=True)
            return

        abd = DiscoveredBooksDialog(plugin_action=self, db=db, parent=self.gui, config=self.config)
        if not abd.result():
            return

        # DiscoveredBooksDialog accepted.
        added_books, updated_books = abd.get_results()
        new_ids = []
        updated_ids = []
        marked_ids = {}

        if added_books:
            # Add selected books to the database while displaying a progress dialog
            progress = QProgressDialog('Adding discovered books', 'Cancel', 0, len(added_books), self.gui)
            progress.setWindowTitle(self.name)
            progress.setWindowFlags(progress.windowFlags() & (~Qt.WindowContextHelpButtonHint))
            progress.setMinimumWidth(400)
            progress.setMinimumDuration(500)   # Show progress quickly
            progress.setModal(True)
            progress.setValue(0)

            new_books = []

            for i, dbook in enumerate(added_books):
                book = CalibreBook(from_book=dbook)
                mi = Metadata(book.title, book.authors)

                if book.publisher:
                    mi.publisher = book.publisher

                if book.pubdate:
                    mi.pubdate = book.pubdate

                if book.isbn:
                    mi.set_identifier(IDENT_ISBN, book.isbn)

                if book.odid:
                    mi.set_identifier(IDENT_AVAILABLE_LINK, book.odid)

                    if DEBUG_MODE:
                        book.odnid = str(ODLinkSet(book.odid).ebooks())     # assume all new e-book links needed
                        book.remove_outdated_links()
                        mi.set_identifier(IDENT_NEEDED_LINK, book.odnid)

                if book.odrid:
                    mi.set_identifier(IDENT_RECOMMENDABLE_LINK, book.odrid)

                if book.odpid:
                    mi.set_identifier(IDENT_PURCHASABLE_LINK, book.odpid)

                if self.config.update_amazon_idents:
                    for odlink in (ODLinkSet(book.odid) | ODLinkSet(book.odrid) |
                                   ODLinkSet(book.odpid)).amazon().sorted_odlinks():
                        mi.set_identifier(odlink.amazon_ident(), odlink.book_id)

                if book.series:
                    mi.series = book.series
                    mi.series_index = book.series_index

                new_books.append((mi, {}))   # create a book with no formats (empty book)

                progress.setValue(i + 1)    # calls QApplication.processEvents() for the modal dialog

            new_ids, duplicates_ids = db.new_api.add_books(new_books)  # add the new books to calibre

            if self.config.mark_changed_books:
                for id in new_ids:
                    marked_ids[id] = 'updated'

            progress.reset()

        if updated_books:
            # Update selected books in the database while displaying a progress dialog
            progress = QProgressDialog('Updating discovered books', 'Cancel', 0, len(updated_books), self.gui)
            progress.setWindowTitle(self.name)
            progress.setWindowFlags(progress.windowFlags() & (~Qt.WindowContextHelpButtonHint))
            progress.setMinimumWidth(400)
            progress.setMinimumDuration(500)   # Show progress quickly
            progress.setModal(True)
            progress.setValue(0)

            for i, dbook in enumerate(updated_books):
                book = CalibreBook(from_book=dbook)
                mi = db.new_api.get_metadata(book.id, get_user_categories=False)

                book.orig_odid = mi.identifiers.get(IDENT_AVAILABLE_LINK, '')
                book.orig_odrid = mi.identifiers.get(IDENT_RECOMMENDABLE_LINK, '')
                book.orig_odpid = mi.identifiers.get(IDENT_PURCHASABLE_LINK, '')

                if DEBUG_MODE:
                    book.orig_odnid = mi.identifiers.get(IDENT_NEEDED_LINK, '')
                    book.odnid = book.odid

                book.preserve_orig_links()   # only add links, don't remove
                book.remove_outdated_links()

                mi.set_identifier(IDENT_AVAILABLE_LINK, book.odid)
                mi.set_identifier(IDENT_RECOMMENDABLE_LINK, book.odrid)
                mi.set_identifier(IDENT_PURCHASABLE_LINK, book.odpid)

                if DEBUG_MODE:
                    mi.set_identifier(IDENT_NEEDED_LINK, book.odnid)

                if self.config.update_amazon_idents:
                    for odlink in (ODLinkSet(book.odid) | ODLinkSet(book.odrid) |
                                   ODLinkSet(book.odpid)).amazon().sorted_odlinks():
                        mi.set_identifier(odlink.amazon_ident(), odlink.book_id)

                if book.isbn and IDENT_ISBN not in mi.identifiers:
                    mi.set_identifier(IDENT_ISBN, book.isbn)

                if book.publisher and not mi.publisher:
                    mi.publisher = book.publisher

                try:
                    if book.pubdate and (mi.pubdate is None or mi.pubdate <= UNDEFINED_DATE):
                        mi.pubdate = book.pubdate
                except TypeError:
                    pass    # avoid TypeError: can't compare offset-naive and offset-aware datetimes

                if book.series and not mi.series:
                    mi.series = book.series
                    mi.series_index = book.series_index

                # Could probably call set_metadata with force_changes=True to force replacement of empty
                # identifiers, but I am concerned that might erase other fields that should be left untouched.
                db.new_api.set_metadata(book.id, mi, set_title=False, set_authors=False)  # replace those fields changed to non-blank
                db.new_api.set_field('identifiers', {book.id: mi.identifiers})  # replace all identifiers to allow removal

                updated_ids.append(book.id)

                if self.config.mark_changed_books:
                    marked_ids[book.id] = 'updated'

                progress.setValue(i + 1)    # calls QApplication.processEvents() for the modal dialog

            progress.reset()

        if marked_ids:
            db.set_marked_ids(marked_ids)  # Mark results

        if new_ids:
            db.data.books_added(new_ids)  # update database view to add new books
            self.gui.library_view.model().books_added(len(new_ids))  # add rows to table for new books
            self.gui.library_view.select_rows(new_ids)  # select for user to see

        if updated_ids:
            self.gui.library_view.model().refresh_ids(updated_ids)
            current = self.gui.library_view.currentIndex()
            self.gui.library_view.model().current_changed(current, current)

        self.gui.db_images.beginResetModel()    # update cover grid view
        self.gui.db_images.endResetModel()

        self.gui.tags_view.recount()    # update tag viewer panel

    def add_menu_action(self, menu, text, icon=None, tooltip=None, triggered=None, enabled=True, submenu=None):
        # Minimal version without keyboard shortcuts, etc.

        ac = menu.addAction(text)

        if tooltip:
            ac.setToolTip(tooltip)
            ac.setStatusTip(tooltip)    # This is the only one actually used
            ac.setWhatsThis(tooltip)

        if triggered:
            ac.triggered.connect(triggered)

        if icon:
            ac.setIcon(icon)

        ac.setEnabled(enabled)

        if submenu:
            ac.setMenu(submenu)

        return ac

    def set_library_group(self, lib_names):
        self.config = config_set_library_group(lib_names)

    def show_configuration(self):
        self.interface_action_base_plugin.do_user_config(self.gui)

    def show_help(self):
        self.show_text(self.help_text, 'Help', contents=True)

    def show_text(self, text, title, confirm=False, contents=False):
        dialog = QDialog(self.gui)
        layout = QVBoxLayout()

        text_browser = QTextBrowser()
        text_browser.setOpenExternalLinks(True)

        text_browser.setHtml(text)
        layout.addWidget(text_browser)

        button_layout = QHBoxLayout()

        if contents:
            top_button = QPushButton('Top (Contents)')
            top_button.setIcon(QIcon(I('arrow-up.png')))
            top_button.setToolTip('Scroll to top and show table of contents links')
            top_button.clicked.connect(functools.partial(text_browser.scrollToAnchor, "contents"))
            button_layout.addWidget(top_button)

        button_layout.addStretch()

        button_box = QDialogButtonBox(QDialogButtonBox.Close)
        button_box.rejected.connect(dialog.reject)

        if confirm:
            button_box.addButton(QDialogButtonBox.Ok)
            button_box.accepted.connect(dialog.accept)

        button_layout.addWidget(button_box)
        layout.addLayout(button_layout)

        dialog.setLayout(layout)
        dialog.setModal(True)
        dialog.resize(QSize(700, 500))
        dialog.setWindowTitle('%s %s' % (self.name, title))
        dialog.setWindowIcon(QIcon(I('help.png')))
        dialog.setWindowFlags(dialog.windowFlags() & (~Qt.WindowContextHelpButtonHint))
        dialog.exec_()

        return dialog.result() == QDialog.Accepted

    def have_new_release_notes(self):
        if tuple(plugin_config[ReleaseNoteVersion]) == DEFAULT_RELEASE_NOTE_VERSION:
            # Plugin version last installed missing. Either new install or previous version was from
            # before release note tracking.
            if self.config.any_enabled():
                plugin_config[ReleaseNoteVersion] = (1, 0, 0)  # plugin installed prior to release note tracking
            else:
                plugin_config[ReleaseNoteVersion] = ActionOverdriveLink.version  # new install of plugin

        return (LAST_RELEASE_NOTE_VERSION is not None) and (LAST_RELEASE_NOTE_VERSION > tuple(plugin_config[ReleaseNoteVersion]))

    def display_new_release_notes(self):
        display_notes = []

        for release_version, release_note in RELEASE_NOTES:
            if release_version > tuple(plugin_config[ReleaseNoteVersion]):
                display_notes.append(release_note)

        if self.show_text('<p><p>'.join(display_notes), 'Release Notes', confirm=True):
            plugin_config[ReleaseNoteVersion] = ActionOverdriveLink.version  # prevent showing again
