# -*- coding: utf-8 -*-
from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__   = 'GPL v3'
__copyright__ = '2014,2015,2016,2017,2018,2019 DaltonST <DaltonShiTzu@outlook.com>'
__my_version__ = "3.6.104"    # BeautifulSoup changed to use Standard Calibre's BS
import os, sys
import apsw
from calibre.library.database import LibraryDatabase
import datetime, re, sre_constants
from calibre.db.backend import DB
from calibre import isbytestring, force_unicode, prints
from calibre.constants import filesystem_encoding, preferred_encoding
from Queue import Empty, Queue
from calibre.gui2.threaded_jobs import ThreadedJob, BaseJob
from calibre.utils.logging import Log as log
import codecs
from time import sleep
import time
import datetime
import unicodedata
from difflib import SequenceMatcher
from copy import deepcopy

from calibre.ebooks.BeautifulSoup import BeautifulSoup

from calibre_plugins.quarantine_and_scrub.titlecase import titlecase
from calibre_plugins.quarantine_and_scrub.booklevel import (scrub_tags, add_book_awards, __build_book_awards_list,
                                                                                                    __scrub_isbn_in_tags, apply_tag_capitalization_rules,
                                                                                                    _convert_isbn_check_digit_13, _convert_isbn_convert_10_to_13,
                                                                                                    __build_regex_list_from_tag_rules,
                                                                                                    __build_regex_list_from_tag_capitalization_rules,
                                                                                                    __explode_custom_column_8_if_needed,
                                                                                                    __explode_custom_column_10_if_needed,
                                                                                                    __explode_custom_column_13_if_needed,
                                                                                                    __refresh_custom_column_15,  __similar,
                                                                                                    add_tag_combinations,
                                                                                                    do_uppercase,
                                                                                                    apply_tag_string_replacement_rules,
                                                                                                    _insert_single_tags_into_work_tags_single,
                                                                                                    add_missing_seriesname_from_web_detail,
                                                                                                    change_work_title_to_web_title_for_previously_validated_work_series,
                                                                                                    __update_non_qs_custom_columns_generic,
                                                                                                    miscellany_change_work_index_to_web_index_standalone )
#~ from calibre_plugins.quarantine_and_scrub.beautifulsoup3 import BeautifulSoup
from calibre_plugins.quarantine_and_scrub.naturalsort import natsort, natsort_key, natsort_coerce
from calibre_plugins.quarantine_and_scrub.heading import log_heading_common

import re

notifications = Queue()
mynothing = ""
my_miscellany_option = "0"
my_run_type = "0"

my_guidb = ""

set_of_letters = set('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')
set_of_letters.add('äëöüÜËÄÖáéíóúñçãÕõ')  #umlauts etc.  #copy to/from work/real is fine, but scrubbing screws it all up due to making a string of raw unicode out of it.

set_of_numbers =  set('0123456789')

set_of_symbols = set(".,[]{}|?<>;:-_=+*#@&!`~\\//")
set_of_symbols.add('"')
set_of_symbols.add("'")
set_of_symbols.add("'.,&-")


set_of_numbers_letters = deepcopy(set_of_letters)
set_of_numbers_letters.add('0123456789')

set_of_numbers_letters_spaces = deepcopy(set_of_numbers_letters)
set_of_numbers_letters_spaces.add(" ")
set_of_numbers_letters_spaces.add("  ")

set_of_title_characters = deepcopy(set_of_numbers_letters)
set_of_title_characters.add(".,[]{}|?<>;:-_=+*#@&!`~\\//")

set_of_numbers_and_symbols = deepcopy(set_of_numbers)
set_of_numbers_and_symbols.add(".,[]{}|?<>;:-_=+*#@&!`~\\//")

probable_tags_to_add = []
tag_regex_rules = []
tag_capitalization_regex_rules = []

single_work_tags_all = []

this_date = "2014-11-02"

my_selected_book_list = []

import urllib2
import socket

current_library_path = ""

header_s1 = None
header_s2 = None
header_s3 = None
header_s4 = None
header_s5 = None

# ---------------------------------------------------------------------------------------
#  Miscellaneous Scrubbing Options

# Option 1: Propagate Tags from/to Books with Identical Series  (except for Book Awards)
# Option 3: Book Level Tag Scrubbing [Selected Books]
# Option 4: Verify Series Index Using Web Lookup [Selected Books]
# Option 5: Rename Work Series Name to Web Series Name[Previously Verified Fiction Series Only]
# Option 9: Minimize Work Tags Using Tag Priorities [Selected Books]
# Option 10: Copy Tag & Title Rules From/To Previously Configured Q&S Libraries
# Option 14: Download Work Series/Titles/Indexes from Web Source for All Series for a Single Author [Single Author Only]
# Option 15: Copy Work Tags to Custom Columns(s) Per Table "Tags CC Mapping Control" [Selected Books]
# Option 16: Update Pseudonym Custom Column [Selected Books]
# ---------------------------------------------------------------------------------------

def main_scrub_miscellany(self, guidb, miscellany_option, run_type, selected_book_list, log=None, abort=None, notifications=True):

    global mynothing
    global my_run_type
    global my_miscellany_option
    global my_guidb
    global this_date
    global my_selected_book_list
    global current_library_path
    global header_s1
    global header_s2
    global header_s3
    global header_s4
    global header_s5

    my_miscellany_option = str(miscellany_option)
    my_run_type = str(run_type)

    my_selected_book_list = deepcopy(selected_book_list)
    del selected_book_list

    notifications.put((0.01, 'Beginning Miscellaneous Scrubbing'))

    sleep(0.5)

    #----------------------------------------------------------------------------------------------------------------
    db = guidb
    path = db.library_path
    __printsafe(path)
    if isbytestring(path):
        path = path.decode(filesystem_encoding)
    path = path.replace(os.sep, '/')
    path = os.path.join(path, 'metadata.db')
    path = path.replace(os.sep, '/')
    current_library_path  = path
    log("Library DB: " + path)
    __printsafe(path)
    try:
        my_db =apsw.Connection(path)
        #my_db = sqlite3.connect(path)
    except Exception as e:
        __printsafe(str(e))
        log(str(e))
        raise e
        return

    my_cursor = my_db.cursor()

    my_guidb = guidb


    header_s1 =  str("SQLite Version: " + str(apsw.SQLITE_VERSION_NUMBER) + "  [APSW]")   #e.g. 3.8.4

    mysql = str("PRAGMA main.locking_mode=EXCLUSIVE;")
    my_cursor.execute(mysql)  #apsw
    header_s2 = mysql

    sleep(0.5)

    Create_SQLite_User_Functions(my_db, my_cursor, log)

    #----------------------------------------------------------------------------------------------------------------

    Scrub_Control(my_db, my_cursor, notifications, log)

    #----------------------------------------------------------------------------------------------------------------

    my_db.close()

    __printsafe("Job complete.")
    log(" ")
    log(" ")
    log(" ")
    log(" ")
    log("=============")
    log("Job complete.")

    return

    #----------------------------------------------------------------------------------------------------------------
def Scrub_Control(my_db, my_cursor, notifications, log):
    global header_s1
    global header_s2
    global header_s3
    global header_s4
    global header_s5
    global mynothing
    global my_run_type
    global my_miscellany_option


    header_s3 = "Beginning Miscellaneous Scrubbing"

    if my_miscellany_option == "1":   #Propagation of Tags
        header_s4 = "Propagation of Work Tags"
        header_s5 = 'Propagating Work Tags from/to Books With The Identical Work Series'
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)
        misc1_control(my_db, my_cursor, notifications, log)
        apply_tag_string_replacement_rules(my_db, my_cursor, log)
        return


    if my_miscellany_option == "3":   #Tag Scrubber Standalone Job
        header_s4 = "Book Level Tag Scrubbing for Selected Books"
        header_s5 = None
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)

        misc3_control(my_db, my_cursor, notifications, log)
        apply_tag_string_replacement_rules(my_db, my_cursor, log)
        remove_duplicate_work_tags(my_db, my_cursor, log)         #possible if book awards for same book already created in book level scrubbing, and then this job does it again...
        return

    if my_miscellany_option == "4":   #Verify Series Index
        header_s4 = "Verify Work Series Index Using Web Lookup"
        header_s5 = "<font size='4'><b><font color='#990000'>Always Consolidate/Rename Series Prior To Running This Job</b></font>"
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)
        misc4_control(my_db, my_cursor, notifications, log)
        return

    if my_miscellany_option == "5":   #Rename Work Series Name to Web Series Name
        header_s4 = "Rename Work Series Name to Web Series Name"
        header_s5 = None
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        log("<font size='4'><b><font color='#990000'>This job uses the Historically Cumulative Web Source Series Validation Data previously downloaded anywhere in your Q&S ecosystem.</b></font>")
        log("<font size='4'><b><font color='#990000'>First run the Copy Tag & Title Rules and WSSVD job to copy all WSSVD among all of your Q&S Libraries.</b></font>")
        log("<font size='4'><b><font color='#990000'>WSSVD is created by the 2 jobs, [A] and [B], listed below:</b></font>")
        log("<font size='4'><b><font color='#990000'>[A] Download Web Series/Title/Indexes from Web Source for All Series for a Single Author (Regardless of Work Series).</b></font>")
        log("<font size='4'><b><font color='#990000'>[B] Validate Work Series/Title/Indexes from Web Source for All Selected Work Series.</b></font>")
        log("===============================================================================================================================")
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)
        misc5_control(my_db, my_cursor, notifications, log)
        return

    if my_miscellany_option == "9":   # Minimize Work Tags Using Tag Priorities for All Books
        header_s4 =  "Minimize Work Tags Using Tag Priorities for All Books"
        header_s5 = None
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)
        remove_duplicate_work_tags(my_db, my_cursor, log)
        misc9_control(my_db, my_cursor, notifications, log)
        apply_tag_string_replacement_rules(my_db, my_cursor, log)
        return

    if my_miscellany_option == "10":   # Copy Tag & Title Rules From/To Previously Configured Q&S Libraries
        header_s4 = "Copy Tag & Title Rules From/To Previously Configured Q&S Libraries"
        header_s5 = None
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)
        misc10_control(my_db, my_cursor, notifications, log)
        return

    if my_miscellany_option == "14":   # Download Work Series/Titles/Indexes from Web Source for All Series for a Single Author [Single Author Only]
        header_s4 = "Download Work Series/Titles/Indexes from Web Source for All Series for a Single Author [Single Author Only]"
        header_s5 = None
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)
        misc14_control(my_db, my_cursor, notifications, log)
        return

    if my_miscellany_option == "15":   # Copy Work Tags to Custom Columns(s) Per Table "Tags CC Mapping Control" [Selected Books]
        header_s4 = 'Copy Work Tags to Custom Columns(s) Per Table "Tags CC Mapping Control" [Selected Books]'
        header_s5 = None
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)
        misc15_control(my_db, my_cursor, notifications, log)
        return

    if my_miscellany_option == "16":   # Update Pseudonym Custom Column [Selected Books]
        header_s4 = 'Update Pseudonym Custom Column [Selected Books]'
        header_s5 = None
        log_heading_common(log,header_s1,header_s2,header_s3,header_s4,header_s5)
        mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
        my_cursor.execute(mysql)
        misc16_control(my_db, my_cursor, notifications, log)
        return


    return

    #----------------------------------------------------------------------------------------------------------------
def misc1_control(my_db, my_cursor, notifications, log):

    notifications.put((0.01, 'Beginning Propagation of Tags'))
    print("Beginning Miscellaneous Option #1 Scrubbing")

    global mynothing
    global my_run_type
    global my_miscellany_option

    sleep(0.02)

    __build_regex_list_from_tag_capitalization_rules(my_db, my_cursor, log)

    sleep(0.02)

    mysql = str('DELETE FROM _tags_by_series ')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    print(str(mysql))

    sleep(0.5)

    mysql = str("INSERT OR REPLACE INTO _tags_by_series SELECT seriesname,tagsall FROM __book_series_tags  \
                        WHERE seriesname NOT NULL AND tagsall NOT NULL AND tagsall <> 'None' ")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(1.5)


    #-----------------------------------------------------------------------------------------------------------------------------------
    #change the oldtags in the book_awards_mapping table to newtags per the tags rule table.
    tmp_rows = []
    del tmp_rows
    mysql = "SELECT book,award,newtag FROM _tag_rules,_book_awards_mapping WHERE _book_awards_mapping.award = _tag_rules.oldtag and _tag_rules.purgetag = '0'  "
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        pass
    else:
        if len(tmp_rows) == 0:
            pass
        else:
            for row in tmp_rows:
                book,award,newtag = row
                mysql = "DELETE FROM _book_awards_mapping WHERE book = ?  AND  award = ?"
                __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql,book,award)
                mysql = "INSERT OR REPLACE INTO _book_awards_mapping (book,award) VALUES (?,?)"
                __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, book,newtag)
            #END FOR

    #-----------------------------------------------------------------------------------------------------------------------------------
    #Do for Unchanged Award Tags and Tag Rule-Changed Award Tags
    #need to remove any book award since the award is by book, not by series; add it back to the correct book later using the _book_award_mapping table.

    book_awards_mapping_list = []

    tmp_rows = []
    del tmp_rows
    mysql = "SELECT book,award FROM _book_awards_mapping "
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        pass
    else:
        if len(tmp_rows) == 0:
            pass
        else:
            for row in tmp_rows:
                book_awards_mapping_list.append(row)


    book_awards_mapping_list.sort()

    awards_list = []
    tag_rules_list = []

    tmp_rows = []
    del tmp_rows
    mysql = "SELECT oldtag,newtag FROM _book_awards,_tag_rules WHERE  (oldtag = award) AND purgetag = 0  GROUP BY oldtag,newtag"
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        pass
    else:
        if len(tmp_rows) == 0:
            pass
        else:
            for row in tmp_rows:
                oldtag,newtag = row       #  "Nebula Award",            "Nebula Award for Science Fiction"
                oldtag = str(oldtag)
                newtag = str(newtag)
                newtag = str(__change_tuple_to_bytestring(newtag))
                oldtag = str(__change_tuple_to_bytestring(oldtag))
                awards_list.append(str(newtag))
                tag_rules_list.append(str(oldtag))

    tmp_rows = []
    del tmp_rows
    mysql = "SELECT award FROM _book_awards "
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        pass
    else:
        if len(tmp_rows) == 0:
            pass
        else:
            for row in tmp_rows:
                for col in row:
                    award = str(col)         #  "Nebula Award"
                    award = str(__change_tuple_to_bytestring(award))
                    if not award in tag_rules_list:  #oldtags no longer exist as tags
                        if str(award) <> str("Award"):
                            award = str(award.title())
                            awards_list.append(str(award))

    tmp_set = set(awards_list)
    awards_list = list(tmp_set)  #duplicates now gone
    awards_list.sort(reverse=True)    #Avoid  Prometheus Award first replacing The Prometheus Award, leaving an artifact of The.

    tmp_rows = []
    del tmp_rows
    mysql = "SELECT workseries,worktags FROM _tags_by_series WHERE workseries NOT NULL AND worktags NOT NULL"
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        pass
    else:
        if len(tmp_rows) == 0:
            pass
        else:
            for row in tmp_rows:
                workseries,worktags  = row         #  Fiction:-Paranormal, Fiction:Alternative-History, Fiction:Fantasy-Dark-Fantasy, Fiction:Thrillers-Medical, Hugo Award, Fiction:Thrillers-Espionage
                orig_tags = worktags
                orig_workseries = workseries
                worktags = unicodedata.normalize('NFKD',worktags).encode('ascii', 'ignore')
                worktags = str(worktags)
                worktags = str(__change_tuple_to_bytestring(worktags))
                try:
                    worktags = str(worktags.lower())
                except:
                    pass
                #log("_tags_by_series raw row: " + workseries + "<>" + str(worktags))
                for item in awards_list:
                    award = str(item)
                    award = str(__change_tuple_to_bytestring(award))
                    try:
                        award = str(award.lower())
                    except:
                        pass
                    s0 = str( ", " + str(award) + ",")
                    #log(str(s0))
                    worktags = str(worktags.replace(s0,"",10))  #
                    s0 = str(str(award) + ",")
                    #log(str(s0))
                    worktags = str(worktags.replace(s0,"",10)) #  Fiction:-Paranormal, Fiction:Alternative-History, Fiction:Fantasy-Dark-Fantasy, Fiction:Thrillers-Medical, , Fiction:Thrillers-Espionage
                    s0 = str( ", " + str(award))
                    #log(str(s0))
                    worktags = str(worktags.replace(s0,"",10))  #
                    if not "," in worktags:
                        s0 = str(award)
                        worktags = str(worktags.replace(s0,"",1))  #
                #log("_tags_by_series row after replacing awards with nothing: " + workseries + "<>" + str(worktags))
                if worktags.endswith(","):
                    worktags = str(worktags[0: -1])
                if worktags.startswith(","):
                    worktags = str(worktags[1: ])
                worktags = str(worktags.replace(",,",",",50))
                worktags = str(worktags.replace(", ,",",",50))       #  Fiction:-Paranormal, Fiction:Alternative-History, Fiction:Fantasy-Dark-Fantasy, Fiction:Thrillers-Medical, Fiction:Thrillers-Espionage
                worktags = str(worktags.replace("  "," ",50))
                if worktags.endswith(","):
                    worktags = str(worktags[0: -1])
                if worktags.startswith(","):
                    worktags = str(worktags[1: ])
                worktags = str(worktags.strip())
                mysql = "DELETE FROM _tags_by_series WHERE workseries = ? AND worktags = ? "
                __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, orig_workseries, orig_tags)
                #log("deleted from tags by series table: " + workseries + "  "  +orig_tags)
                mysql = "INSERT OR REPLACE INTO _tags_by_series (workseries,worktags) VALUES (?,?) "
                __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, orig_workseries, worktags)
                #log("inserted into tags by series table:    >" + workseries + ">" + worktags )


    #-----------------------------------------------------------------------------------------------------------------------------------
    notifications.put((0.40, 'Collecting Tags By Series'))

    mysql = str("SELECT book, seriesname from __book_series_tags WHERE book NOT NULL AND seriesname NOT NULL ")
    print(str(mysql))
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        log("No Tags for Books With Series Found: " + str(seriesname))
        print("No Tags for Books With Series Found")
        return
    else:
        notifications.put((0.60, 'Propagating Work Tags by Book'))
        for row in tmp_rows:
            book, seriesname = row
            book = str(book)
            book = str(__strip_numerics(book))
            utf8_seriesname = seriesname
            seriesname = str(__change_tuple_to_bytestring(seriesname))
            misc1__process_tags(my_db, my_cursor, log, book, seriesname, utf8_seriesname, book_awards_mapping_list)
        #END FOR

    #     delete unused tags from cc13 itself
    mysql = 'DELETE FROM custom_column_13 WHERE id IN __tags_unused'
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    print(str(mysql))
    print("unused cc13 rows deleted")

    sleep(0.05)

    #now need to convert to utf8 value in cc13....
    mysql = "UPDATE custom_column_13 SET value = (update_utf8_for_display(value) ) WHERE value NOT NULL;"
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    print(str(mysql))
    print("cc13 updated for UTF8 display")


    sleep(0.05)

    mysql = str('DELETE FROM _tags_by_series ')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    #-----------------------------------------------------------------------------------------------------------------
def misc1__process_tags(my_db, my_cursor, log, book, seriesname, utf8_seriesname, book_awards_mapping_list):

    global mynothing

    #skip if there is only a single book with a particular series; nothing to propagate.
    mysql = "SELECT Count(*) FROM __book_series_tags WHERE seriesname = ? AND seriesname NOT NULL"
    my_cursor.execute(mysql,[utf8_seriesname])
    count1 = my_cursor.fetchall()
    for row in count1:
        for col in row:
            count1 = col
    count1 = str(__change_tuple_to_bytestring(count1))
    if count1 == "1" or count1 == "0" :
        #log("count1a [count is 1 or 0]: " + count1 + " for: " + utf8_seriesname)
        return
    else:
        #log("count1b [count > 1]: " + count1 + " for: " + utf8_seriesname)
        pass


    tmp_tagsall_list = misc1_fetch_tagsall(my_db, my_cursor, log, seriesname)


    #     merge the tagsall from all of the rows for the book's series from _tags_by_series
    #     split along commas, creating a list of single tags per row

    tmp_list_all = []
    del tmp_list_all
    tmp_list_all = []
    for row in tmp_tagsall_list:
        s = str(row)
        s_list = s.split(",")
        for item in s_list:
            s1 = str(item)
            s1 = str(__change_tuple_to_bytestring(s1))
            if str(s1) == str('None'):
                continue
            try:
                s1 = s1.lower()  #for duplicate elimination
            except:
                pass
            tmp_list_all.append(str(s1))
        #END FOR
    #END FOR

    tmp_set = set(tmp_list_all)         #eliminate duplicates in the list
    tmp_list_all = list(tmp_set)
    tmp_list_all.sort()
    del tmp_set

    #capitalize the new tags using table _tag_capitalization_rules, then reconcatenate into tagsall format
    new_tagsall = str("")
    for row in tmp_list_all:
        s = str(row)
        s = str(__change_tuple_to_bytestring(s))
        try:
            s = str(s.lowercase())
        except:
            pass
        if not "\\" in s:
            s = str(apply_tag_capitalization_rules(str(s),log))
        if s > " ":
            if "isbn" in s or s == "None" or s == "none" :
                pass
            else:
                new_tagsall = str(new_tagsall + ", " + str(s))
    #END FOR

    #log("[1]for book:", str(book), " new_tagsall is:", str(new_tagsall))

    #now add any awards, if any
    for row in book_awards_mapping_list:
        awardbook,award = row
        awardbook = str(awardbook)
        award = str(award)
        book = str(book)
        if str(book) == str(awardbook):
            if not award in new_tagsall:     #avoid duplicate tags
                if not "\\" in award:
                    award = str(apply_tag_capitalization_rules(str(award),log))
                new_tagsall = new_tagsall + ", " + award


    if new_tagsall.startswith(","):
        new_tagsall = str(new_tagsall[1: ])
        new_tagsall = str(new_tagsall.strip())
    if new_tagsall.endswith(","):
        new_tagsall = str(new_tagsall[0: -1])
        new_tagsall = str(new_tagsall.strip())


    #log("[2]for book:", str(book), " new_tagsall is:", str(new_tagsall))

    if new_tagsall == mynothing:
        return
    else:
        #new_tagsall = unicode(new_tagsall)
        new_tagsall = "u'" + str(new_tagsall) + "'"

    sleep(0.02)

    #     delete the old tagsall link for the book  from the cc13 link table
    mysql = str('DELETE FROM books_custom_column_13_link WHERE book = "' + book  + '" ;')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    print(str(mysql))

    sleep(0.02)

    #     add the new tags to cc13 using the book as the id
    mysql = str("INSERT OR REPLACE INTO custom_column_13 (id,value) VALUES (?,?)")
    __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, book, new_tagsall)
    print(str(mysql))

    sleep(0.02)

    #     add the new tags to bookscc13link with id=book, book=book,value=book
    mysql = str("INSERT OR REPLACE INTO books_custom_column_13_link (id,book,value) VALUES (?,?,?)")
    __execute_mysql_for_custom_column_generic_3_args(my_db, my_cursor, log, mysql, book,book,book)
    print(str(mysql))

    sleep(0.02)
    #-----------------------------------------------------------------------------------------------------------------
def misc1_fetch_tagsall(my_db, my_cursor, log, seriesname):
    # using seriesname, fetch the tagsall from table _tags_by_series  (1 series can have multiple rows)

    tmp_tagsall_list = []
    del tmp_tagsall_list
    tmp_tagsall_list = []

    sleep(0.05)

    mysql = str('SELECT worktags  from  _tags_by_series  WHERE workseries = "' + seriesname + '" ;')
    tmp_tagsall = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    for row in tmp_tagsall:
        tagsall = str(row)
        tagsall = str(__change_tuple_to_bytestring(tagsall))
        try:
            tagsall = str(tagsall.lower())
        except:
            pass
        #print("for series:", str(seriesname), "   misc1_fetch_tagsall: ", str(tagsall))
        tmp_tagsall_list.append(tagsall)
        #log("for series: " + str(seriesname) + "   misc1_fetch_tagsall: " + str(tagsall))

    sleep(0.05)


    return tmp_tagsall_list
    #-----------------------------------------------------------------------------------------------------------------
def misc3_control(my_db, my_cursor, notifications, log):
    #Tag Scrubber Standalone Job.  Can run before Booklevel has run, but Booklevel will run again automatically anyway.  Remember new tags derived from title too.
    #uses the identical functions as Booklevel via import.
    global mynothing
    global set_of_letters
    global set_of_numbers
    global set_of_numbers_letters
    global set_of_symbols
    global set_of_numbers_and_symbols
    global probable_tags_to_add
    global tag_regex_rules
    global single_work_tags_all    #same as in booklevel.py for use in _insert_single_tags_into_work_tags_single()
    global my_selected_book_list

    if isinstance(probable_tags_to_add, list):
        probable_tags_to_add[:] = []
    else:
        probable_tags_to_add = []

    n = len(my_selected_book_list)
    log(" ")
    log("Number of books selected for Tag Scrubbing: " + str(n))
    log("New Work Tags also will be derived from Comments per table _tags_by_comment.")
    log("New Work Tags also will be derived per table _tag_combination_rules.")
    log(" ")

    sleep(0.02)

    __build_regex_list_from_tag_rules(my_db, my_cursor, log)

    sleep(0.02)

    __build_regex_list_from_tag_capitalization_rules(my_db, my_cursor, log)

    sleep(0.02)

    __explode_custom_column_13_if_needed(my_db, my_cursor, log)

    sleep(0.02)

    __build_book_awards_list(my_db, my_cursor, log)

    sleep(0.02)

    tmp_books_list = []

    for row in my_selected_book_list:
        books = str(row)
        books = str(__strip_numerics(books))
        tmp_books_list.append(books)
    #END FOR
    del my_selected_book_list

    n_total_tags_added = 0

    n_total = len(tmp_books_list)
    n_count = 0
    if n_total == 0:
        log("No books selected for Tag Scrubbing")
        return
    for row in tmp_books_list:
        my_current_book = str(row)
        probable_tags_to_add[:] = []
        probable_tags_to_add = misc3_derive_tags_from_comments(my_db, my_cursor, my_current_book, notifications, log)
        n = len(probable_tags_to_add)
        n_total_tags_added  = n_total_tags_added + n
        sleep(0.02)
        probable_tags_to_add = add_book_awards(my_db, my_cursor, my_current_book, notifications, log, probable_tags_to_add)
        scrub_tags(my_db, my_cursor, my_current_book, notifications, log,probable_tags_to_add)
        n_count = n_count + 1
        n_progress = float(n_count/n_total)
        notifications.put((n_progress, 'Deriving & Scrubbing Tags for Selected Books'))
    #END FOR

    log(" ")
    log("Total New Work Tags Derived From Comments Prior to Tag Scrubbing:  " + str(n_total_tags_added))
    log(" ")

    sleep(0.02)

    add_tag_combinations(my_db, my_cursor, notifications, log)

    mysql = 'DELETE FROM custom_column_13 WHERE id IN __tags_unused'
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.02)

    #now need to convert to utf8 value in cc13....
    mysql = 'UPDATE custom_column_13 SET value = (update_utf8_for_display(value) ) WHERE  id NOT NULL  ;'
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.5)

    mysql = 'DELETE FROM _tags_work_single'            #delete all records since all new latest single tags just were made available
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.5)

    _insert_single_tags_into_work_tags_single(my_db, my_cursor, notifications, log)
    #-----------------------------------------------------------------------------------------------------------------
def misc3_derive_tags_from_comments(my_db, my_cursor, my_current_book, notifications, log):

    tmp_list = []
    del tmp_list

    tags_to_add = []
    del tags_to_add
    tags_to_add = []

    mysql = str("SELECT tag FROM _tags_by_comment WHERE EXISTS (SELECT text FROM comments  WHERE book = '[CURRENTBOOK]' \
                        AND comments.text LIKE '%'||_tags_by_comment.comment||'%'  ) ")
    mysql = str(mysql.replace("[CURRENTBOOK]", my_current_book, 1))
    tmp_list = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_list:
        return tags_to_add
    else:
        if len(tmp_list) == 0:
            return tags_to_add
        else:
            for row in tmp_list:
                for col in row:
                    u0 = col
                    try:
                        tmp_tag = unicodedata.normalize('NFKD',u0).encode('ascii', 'ignore')
                    except:
                        pass
                    tmp_tag = str(tmp_tag)
                    tmp_tag = str(__change_tuple_to_bytestring(tmp_tag))
                    if tmp_tag <> mynothing:
                        tags_to_add.append(tmp_tag)
                        #__printsafe("Work Tag of: " + str(tmp_tag) + " Derived from Comments for Book: " + str(my_current_book))
                #END FOR
            #END FOR

    n = len(tags_to_add)
    if n  == 0:
        return tags_to_add

    temp = set(tags_to_add)
    tags_to_add = list(temp)
    del temp

    return tags_to_add
    #-----------------------------------------------------------------------------------------------------------------
def misc4_control(my_db, my_cursor, notifications, log):
    #download html to scrape for Series and Index

    global mynothing
    global my_run_type
    global this_date
    global my_selected_book_list

    log("A Series need have only one (1) book selected to invoke the Series Validation for all books having that Series.")
    n = len(my_selected_book_list)
    log("Number of books selected for Series Validation: " + str(n))

    tmp_books_list = []

    for row in my_selected_book_list:
        books = str(row)
        books = str(__strip_numerics(books))
        tmp_books_list.append(books)
    #END FOR
    del my_selected_book_list

    n_total = len(tmp_books_list)
    if n_total == 0:
        log("No books selected for Series Validation")
        return

    __explode_custom_column_10_if_needed(my_db, my_cursor, log)     #imported from booklevel.py

    socket.setdefaulttimeout(15)

    text_list = []

    misc4_purge_old_web_table_data(my_db, my_cursor, log)

    tmp_series_list = misc4_fetch_series(my_db, my_cursor, log, tmp_books_list)

    tmp_set = set(tmp_series_list)       #remove duplicate entries
    tmp_series_list = list(tmp_set)

    n = len(tmp_series_list)
    log("Number of Series Being Validated: " + str(n))
    log(" ")
    if n == 0:
        return
    n_total = len(tmp_series_list)
    n_counter = 0
    start_time = time.time()

    for row in tmp_series_list:

        #authname, seriesname = row    does not work...??
        s1 = str(row)
        s1 = str(__change_tuple_to_bytestring(str(s1)))
        n = s1.find("||@||")
        if n < 0:
            continue   #should never happen
        authname = str(s1[0:n])
        s1 = str(s1.replace(authname, mynothing,1))
        s1 = str(s1.replace("||@||", mynothing, 1))
        seriesname = str(s1)

        if "-" in authname:      #e.g. Cindy Holby - Wind 01 - Chase the Wind
            n = authname.find("-")
            if n > 0:    #not a leading - in authors name!
                print("authname with a dash in it:", str(authname))
                authname = str(authname[0:n])
                if authname.endswith("-"):
                    authname = str(authname[0:-1])
                authname = str(authname.strip())
        if not "," in authname:
            if " " in authname:
                auth_split = authname.split(" ")
                firstname = str(auth_split[0])
                lastname = str(auth_split[1])
            else:
                firstname = str(" ")
                lastname = str(authname)
        else:
            auth_split = authname.split(",")
            firstname = str(auth_split[1])
            lastname = str(auth_split[0])
        firstname = str(__change_tuple_to_bytestring(str(firstname)))
        lastname = str(__change_tuple_to_bytestring(str(lastname)))
        seriesname = str(__change_tuple_to_bytestring(str(seriesname)))
        seriesname = str(seriesname.strip())

        firstname = firstname.replace(".",mynothing,4)    #  J.D. Robb should be JD Robb for this purpose.

        n_progress = float(n_counter/n_total)
        #log("n_progress is:" + str(n_progress))
        notifications.put((n_progress, 'Validating Series'))
        n_counter = n_counter + 1

        text_list[:] = []
        log(" ")
        log("===================================================================================================================================================")
        log(" ")
        log(str(n_counter) + " of: " + str(n_total) )
        log(" ")
        log("Current Actual Work Series:    " + str(seriesname) + " --- for Work Author: " + str(firstname) + " " + str(lastname))

        orig_series = str(seriesname)

        seriesname = str(misc4_genericize_seriesname(str(seriesname)))

        log("Current Generic Work Series:   " + str(seriesname))

        use_url_1 = False
        use_url_2 = True

        if use_url_1 and use_url_2:
            return   #pick only one
        else:
            if not use_url_1 and not use_url_2:
                return  #pick one

        if use_url_1:
            my_url_domain = str("ww2.kdl.org")
            my_url = str("http://ww2.kdl.org/libcat/WhatsNext.asp?AuthorLastName=[AUTHORLASTNAME]&AuthorFirstName=[AUTHORFIRSTNAME]&SeriesName=[SERIESNAME]&BookTitle=&CategoryID=0&cmdSearch=Search&Search=1&grouping= ")
            my_url = str(my_url.replace("[AUTHORLASTNAME]", lastname, 1))
            my_url = str(my_url.replace("[AUTHORFIRSTNAME]", firstname, 1))
            seriesname = str(seriesname.replace(" ", "+", 20))
            my_url = str(my_url.replace("[SERIESNAME]", seriesname, 1))

        elif use_url_2:
            my_url_domain = str("www.fictiondb.com")
            my_url = str("http://www.fictiondb.com/series/author-series~[LETTER].htm")    #not really; just a placeholder.
        else:
            log("Domain Not Supported for Download of Series Info")
            return


        try:

            elapsed_time = time.time() - start_time
            if elapsed_time < 2.0 :
                #log("sleeping for:" + str(elapsed_time))
                sleep((2.0 - elapsed_time))
            start_time = time.time()

            #-------------------------------------------------------
            if use_url_1:
                try:
                    connection = urllib2.urlopen(my_url)
                    soup = BeautifulSoup(connection)    #http://www.crummy.com/software/BeautifulSoup/bs4/doc/
                    connection.close()
                    results = soup.body(text=True)
                    if not soup:
                        log("==========================================================")
                        log("ERROR[0].  Web URL Unavailable For This Series.  Continuing With Next.")
                        log("==========================================================")
                        continue
                except:  #socket timeout error
                    log("==========================================================")
                    log("ERROR[1].  Web URL Timed Out For This Series.  Continuing With Next.")
                    log("==========================================================")
                    continue

                if not results or len(results) == 0:
                    log("Download of Series Info Failed.  Job Terminating Early.")
                    return

                for row in results:
                    s = str(__change_tuple_to_bytestring(str(row)))
                    text_list.append(str(s))

                n_real_work_book_count, tmp_auth = misc4_unpack_soup_ww2_kdl_org(my_db, my_cursor, my_url, text_list, orig_series, log)
            #-------------------------------------------------------
            else:
                #-------------------------------------------------------
                if use_url_2:
                    n_real_work_book_count, tmp_auth = misc4_unpack_soup_www_fictiondb_com(my_db, my_cursor,orig_series, seriesname, firstname, lastname, log)
                    #-------------------------------------------------------
                else:
                    log("Domain Not Supported for Download of Series Info")
                    return
            #-------------------------------------------------------

            misc4_delete_web_table_duplicates(my_db, my_cursor, log)

            if n_real_work_book_count == 0 or tmp_auth == "None":
                log("<font size='3'><b><font color='#FF0000'>Web Source Did Not Respond For This Work Series</b></font>")
                continue

            misc4_validate_work_series_index(my_db, my_cursor, log, tmp_auth, orig_series, my_url, n_real_work_book_count, notifications)

            is_testing = False
            if is_testing:
                if n_counter > 2:   #good series with results only...
                    break
            else:
                pass

            continue
        except Exception as e:
            log(str(e))
            my_db.close()
            return
    #END FOR
    sleep(0.05)
    mysql = str('INSERT OR REPLACE INTO _web_source_url_control (source_url) VALUES( "' + my_url_domain + '") ')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    #keep everything downloaded in other tables available for searching without re-downloading
    mysql = str('INSERT OR REPLACE INTO _global_web_author_series SELECT * FROM _web_author_series')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.05)
    mysql = str('INSERT OR REPLACE INTO _global_web_series_detail SELECT * FROM _web_series_detail')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.05)
    mysql = str("DELETE FROM _global_web_author_series WHERE authname LIKE '%-%' OR authname like '%0%' or  authname like '%1%' \
            OR authname like '%2%' or  authname like '%3%' OR authname like '%4%' OR  authname like '%5%' OR authname like '%6%' \
            OR  authname like '%7%' OR authname like '%8%' OR  authname like '%9%' OR  authname like '%\%' \
            OR authname LIKE '%</span>%' OR authname LIKE '%</a>%' OR authname LIKE '%~%'  OR authname LIKE '%unknown%' \
            OR seriesname LIKE '%</span>%'   ")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.05)
    mysql = str("DELETE FROM _global_web_series_detail WHERE authname LIKE '%-%' OR authname like '%0%' or  authname like '%1%' \
            OR authname like '%2%' or  authname like '%3%' OR authname like '%4%' OR  authname like '%5%' OR authname like '%6%' \
            OR  authname like '%7%' OR authname like '%8%' OR  authname like '%9%' OR  authname like '%\%' \
            OR authname LIKE '%</span>%' OR authname LIKE '%</a>%' OR authname LIKE '%~%'  OR authname LIKE '%unknown%'  \
            OR seriesname LIKE '%</span>%' OR seriesindex = '0' \
            OR booktitle LIKE '%</span>%' OR booktitle LIKE '%</a>%' OR booktitle LIKE '%~%' \
            OR seriesname LIKE '%</span>%' OR booktitle LIKE '%</a>%' OR booktitle LIKE '%~%'    ")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.05)
    mysql = str("INSERT OR IGNORE INTO _global_authors SELECT null,authname,'none',' ' FROM _web_author_series")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.05)
    mysql = str("INSERT OR IGNORE INTO _global_series SELECT null,seriesname,seriesname FROM _web_author_series")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.05)



    my_db.close()
    #-----------------------------------------------------------------------------------------------------------------
def misc4_genericize_seriesname(seriesname):
    #this is also used by the misc5 rename function that mines the historical global web source data.

    global mynothing

    orig_seriesname = str(seriesname)

    seriesname = str(seriesname.replace("!", mynothing, 10))
    seriesname = str(seriesname.replace("?", mynothing, 10))
    seriesname = str(seriesname.replace(",", mynothing, 10))
    seriesname = str(seriesname.replace(";", mynothing, 10))
    seriesname = str(seriesname.replace(":", mynothing, 10))
    seriesname = str(seriesname.replace(".", mynothing, 10))
    seriesname = str(seriesname.replace("&", mynothing, 10))
    seriesname = str(seriesname.replace("'s", mynothing, 10))
    seriesname = str(seriesname.replace("'", mynothing, 10))
    seriesname = str(seriesname.replace('"', mynothing, 10))
    seriesname = str(seriesname.replace("/", " ", 10))
    seriesname = str(seriesname.replace(" and ", " ", 10))
    seriesname = str(seriesname.replace(" or ", " ", 10))
    seriesname = str(seriesname.replace(" of ", " ", 10))

    seriesname = str(seriesname.strip())

    if seriesname.startswith("Dr "):
        seriesname = str(seriesname[2: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("In the "):
        seriesname = str(seriesname[6: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("The "):
        seriesname = str(seriesname[4: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("An "):
        seriesname = str(seriesname[3: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("A "):
        seriesname = str(seriesname[2: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("It Is "):    # "It's
        seriesname = str(seriesname[6: ])
        seriesname = str(seriesname.strip())
    seriesname = str(seriesname.replace(" & the ", " ", 1))
    seriesname = str(seriesname.replace(" and the ", " ", 1))

    if seriesname.startswith("D.I. "):
        seriesname = str(seriesname[5: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("DI "):
        seriesname = str(seriesname[3: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("DS "):
        seriesname = str(seriesname[3: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("DCI "):
        seriesname = str(seriesname[4: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("D.i. "):
        seriesname = str(seriesname[5: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("Di "):
        seriesname = str(seriesname[3: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("Ds "):
        seriesname = str(seriesname[3: ])
        seriesname = str(seriesname.strip())
    if seriesname.startswith("Dci "):
        seriesname = str(seriesname[4: ])
        seriesname = str(seriesname.strip())


  #if more than 2 words separated by spaces, truncate after 2 words to increase probability of web source finding it along with the author name

    seriesname = str(seriesname.replace("  ", " ", 10))
    seriesname = str(seriesname.strip())

    for x in range(1,10):
        n = seriesname.count(" ")
        if n > 1:
            n1 = seriesname.find(" ")
            n2 = seriesname.rfind(" ")
            if n1 <> n2:
                seriesname = str(seriesname[0:n2])
                seriesname = str(seriesname.strip())
            else:
               break
        else:
            pass
    #END FOR


    if seriesname.endswith("s"):    #  Plural e.g. Bridgertons instead of Bridgerton (but also Alex Cross becomes Alex Cros which is okay for this purpose)
        seriesname = str(seriesname[0:-1])
        seriesname = str(seriesname.strip())
    if seriesname.endswith("Volume"):
        seriesname = str(seriesname[0:-6])
        seriesname = str(seriesname.strip())
    if seriesname.endswith("Novel"):
        seriesname = str(seriesname[0:-5])
        seriesname = str(seriesname.strip())
    if seriesname.endswith("'"):
        seriesname = str(seriesname[0:-1])
        seriesname = str(seriesname.strip())
    if seriesname.endswith("Mystery"):
        seriesname = str(seriesname[0:-7])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" and"):
        seriesname = str(seriesname[0:-3])
        seriesname = str(seriesname.strip())
    if seriesname.endswith("Trilogy"):
        seriesname = str(seriesname[0:-7])
        seriesname = str(seriesname.strip())
    if seriesname.endswith("&"):
        seriesname = str(seriesname[0:-1])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" by"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith("Mysterie"):
        seriesname = str(seriesname[0:-8])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" of"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" the"):
        seriesname = str(seriesname[0:-3])
        seriesname = str(seriesname.strip())

    if seriesname.endswith(" 01"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 02"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 03"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 04"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 05"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 06"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 07"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 08"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 09"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())
    if seriesname.endswith(" 10"):
        seriesname = str(seriesname[0:-2])
        seriesname = str(seriesname.strip())

    seriesname = str(seriesname.replace("  ", " ", 10))
    seriesname = str(seriesname.strip())

    n = len(seriesname)
    if seriesname == mynothing or (not seriesname > " ") or (n <= 3):
        seriesname = str(orig_seriesname)    #revert to original because now blank or The etc.


    return str(seriesname)
    #-----------------------------------------------------------------------------------------------------------------
def misc4_unpack_soup_www_fictiondb_com(my_db, my_cursor,orig_series, seriesname, firstname, lastname, log):
    # for www.fictiondb.com only

    my_list1 = []
    results = []
    results1 = []
    results2 = []
    results3 = []
    results_final = []


    firstname = firstname.replace(".",mynothing,4)    #  J.D. Robb should be JD Robb for this purpose.

    try:
        my_letter = str(lastname[0].lower())
    except:
        my_letter = str(lastname[0])

    my_author_link = str(str(firstname) + "-" + str(lastname))       #e.g.  rachel-caine
    try:
        my_author_link = str(my_author_link.lower())
    except:
        pass

    my_series_link = str(seriesname.replace(" ", "-", 20))            #e.g.  morganville-vampire  (seriesname is the generic for orig_series of:  The Morganville Vampires)
    try:
        my_series_link = str(my_series_link.lower())
    except:
        pass

    #<../author/amanda-cabot~series
    my_find_link = str("<../author/[FIRST-LAST]~series")
    my_find_link = str(my_find_link.replace("[FIRST-LAST]", my_author_link, 1))
    #log(str(my_find_link))

    my_url = str("http://www.fictiondb.com/series/author-series~[LETTER].htm")
    my_url = str(my_url.replace("[LETTER]", str(my_letter), 1))
    #log(str(my_url))

    try:
        req = urllib2.Request(my_url)
        response = urllib2.urlopen(req)
        soup = BeautifulSoup(response)
        response.close()
    except Exception as e:
        print(str(e))
        return 0, "None"
    if not soup:
        log("no soup...")
        return 0, "None"
    else:
        print("soup was found")
        pass

    results = soup.tbody
    if not results:
        print("no results; returning")
        return 0, "None"

    n = len(results)
    print("results length:" + str(n))
    if n == 0:
        return 0, "None"
    for row in results:
        s = str(row)
        n = s.find(str(firstname))
        if not n >= 0:
            continue
        else:
            n = s.find(lastname)
            if not n >= 0:
                continue
            else:
                n = s.find(seriesname)
                if not n >= 0:
                    continue
                else:
                    my_list1.append(str(s))
                    break
    #END FOR
    '''       now have (single line):
            <tr class="g"><td><span class="srtkey">Morganvill</span><a class="hghlt" href=".
            ./author/rachel-caine~series~the-morganville-vampires~9655.htm">The Morganville
            Vampires</a></td><td><a href="../author/rachel-caine~39124.htm">Caine, Rachel</a
            ></td></tr>
    '''


    n = len(my_list1)
    print("my_list1 length:" + str(n))
    if n == 0:
        print("no results in my_list1")
        return 0, "None"

    '''
                regex will search: <tr class="g"><td><span class="srtkey">Morganvill</span><a clas
                s="hghlt" href="../author/rachel-caine~series~the-morganville-vampires~9655.htm"
                >The Morganville Vampires</a></td><td><a href="../author/rachel-caine~39124.htm"
                >Caine, Rachel</a></td></tr>
    '''
    my_re_auth_series = str('href=["][.][.][/]author[/][a-z -]*[~]series[~][a-z -]*[SERIES-NAME][a-z -]*[~][0-9]+[.]htm["]')
    my_re_auth_series = str(my_re_auth_series.replace("[SERIES-NAME]", str(my_series_link), 1))
    print(str(my_re_auth_series))
    try:
        p = re.compile(my_re_auth_series, re.IGNORECASE)
    except:
        log("REGEX COMPILE ERROR in Parsing Web Page for: " + str(my_series_link) )
        log("REGEX COMPILE ERROR in misc4_unpack_soup_www_fictiondb_com for: "  + str(my_re_auth_series) )
        log("REGEX COMPILE ERROR: Please Forward This Log to the Developer.  Thank you.")
        return 0, "None"
    s_new_url = mynothing
    for row in my_list1:
        s = str(row)
        print("regex searching:", str(s))
        match1 = p.search(str(s))
        if not match1:
            print("not match1...continuing")
            continue
        else:
            print("yes match1")
            s_new_url = str(match1.group(0))
            print(str(s_new_url))
            break
    #END FOR

    if s_new_url == mynothing:
        #sys.exit("1 Terminated Purposefully")
        print("s_new_url == mynothing")
        return 0, "None"
    # now have:                href="../author/rachel-caine~series~the-morganville-vampires~9655.htm"
    s_new_url = str(s_new_url.replace('href="..', 'http://www.fictiondb.com', 1))
    # now have:                http://www.fictiondb.com/author/rachel-caine~series~the-morganville-vampires~9655.htm"
    s_new_url = str(s_new_url.replace('.htm"', '.htm', 1))
    # now have:                http://www.fictiondb.com/author/rachel-caine~series~the-morganville-vampires~9655.htm
    s_web_series_name = misc4_parse_web_series_name(s_new_url, log)
    log("Retrieved Web Series Name:     " + str(s_web_series_name))
    log(" ")
    try:
        req = urllib2.Request(s_new_url)   #http://www.fictiondb.com/author/rachel-caine~series~the-morganville-vampires~9655.htm
        response = urllib2.urlopen(req)
        soup = BeautifulSoup(response)
        response.close()
    except Exception as e:
        print(str(e))
        return 0, "None"
    if not soup:
        print("no second soup...")
        return 0, "None"
    else:
        pass
        print("second soup was found")

    results2 = soup.tbody
    for row in results2:
        s = str(row)
        n = s.find("td>")     #<td>1</td> [...and...] <td><a itemprop="url" href="../author/rachel-caine~glass-houses~159550~b.htm"><span itemprop='name'>Glass Houses</span></a>
        if n >= 0:
            results3.append(str(s))
    #END FOR
    n = len(results3)
    print("results3 length:" + str(n))
    s_total_books = str(n)
    if n == 0:
        print("results3 has no rows")
        return 0, "None"
    '''   now have as a row:
        <td>14</td> <td><a itemprop="url" href="../author/rachel-caine~fall-of-night~398
        650~b.htm"><span itemprop="name">Fall of Night</span></a><span title=""><a class
        ="anth" href=""></a></span><span class="reissue" title="reissue"></span></td>
        <td class="nowrap"></td><td class=""><span class="visible-xs" itemprop="genre">U
        rban Fantasy</span><span class="hghlt hidden-xs" title="Urban Fantasy">UF</span>
        </td> (plus some junk)
    '''

    book_dict = {}

    my_re_index       = str("<td>[0-9]+</td>\s")
    my_re_booktitle = str("~.+[~][0-9]")

    try:
        p1 = re.compile(my_re_index, re.IGNORECASE)
        p2 = re.compile(my_re_booktitle, re.IGNORECASE)
    except:
        log("REGEX COMPILE ERROR in Parsing Web Page for: " + str(my_re_index) + " and: " + str(my_re_booktitle) )
        log("REGEX COMPILE ERROR in misc4_unpack_soup_www_fictiondb_com for: "  + str(s_new_url) )
        log("REGEX COMPILE ERROR: Please Forward This Log to the Developer.  Thank you.")
        return 0, "None"
    #now clean it up, and put index and booktitle into a dict
    tmp_index = "0"
    tmp_title = ""
    for row in results3:
        s = str(row)
        s = str(__change_tuple_to_bytestring(str(row)))
        print("Raw Row:", str(s))
        match1 = p1.search(str(s))
        match2 = p2.search(str(s))
        if match1:        #   <td>1</td>
            s = str(match1.group())
            s = str(s.replace("<td>", mynothing, 2))
            s = str(s.replace("</td>", mynothing, 1))
            tmp_index = str(s.strip())
            print("[match1]:", str(tmp_index), str(tmp_title))
        if match2 :       #    ~fall-of-night~3
            s = str(match2.group())
            s = str(s[1:-2])   #   fall-of-night
            s = str(s.replace("-", " ", 1))
            tmp_title = str(s.strip())
            tmp_title = str(tmp_title.replace("-", " ", 30))
            tmp_title = str(titlecase(tmp_title))
            book_dict[tmp_title] = str(tmp_index)
            print("[match2]:",str(tmp_index), str(tmp_title))
            tmp_index = "0"
            tmp_title = "NOTHINGTITLE"
        continue
    #END FOR


    n = len(book_dict)  #web books in series
    if n >= 60  or n == 0:
        log("==========================================================")
        log("ERROR[2].  Web URL May Have Changed (Or Not). Notify Developer If This Continues.")
        log("==========================================================")
        return 0, "None"

    seriesname = str(orig_series)

    seriesname = str(misc4_genericize_seriesname(str(seriesname)))

    mysql = 'SELECT Count(*) FROM __books_work_populate WHERE seriesname LIKE "%' + seriesname + '%" ; '
    my_cursor.execute(mysql)
    tmp_rows = my_cursor.fetchall()
    for row in tmp_rows:
        s = str(row)
        s = str(__change_tuple_to_bytestring(s))
        s = str(s.replace("(", mynothing, 1))
        s = str(s.replace(")", mynothing, 1))
        s = str(s.replace(",", mynothing, 1))
        n_real_work_book_count = str(s)
        break
    #END FOR
    log("Number of Related Books In the Q&S Library:    " + str(n_real_work_book_count))
    log(" ")

    mysql = str('SELECT authname FROM __books_work_populate WHERE seriesname = "' + orig_series + '" ; ')
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        tmp_auth = "unknown"
    else:
        for row in tmp_rows:
            try:
                tmp_auth = unicodedata.normalize('NFKD', tmp_auth).encode('ascii', 'ignore')
            except:
                pass
            tmp_auth = str(row)
            tmp_auth =  str(__change_tuple_to_bytestring(tmp_auth))
            break
    log(str("[Work Author]: " + str(tmp_auth)))
    my_url = str(my_url)   #used when inserting new rows into web tables
    orig_series = str(__change_tuple_to_bytestring(orig_series))

    if not s_total_books[0].isdigit():
        s_total_books = "0"
    for title,index in book_dict.iteritems():
        tmp_title = str(title)
        n = tmp_title.find("~")
        if n > 0:
            tmp_title = str(tmp_title[0:n])
            tmp_title = str(tmp_title.replace("~", mynothing, 1))
        tmp_title = str(tmp_title.replace("it's", "it is", 2))  #because booklevel scrubbing does this to avoid loss of the apostrophe due to utf8 to bytestring conversion
        tmp_title = str(titlecase(str(tmp_title)))  #titlecase to aid in comparison
        tmp_index = str(index)
        tmp_index = str(tmp_index.replace("&nbsp;", mynothing, 2))
        tmp_index = str(tmp_index.strip())
        if tmp_title > " ":
            if (("Search by This Series Name" or "&nbsp") in tmp_title) or  (not tmp_index[0].isdigit()):
                log("passing to avoid update for:" + str(tmp_index) + str(tmp_title))
                pass
            else:
                sleep(0.05)
                mysql = str('INSERT OR REPLACE INTO _web_author_series (authname,seriesname,source_url,book_count) VALUES (?,?,?,?) ')
                __execute_mysql_for_custom_column_generic_4_args(my_db, my_cursor, log, mysql, tmp_auth, s_web_series_name, my_url, s_total_books)
                sleep(0.5)
                mysql = "INSERT OR  REPLACE INTO _web_series_detail (authname, seriesname, booktitle, seriesindex, source_url) VALUES (?,?,?,?,?) "
                __execute_mysql_for_custom_column_generic_5_args(my_db, my_cursor, log, mysql, tmp_auth, s_web_series_name, tmp_title, tmp_index, my_url)
                sleep(0.5)
                mysql = str('INSERT OR REPLACE INTO _web_series_rename_detail (work_series,work_author,web_series,web_author) VALUES (?,?,?,?) ')
                __execute_mysql_for_custom_column_generic_4_args(my_db, my_cursor, log, mysql, orig_series, tmp_auth, s_web_series_name, tmp_auth)
                sleep(0.5)
    #END FOR

    mysql = str("INSERT OR REPLACE INTO _web_series_rename_detail_cumulative SELECT * FROM _web_series_rename_detail ")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    mysql = str("DELETE FROM _web_series_rename_detail WHERE rowid NOT NULL")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    return n_real_work_book_count, str(tmp_auth)
    #-----------------------------------------------------------------------------------------------------------------
def misc4_parse_web_series_name(s_new_url, log):
    # now have:   http://www.fictiondb.com/author/rachel-caine~series~the-morganville-vampires~9655.htm

    #log("0 parsing:" + s_new_url)

    s_new_url = str(s_new_url)

    n = s_new_url.find("~series~")           #   ~series~scimitar-seas~
    n = n + 8
    s_web_series_name = str(s_new_url[n: ])
    #log("1:" + str(s_web_series_name))
    n = s_web_series_name.find("~")
    s_web_series_name = str(s_web_series_name[0:n])
    #log("2:" + str(s_web_series_name))
    s_web_series_name = str(s_web_series_name.replace("-", " ", 30))
    #log("3:" + str(s_web_series_name))
    s_web_series_name = str(s_web_series_name.strip())
    s_web_series_name = str(titlecase(s_web_series_name))

    #log("returning with:" + str(s_web_series_name))

    return str(s_web_series_name)
    #-----------------------------------------------------------------------------------------------------------------
def misc4_unpack_soup_ww2_kdl_org(my_db, my_cursor, my_url, text_list, orig_series, log ):
    # SPECIFIC to:   ww2.kdl.org
    #parse the text returned by BeautifulSoup

    global mynothing
    book_dict = {}
    s_line = mynothing
    n_total_books = 0
    s_total_books = "0"
    match_was_found = False


    my_re_nothing = "search produced no results for the parameters"
    q = re.compile(my_re_nothing, re.IGNORECASE)
    my_re_total = "[0-9]+\sbook[(s)]+\swere found with your criteria"
    p = re.compile(my_re_total, re.IGNORECASE)
    for row in text_list:
        line = str(row)
        match0 = q.search(str(line))
        if match0:
            return 0, "None"
        match1 = p.search(str(line))
        if not match1:
            continue
        else:
            s_total_books = str(match1.group(0))
            s_total_books = str(s_total_books.replace(" book(s) were found with your criteria", mynothing, 1))
            s_total_books = str(__change_tuple_to_bytestring(str(s_total_books)))
            s_total_books = str(s_total_books.replace(",", mynothing, 1))
            s_total_books = str(s_total_books.strip())
            if s_total_books == "0":
                return 0, "None"
            log("Total Books in This Series Per Web Source:     " + str(s_total_books))
            break
    #END FOR


    my_list = []

    add_rest = False
    for row in text_list:
        line = str(row)
        n = line.count("Search for a Book")
        if n > 0:
            break
        if add_rest:
            if str(line) > " ":
                my_list.append(str(line))
            continue
        n = line.count("search by this series name")
        if n > 0:
            add_rest = True
    #END FOR

    n_counter = 0
    for row in my_list:    #from soup
        s = str(row)
        if not s > " ":
            continue
        n_counter = n_counter + 1
        if n_counter%2 == 0 :
            s_current_title = str(s)
            book_dict[s_current_title] = str(s_current_index)
            #log(str(s_current_title) + "   " + str(s_current_index))
        else:
            s_current_index = str(s)
            s_current_index = str(s_current_index.replace(".", mynothing, 1))
    #END FOR

    n = len(book_dict)  #web books in series
    if n >= 99:
        log("==========================================================")
        log("ERROR[2].  Web URL May Have Changed (Or Not). Notify Developer If This Continues.")
        log("==========================================================")
        return 0, "None"

    seriesname = str(orig_series)

    seriesname = str(misc4_genericize_seriesname(str(seriesname)))

    mysql = 'SELECT Count(*) FROM __books_work_populate WHERE seriesname LIKE "%' + seriesname + '%" ; '
    my_cursor.execute(mysql)
    tmp_rows = my_cursor.fetchall()
    for row in tmp_rows:
        s = str(row)
        s = str(__change_tuple_to_bytestring(s))
        s = str(s.replace("(", mynothing, 1))
        s = str(s.replace(")", mynothing, 1))
        s = str(s.replace(",", mynothing, 1))
        n_real_work_book_count = str(s)
        break
    #END FOR
    log("Number of Related Books In the Q&S Library:    " + str(n_real_work_book_count))

    mysql = str('SELECT authname FROM __books_work_populate WHERE seriesname = "' + orig_series + '" ; ')
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        tmp_auth = "unknown"
    else:
        for row in tmp_rows:
            try:
                tmp_auth = unicodedata.normalize('NFKD', tmp_auth).encode('ascii', 'ignore')
            except:
                pass
            tmp_auth = str(row)
            tmp_auth =  str(__change_tuple_to_bytestring(tmp_auth))
            break
    log(str("[Work Author]: " + str(tmp_auth)))
    my_url = str(my_url)
    orig_series = str(__change_tuple_to_bytestring(orig_series))
    if not s_total_books.isdigit():
        s_total_books = "0"
    for title,index in book_dict.iteritems():
        tmp_title = str(title)
        tmp_title = str(tmp_title.replace("it's", "it is", 2))  #because booklevel scrubbing does this to avoid loss of the apostrophe due to utf8 to bytestring conversion
        tmp_title = str(titlecase(str(tmp_title)))  #titlecase to aid in comparison
        tmp_index = str(index)
        tmp_index = str(tmp_index.replace("&nbsp;", mynothing, 2))
        tmp_index = str(tmp_index.strip())

        if tmp_title > " ":
            if (("Search by This Series Name" or "&nbsp") in tmp_title) or  (not tmp_index[0].isdigit()):
                pass
            else:
                sleep(0.05)
                mysql = str('INSERT OR REPLACE INTO _web_author_series (authname,seriesname,source_url,book_count) VALUES (?,?,?,?) ')
                __execute_mysql_for_custom_column_generic_4_args(my_db, my_cursor, log, mysql, tmp_auth, orig_series, my_url, s_total_books)
                sleep(0.05)
                mysql = str('INSERT OR REPLACE INTO _web_series_detail (authname, seriesname,booktitle,seriesindex,source_url) VALUES (?,?,?,?,?) ')
                __execute_mysql_for_custom_column_generic_5_args(my_db, my_cursor, log, mysql, tmp_auth, orig_series, tmp_title, tmp_index, my_url)
                sleep(0.05)
    #END FOR

    return n_real_work_book_count, str(tmp_auth)
    #-----------------------------------------------------------------------------------------------------------------
def misc4_validate_work_series_index(my_db, my_cursor, log, tmp_auth, orig_series, my_url, n_real_work_book_count, notifications):

    global my_run_type

    seriesname = str(orig_series)

    seriesname = str(misc4_genericize_seriesname(str(seriesname)))


    mysql = str('SELECT authname, booktitle, seriesname, seriesindex FROM __books_work_populate WHERE seriesname LIKE  "%' + seriesname + '%" ;')
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        log("Warning: Original Work Series Not Found (Which Is Impossible Unless Odd Abbreviations and/or Punctuation Exist.):" + str(orig_series))
        return
    num_work_series_books = len(tmp_rows)

    all_books_okay = True
    tmp_rows.sort()
    for row in tmp_rows:    #for each book with the same current original work series...
        work_auth,work_title,work_series,work_index = row
        work_auth = str(__change_tuple_to_bytestring(str(work_auth)))
        work_title = str(__change_tuple_to_bytestring(str(work_title)))
        work_title = str(work_title.replace("'s", mynothing, 10))       # e.g.  John's
        work_title = str(work_title.replace("' ", " ", 10))       # e.g.  The Girls' Horse versus  The Girls Horse, which is common on web search results
        work_title_title = str(titlecase(str(work_title)))  #titlecase to aid in comparison
        work_series = str(__change_tuple_to_bytestring(str(work_series)))
        work_index = str(__change_tuple_to_bytestring(str(work_index)))
        work_index = str(work_index.replace(".0", mynothing, 1))     #so 2.0 becomes 2
        #now we have single book with its author, series and index...
        #so now we want to validate that data against table _web_series_detail
        mysql = str('SELECT exists (SELECT booktitle FROM _web_series_detail WHERE booktitle LIKE  "%The ' + work_title + '%" \
                                                                                                                                OR booktitle = "' + work_title + '" \
                                                                                                                                OR booktitle LIKE  "%A ' + work_title + '%" \
                                                                                                                                OR booktitle LIKE  "%An ' + work_title + '%" ); ' )
        results_list = my_cursor.execute(mysql)
        n_title_exists = 0
        n_index_exists = 0
        for row in results_list:
            for col in row:
                n_title_exists = str(__change_tuple_to_bytestring(str(col)))
            #END FOR
        #END FOR

        if n_title_exists == "1":
            s_msg = str("Work Title Same As Web Title.")
            title_same = True
        else:
            s_msg = str("<font color='#FF0000'>Work Title Was Not Found As A Web Title For This Web Author</font>")
            title_same = False
            index_same = False
            all_books_okay = False

        if title_same:  # now validate the index since the title is valid
            mysql = str('SELECT exists (SELECT seriesindex FROM _web_series_detail WHERE ( booktitle LIKE  "%The ' + work_title + '%" \
                                                                                                                                        OR booktitle = "' + work_title + '" \
                                                                                                                                        OR booktitle LIKE  "%A ' + work_title + '%" \
                                                                                                                                        OR booktitle LIKE  "%An ' + work_title + '%" )\
                                                                                                                                        AND (seriesindex = "' + work_index + '") ); ' )
            results_list = my_cursor.execute(mysql)
            for row in results_list:
                for col in row:
                    n_index_exists = str(__change_tuple_to_bytestring(str(col)))
                #END FOR
            #END FOR
            if n_index_exists == "1":
                web_index = str(work_index + "     ")  #since the same, don't need to retrieve it; use work_index instead
                web_index = str(web_index[0:4])
                s_msg = str("<font color='#21610B'>Web Series Index:" + str(web_index) + "  -- Identical</font>")
                index_same = True
            else:
                s_msg = str("<font color='#FF0000'>Work Series Index Not Same as Web Series Index</font>")
                index_same = False
                all_books_okay = False
                mysql = str('SELECT seriesindex FROM _web_series_detail WHERE ( booktitle LIKE  "%The ' + work_title + '%" \
                                                                                                                                        OR booktitle = "' + work_title + '" \
                                                                                                                                        OR booktitle LIKE  "%A ' + work_title + '%" \
                                                                                                                                        OR booktitle LIKE  "%An ' + work_title + '%" ) ;')
                results_list = my_cursor.execute(mysql)
                if not results_list:
                    pass
                else:
                    for row in results_list:
                        web_index = str(__change_tuple_to_bytestring(str(row)))
                        web_index = str(web_index + "     ")
                        web_index = str(web_index[0:4])
                        s_msg = str("<font color='#FF0000'>Web Series Index: " + str(web_index)  + "  -- DIFFERENT</font>")
                        break
        else:
            pass

        s_work_title = str(work_title) + "                                             "
        s_work_title = str(s_work_title[0:30])
        s_work_index = str(work_index) + "             "
        s_work_index = str(s_work_index[0:3])

        log(str("[Work Series]: " + str(orig_series) + " [Work Title]: " + str(s_work_title) + " [Work Index]: " + str(s_work_index) + ": " + s_msg ))

    #END FOR

    if all_books_okay :
        log("<font size='5'><b><font color='#21610B'>All Valid</b></font>")
    else:
        log("<font size='5'><b><font color='#FF0000'>Discrepencies Noted</b></font>")
    #-----------------------------------------------------------------------------------------------------------------
def misc4_fetch_series(my_db, my_cursor, log, tmp_books_list):
    # fetch all non-null series from view  __books_work_populate based on selected book list pass from ui.py

    tmp_series_list = []
    tmp_rows_save = []
    tmp_rows = []
    for row in tmp_books_list:
        del tmp_rows
        my_book = str(row)
        mysql = str('SELECT authname,seriesname FROM __books_work_populate  WHERE seriesname NOT NULL AND book = [BOOK]  ;')
        mysql = str(mysql.replace("[BOOK]", str(my_book), 1))
        sleep(0.01)
        tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
        if not tmp_rows:
            tmp_rows = []
        else:
            for item in tmp_rows:
                tmp_rows_save.append(item)
            #END FOR
    #END FOR

    if not tmp_rows_save:
        return tmp_series_list

    tmp_rows_save_set = set(tmp_rows_save)
    tmp_rows_save = list(tmp_rows_save_set)   #duplicates removed
    n = len(tmp_rows_save)
    log("Number of Unique Series Selected: " + str(n))
    if n == 0:
        return tmp_series_list

    for row in tmp_rows_save:
        authname, seriesname = row
        try:
            authname = unicodedata.normalize('NFKD', authname).encode('ascii', 'ignore')
        except:
            pass
        try:
            seriesname = unicodedata.normalize('NFKD', seriesname).encode('ascii', 'ignore')
        except:
            pass
        authname = str(authname)
        authname = str(__change_tuple_to_bytestring(str(authname)))
        auth_list = authname.split(" ")
        n = len(auth_list)
        if n == 0:
            pass
        else:
            if n == 1:
                pass
            else:
                if n == 2:
                    pass
                else:
                    if n == 3:
                        #log("original authname:" + str(authname))
                        authname = str(str(auth_list[0]) + " " + str(auth_list[2]))
                        #log("original authname:" + str(authname))
                    else:
                        pass
        seriesname = str(seriesname)
        seriesname = str(__change_tuple_to_bytestring(str(seriesname)))
        s = str(str(authname) + str("||@||") + str(seriesname))
        tmp_series_list.append(str(s))
    #END FOR

    tmp_series_list.sort()

    return tmp_series_list
    #-----------------------------------------------------------------------------------------------------------------
def misc4_purge_old_web_table_data(my_db, my_cursor, log):
    #also used by misc14

    sleep(0.5)
    mysql = 'DELETE FROM _web_author_series WHERE  rowid NOT NULL'
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.5)
    mysql = 'DELETE FROM _web_series_detail WHERE  rowid NOT NULL'
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.5)

    mysql = str('DELETE FROM _global_web_author_series WHERE date_added  <  "2014-11-06" ')   #logic to update table changed on 2014-11-05, so delete legacy records
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.5)
    mysql = str('DELETE FROM _global_web_series_detail WHERE date_added  <  "2014-11-06" ')   #logic to update table changed on 2014-11-05, so delete legacy records
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.5)
    #-----------------------------------------------------------------------------------------------------------------
def misc4_delete_web_table_duplicates(my_db, my_cursor, log):
    #also used by misc14

    mysql = "DELETE FROM _web_series_detail WHERE  rowid NOT IN (SELECT min(rowid) FROM   _web_series_detail GROUP BY seriesname,booktitle) "
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    mysql = "DELETE FROM _web_author_series WHERE  rowid NOT IN (SELECT min(rowid) FROM   _web_author_series GROUP BY authname,seriesname) "
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    #-----------------------------------------------------------------------------------------------------------------
def misc5_control(my_db, my_cursor, notifications, log):
    #uses _web_series_rename_detail as the source of data
    #note that custom_column_10.id is always = book in Q&S due to various "explode cc10 if needed" functions in multiple jobs.
    #ditto for cc8

    global mynothing

    there_are_renamable = True

    sleep(0.5)

    __explode_custom_column_8_if_needed(my_db, my_cursor, log)     #imported from booklevel.py

    sleep(0.5)

    __explode_custom_column_10_if_needed(my_db, my_cursor, log)     #imported from booklevel.py

    sleep(0.02)

    misc5_mine_global_historical_data_for_use_in_renaming(my_db, my_cursor, notifications, log)

    sleep(0.25)

    #ensure everything in this table is valid and desirable
    mysql = "DELETE FROM _web_series_rename_detail_cumulative WHERE  work_series = web_series \
                        OR web_series IS NULL OR work_series IS NULL OR work_series = 'Novel'  OR  work_series = 'novel' \
                        OR work_series = ' '  OR  work_series = ''      ;  "
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.02)

    #now populate _web_series_rename_detail with *only rows that are germane* to the current work series in custom_column_10 (speeds overall execution time using views)
    mysql = "INSERT OR IGNORE INTO _web_series_rename_detail SELECT * FROM _web_series_rename_detail_cumulative \
                                                    WHERE work_series IN(SELECT value FROM custom_column_10 WHERE value NOT NULL)  "
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.02)

    mysql = "SELECT book, work_series FROM __web_series_rename_detail_view_part1"
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
            log("Not able to rename any Work Series to Web Series.  Perhaps renaming was previously done, and/or nothing now needs renaming.  Refer to the notes at the top of this log. ")
            log(" ")
            renamable = False   #return early

    print("misc5_control - Renamable: ", there_are_renamable)

    notifications.put((0.01, 'Work Title to Web Title Renaming'))
    log("Beginning Work Title to Web Title Renaming.")
    log(" ")
    #imported from booklevel so common function
    change_work_title_to_web_title_for_previously_validated_work_series(my_db, my_cursor, notifications, log)

    if not there_are_renamable:     #return early
        tmp_rows = []
        del tmp_rows
        notifications.put((0.50, 'Adding Missing Work Series If Found in Web Data'))
        mysql = "SELECT book FROM __books_work_populate WHERE __books_work_populate.authname NOT NULL \
                    AND __books_work_populate.booktitle NOT NULL AND __books_work_populate.seriesname IS NULL "
        tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
        if tmp_rows:
            n = len(tmp_rows)
            if n > 0:
                log("Missing Work Series will be added if any Web Series for the Author/Title is found in the historical Web Data for: " + str(n) + " potential books.")
                log(" ")
                for row in tmp_rows:
                    book = str(row)
                    book = str(__strip_numerics(book))
                    book = str(book.strip())
                    #imported from booklevel so common function
                    add_missing_seriesname_from_web_detail(my_db, my_cursor, book,notifications, log)
                    __refresh_custom_column_15(my_db, my_cursor, book, notifications, log)
                #END FOR
                pass
            else:
                pass
        else:
            pass

    if not there_are_renamable:
        log(" ")
        log("Changing Work Series Index to Web Series Index if appropriate.")
        miscellany_change_work_index_to_web_index_standalone(my_db, my_cursor, notifications, log)
        print("misc5_control - Not there_are_renamable, so returning early.")
        return

    print("misc5_control - there_are_renamable, so continuing.")

    sleep(0.02)

    notifications.put((0.50, 'Renaming Work Series to Web Series'))

    n_total = len(tmp_rows)
    n_counter = 0
    for row in tmp_rows:
        id,work_series = row
        id = str(id)    #id == book
        id = str(__change_tuple_to_bytestring(str(id)))
        id = str(id.strip())
        work_series = str(work_series)    #work_series == work series name
        work_series = str(__change_tuple_to_bytestring(str(work_series)))
        work_series = str(work_series.strip())
        work_series = str(work_series + "                                   ")
        work_series = str(work_series[0:35])
        sleep(0.02)
        mysql = str("UPDATE custom_column_10 SET value = ( SELECT web_series FROM __web_series_rename_detail_view_part2 \
                      WHERE __web_series_rename_detail_view_part2.work_series =  custom_column_10.value \
                      AND __web_series_rename_detail_view_part2.web_series <> custom_column_10.value \
                      AND __web_series_rename_detail_view_part2.web_series NOT NULL ) \
                      WHERE (custom_column_10.value NOT NULL AND custom_column_10.id = ? AND custom_column_10.id = ? ) \
                      AND (EXISTS (SELECT * FROM __web_series_rename_detail_view_part2 \
                                            WHERE __web_series_rename_detail_view_part2 .work_series = custom_column_10.value \
                                                AND __web_series_rename_detail_view_part2.web_series <> custom_column_10.value \
                                                AND __web_series_rename_detail_view_part2.web_series NOT NULL )) ")
        __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, id, id)
        log(" ")
        log("[Invalid Work Series Name]: " + str(work_series) + "   --- Changed to the correct Web Source Web Series Name per previously downloaded WSSVD.")
        sleep(0.02)
        __refresh_custom_column_15(my_db, my_cursor, id, None, log)
        n_counter = n_counter + 1
        n_progress = float((.50*(n_counter/n_total)) + 0.50)
        notifications.put((n_progress, 'Renaming Work Series to Web Series'))
    #END FOR
    notifications.put((0.99, 'Renaming Other Work Values to Web Values'))
    log(" ")
    log("-------------------------------------------------------------------------------------------------------------------------------------------")
    log(" ")
    log("Total Invalid Work Series Names Changed to Valid Web Series Names Previously Logged: " + str(n_counter))
    log(" ")
    log("-------------------------------------------------------------------------------------------------------------------------------------------")
    log(" ")

    tmp_rows = []
    del tmp_rows

    sleep(0.02)

    mysql = "SELECT book FROM __web_global_seriesindex_not_match_work_seriesindex WHERE web_seriesindex NOT NULL and web_seriesindex <> '0'  "
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if tmp_rows:
        n = len(tmp_rows)
        if n > 0:
            for row in tmp_rows:
                book = str(row)
                book = __strip_numerics(book)
                book = str(book.strip())
                mysql = str("UPDATE custom_column_12 SET value = (SELECT web_seriesindex FROM __web_global_seriesindex_not_match_work_seriesindex \
                                    WHERE __web_global_seriesindex_not_match_work_seriesindex.book = custom_column_12.book \
                                    AND __web_global_seriesindex_not_match_work_seriesindex.work_seriesindex = custom_column_12.value ) \
                                    WHERE custom_column_12.book =?  AND custom_column_12.book = ?  ;")
                __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, book, book)
                __refresh_custom_column_15(my_db, my_cursor, book, None, log)
            #END FOR
            log("Total Incorrect Book Work Series Indexes Changed to Correct Book Web Series Indexes for Identical Series/Titles:  " + str(n))
            log(" ")
            log("-------------------------------------------------------------------------------------------------------------------------------------------")
            log(" ")

    tmp_rows = []
    del tmp_rows

    #imported from booklevel so common function
    change_work_title_to_web_title_for_previously_validated_work_series(my_db, my_cursor, notifications, log)

    sleep(0.02)
    mysql = str("DELETE FROM _web_series_rename_detail WHERE work_series NOT NULL")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    log(" ")
    log(" ")
    log(" ")

    sleep(0.02)

    tmp_rows = []
    del tmp_rows

    #identical to the "if not renamable" above...
    notifications.put((0.99, 'Adding Missing Work Series If Found in Web Data'))
    mysql = "SELECT book FROM __books_work_populate WHERE __books_work_populate.authname NOT NULL \
                AND __books_work_populate.booktitle NOT NULL AND __books_work_populate.seriesname IS NULL "
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if tmp_rows:
        n = len(tmp_rows)
        if n > 0:
            log("Missing Work Series will be added if any Web Series for the Author/Title is found in the historical Web Data.")
            log(" ")
            for row in tmp_rows:
                book = str(row)
                book = str(__strip_numerics(book))
                book = str(book.strip())
                #imported from booklevel so common function
                add_missing_seriesname_from_web_detail(my_db, my_cursor, book,notifications, log)
                __refresh_custom_column_15(my_db, my_cursor, book, notifications, log)

    sleep(0.02)

    my_db.close()

    return
    #-----------------------------------------------------------------------------------------------------------------
def misc5_mine_global_historical_data_for_use_in_renaming(my_db, my_cursor, notifications, log):
    #This is where the value in accumulating the global web source data is obtained.
    #Populate _web_series_rename_detail_cumulative FROM _global_web_author_series AND _global_web_series_detail AND whatever else
    #    so the RENAME job can be run with only historical data without first downloading new data via the VALIDATE job.
    #step 1: using current work series, create a GENERIC version just like that used in the downloading of web source data.
    #step 2: using the GENERIC work series, search the historical global web source data in the same way that the download data from the url was searched.
    #step 3: insert new records into _web_series_rename_detail_cumulative.
    #step 4: populate _web_author_series from _global_web_author_series
    #step 5: populate _web_series_detail from _global_web_series_detail

    mynothing = ""
    n_count = 0

    tmp_books_list = misc5_fetch_all_books_with_series(my_db, my_cursor, log)

    tmp_series_list = misc4_fetch_series(my_db, my_cursor, log, tmp_books_list)

    for row in tmp_series_list:         # row = str(str(authname) + str("||@||") + str(seriesname))
        s = str(row)
        s_list = s.split("||@||")
        work_auth = str(s_list[0])
        work_series = str(s_list[1])
        work_series = str(work_series.replace("'", mynothing, 10))
        work_series = str(work_series.replace('"', mynothing, 10))
        work_auth = str(work_auth.replace("'", mynothing, 10))
        work_auth = str(work_auth.replace('"', mynothing, 10))
        generic_series = str(misc4_genericize_seriesname(work_series))
        #search the historical global web source data like misc4 did, using the generic seriesname and/or full seriesname to match
        #must match book author to global web source author since the same series name can be used by different authors!
        mysql = "SELECT authname,seriesname FROM _global_web_author_series \
                                WHERE (authname = '[work_auth]' ) \
                                    AND ( (seriesname = '[work_series]' ) OR \
                                             (seriesname like '%[work_series]%' ) OR \
                                             (seriesname like '%[generic_series]%' ) OR \
                                             (seriesname = '[generic_series]' ) \
                                            ) "

        mysql = str(mysql.replace("[work_auth]", str(work_auth), 3))
        mysql = str(mysql.replace("[work_series]", str(work_series), 3))
        mysql = str(mysql.replace("[generic_series]", str(generic_series), 3))
        tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
        if not tmp_rows:
            continue
        else:
            for row in tmp_rows:
                web_auth, web_series_name = row
                web_auth = str(__change_tuple_to_bytestring(web_auth))
                web_series_name = str(__change_tuple_to_bytestring(web_series_name))
                mysql = str('INSERT OR IGNORE INTO _web_series_rename_detail_cumulative (work_series,work_author,web_series,web_author) VALUES (?,?,?,?) ')
                __execute_mysql_for_custom_column_generic_4_args(my_db, my_cursor, log, mysql, work_series, work_auth, web_series_name, web_auth)
                sleep(0.02)
                n_count = n_count + 1
            #END FOR
            del tmp_rows
    #END FOR

    log("Number of Historically Cumulative Web Source Records Matching Current List of Work Series Names: " + str(n_count) )

    mysql = str('INSERT OR REPLACE INTO _web_author_series SELECT * FROM _global_web_author_series')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    mysql = str('INSERT OR REPLACE INTO _web_series_detail SELECT * FROM _global_web_series_detail')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    log(" ")
    log("All Historically Cumulative Web Source Records from All Q&S Libraries are Available for Use in Renaming in This Q&S Library.")
    log(" ")
    #-----------------------------------------------------------------------------------------------------------------
def misc5_fetch_all_books_with_series(my_db, my_cursor, log):

    tmp_books_list = []

    mysql = str('SELECT id FROM custom_column_10 ;')
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        pass
    else:
        for row in tmp_rows:
            book = str(row)
            book = str(__strip_numerics(book))
            tmp_books_list.append(book)
        #END FOR

    return  tmp_books_list
    #-----------------------------------------------------------------------------------------------------------------
def misc9_control(my_db, my_cursor, notifications, log):
    # Minimize Work Tags Using Tag Priorities for Selected Books

    __build_regex_list_from_tag_capitalization_rules(my_db, my_cursor, log)
    sleep(0.02)
    __explode_custom_column_13_if_needed(my_db, my_cursor, log)        #just in case...otherwise sql will find nothing to minimize if id <> book in cc13.
    sleep(0.5)

    #get all awards related tags, taking into account renaming of tags via the tag rules table...
    awards_list = []

    mysql = "SELECT newtag FROM _tag_rules WHERE oldtag IN(SELECT award FROM _book_awards) AND purgetag = '0'  AND newtag NOT NULL "
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        pass
    else:
        for row in tmp_rows:
            for col in row:
                award = str(col)
                award = str(__change_tuple_to_bytestring(str(award)))
                award = str(award.title())
                awards_list.append(award)
    tmp_rows = []
    del tmp_rows
    mysql = "SELECT award FROM _book_awards GROUP BY award"
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        pass
    else:
        for row in tmp_rows:
            for col in row:
                award = str(col)
                award = str(__change_tuple_to_bytestring(str(award)))
                award = str(award.title())
                awards_list.append(str(award))

    tmp_set = set(awards_list)
    awards_list = list(tmp_set)  #eliminate duplicates
    awards_list.sort(reverse=True)     # avoid:  Prometheus Award, The Prometheus Award later causing a residual "The " because the shorter version was replaced with nothing before the longer version

    global my_selected_book_list

    tmp_books_list = []
    for row in my_selected_book_list:
        books = str(row)
        books = str(__strip_numerics(books))
        tmp_books_list.append(books)
    #END FOR

    del my_selected_book_list
    n_total = len(tmp_books_list)
    if n_total == 0:
        log("No books selected for Minimizing Work Tags Using Tag Priorities")
        return
    else:
        log("Books selected for Minimizing Work Tags Using Tag Priorities: " + str(n_total))

    log(" ")
    from calibre_plugins.quarantine_and_scrub.config import prefs
    maximum_tags = prefs['maximum_tags']
    if not maximum_tags:
        maximum_tags = 10
    log("Configured 'maximum tags' to be used is:  " + str(maximum_tags))
    ignore_award_tags_prefs = prefs['ignore_award_tags']
    log(" ")
    if ignore_award_tags_prefs == "True" or ignore_award_tags_prefs[0] == 'T' :
        ignore_award_tags = 'True'
    else:
        ignore_award_tags = 'False'
        del awards_list
        awards_list = []   #empty it so no matches can possibly occur
    log("Configured 'Ignore Award Tags' is:  " + str(ignore_award_tags))
    if ignore_award_tags:
        n = len(awards_list)
        log("The number of Awards as Work Tags to be ignored: " + str(n))
        for row in awards_list:
            log("Award as Work Tag to be Ignored: " + str(row))
        log(" ")


    try:
        maximum_tags = int(float(maximum_tags))
        if maximum_tags <= 0 or maximum_tags > 100:
            print("ERROR.  Configured maximum_tags is invalid.  Terminating.")
            log("ERROR.  Configured maximum_tags is invalid.  Terminating.")
            return
    except:
        print("ERROR.  Configured maximum_tags is invalid.  Terminating.")
        log("ERROR.  Configured maximum_tags is invalid.  Terminating.")
        return


    maximum_commas = maximum_tags - 1     #e.g. 6 tags have 5 commas:  Crime, Fiction:mystery&detective, Scandinavia Finland, Suspense&thriller, Sweden, Travel
    award_maximum_commas = maximum_tags  #the awards related tags will never be deleted

    book_dict = {}
    tmp_rows = []
    for row in tmp_books_list:
        del tmp_rows
        my_current_book = str(row)
        mysql = str("SELECT id, value FROM custom_column_13 WHERE (value NOT NULL) AND ( instr(value,',') > 0 ) AND  (instr(value,',') NOT NULL) \
                                                                                                        AND id = [BOOK] ")
        mysql = str(mysql.replace("[BOOK]", str(my_current_book), 1))
        tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
        if not tmp_rows:
            continue
        else:
            for row in tmp_rows:
                id, value = row
                try:
                    tagsconcat = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
                except:
                    pass
                tagsconcat = str(tagsconcat)
                tagsconcat = str(__change_tuple_to_bytestring(str(tagsconcat)))
                found_award = False
                for award in awards_list:
                    if tagsconcat.count(str(award)) > 0:
                        found_award = True
                        break
                    else:
                        continue
                #END FOR
                #log("found_award: " + str(found_award))
                n1 = tagsconcat.count(",")
                if not found_award:
                    if n1 < maximum_commas:
                        continue
                else:
                    if n1 < award_maximum_commas:
                        continue
                    else:
                        pass
                book = str(id)
                book = str(__change_tuple_to_bytestring(str(book)))
                book_dict[book] = str(tagsconcat)
            #END FOR
    #END FOR

    n = len(book_dict)
    if n == 0:
        log("[2] Nothing found to minimize.  Finished.")
        return
    else:
        #log("length of book_dict: " + str(n))
        priority_dict = misc9_fetch_tag_priorities(my_db, my_cursor, notifications, log)
        new_book_dict = misc9_process_tags(my_db, my_cursor, notifications, log, book_dict, priority_dict, maximum_tags, awards_list)
        misc9_update_custom_column_13(my_db, my_cursor, notifications, log, new_book_dict)
    #-----------------------------------------------------------------------------------------------------------------
def misc9_fetch_tag_priorities(my_db, my_cursor, notifications, log):

    priority_dict = {}

    mysql = str("SELECT tag, priority FROM _tag_priorities ")
    tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not tmp_rows:
        log("[3] No Tag Priorities Found in the Tag Priorities Table.  Finished.")
        return
    else:
        for row in tmp_rows:
            tag, priority = row
            try:
                tag = unicodedata.normalize('NFKD', tag).encode('ascii', 'ignore')
            except:
                pass
            tag = str(tag)
            tag = str(__change_tuple_to_bytestring(str(tag)))
            priority = str(priority)
            priority = str(__strip_numerics(priority))
            priority = int(float(priority))
            priority_dict[tag] = priority
        #END FOR

    return priority_dict
    #-----------------------------------------------------------------------------------------------------------------
def misc9_process_tags(my_db, my_cursor, notifications, log, book_dict, priority_dict, maximum_tags, awards_list):

    have_awards = False
    n0 = len(awards_list)
    if n0 > 0:
        have_awards = True
        #log("have awards")

    awards_list.sort(reverse=True)

    new_book_dict = {}
    tmp_list = []
    list_of_tags = []
    for book, tagsconcat in book_dict.iteritems():
        #log("book: " + str(book))
        b = str(book)
        t = str(tagsconcat)
        del tmp_list
        if not "," in t:     #failsafe for split
            t = str(str(t) + str(", "))
        tmp_list = t.split(",")
        del list_of_tags
        list_of_tags = []
        for row in tmp_list:
            tag = str(row)
            tag = str(__change_tuple_to_bytestring(str(tag)))
            tag = str(tag.strip())
            tag = str(tag.title())
            list_of_tags.append(str(tag))
        #END FOR
        #now have all single tags for a single book in list_of_tags
        award_count = 0
        if have_awards:
            for tag in list_of_tags:
                #tag = str(tag.title())
                for award in awards_list:
                    award = str(award.title())
                    #log("comparing item to award: " + str(tag) + " <> " + str(award))
                    if str(tag) == str(award):
                        award_count = award_count + 1     #need number of award tags found for this specific single book in its list of tags...
                        #log("award count is now: " + str(award_count))
                    else:
                        pass
            #END FOR
        else:
            pass
        n = len(list_of_tags)
        #log("len of list of tags: " + str(n))
        #log("max tags + award count: " + str(maximum_tags + award_count) )
        if n >  (maximum_tags + award_count) :
            #log ("n >  (maximum_tags + award_count)")
            new_list_of_tags = misc9_prioritize_tags_for_book(my_db, my_cursor, log, list_of_tags,priority_dict, maximum_tags, award_count, awards_list)

            tagsconcat = str(misc9_concat_tags_for_book(new_list_of_tags))
            new_book_dict[book] = str(tagsconcat)
            #log("tagsconcat added to new_book_dict is: " + str(tagsconcat))
        else:
            continue
    #END FOR

    return new_book_dict
    #-----------------------------------------------------------------------------------------------------------------
def misc9_prioritize_tags_for_book(my_db, my_cursor, log, list_of_tags,priority_dict, maximum_tags, award_count, awards_list):

    new_list_of_tags = []

    tmp_list_of_tags = []

    tags_to_add_to_table_tag_priorities = []

    for row in list_of_tags:
        t = str(row)
        t = str(t.title())
        #log("row in list of tags in prioritize: " + str(t))
        p = 0
        for award in awards_list:   #the list will normally be empty, of course
            award = str(award.title())
            if str(award) == str(t):
                p = 999
                #log("priority of 999 for " + str(award))
                break
        if p == 999:
            pass
        else:
            try:
                p = priority_dict[t]
            except:
                #no priority for this tag
                log("No Priority in Table _tag_priorities for Tag: " + str(t) + "  so Priority Defaulted to 0 and Added to Table _tag_priorities.  Now needs review.")
                p = 0
                tags_to_add_to_table_tag_priorities.append(str(t))

        if p < 100:
            if p < 10:
                x = str("00" + str(p))
            else:
                x = str("0" + str(p))
        else:
            if p > 999 :
                p = 999
            x = str(p)
        tmp_s = str(str(x) + str("|||") + str(t))
        tmp_list_of_tags.append(str(tmp_s))
        #log("tmp_s appended to tmp list of tags: " + str(tmp_s))
    #END FOR

    #log("tmp list of tags: " + str(tmp_list_of_tags))

    sorted_list = natsort(tmp_list_of_tags, reverse=True)   #sort descending using a natural sort comparison

    maximum_tags = maximum_tags + award_count  # award_count will normally be 0

    n_count = len(sorted_list)
    n_index = 0
    while (n_index < n_count) and (n_index <= (maximum_tags - 1)) :
        s = sorted_list[n_index]
        s_list = s.split("|||")
        t = str(s_list[1])
        t = str(apply_tag_capitalization_rules(str(t),log))
        new_list_of_tags.append(str(t))
        n_index = n_index + 1
    #END WHILE

    #print("new_list_of_tags:", str(new_list_of_tags))

    n = len(tags_to_add_to_table_tag_priorities)
    if n > 0:
        mysql = "INSERT OR IGNORE INTO _tag_priorities (tag,priority) VALUES (?,?) "
        for row in tags_to_add_to_table_tag_priorities:
            tag = str(row)
            priority = 0
            __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, tag, priority)
        #END FOR
    else:
        pass


    return new_list_of_tags
    #-----------------------------------------------------------------------------------------------------------------
def misc9_concat_tags_for_book(list_of_tags):

    list_of_tags.sort()

    tagsconcat = str("")
    for row in list_of_tags:
        tagsconcat = str(str(tagsconcat) + str(row) + str(", "))
    #END FOR
    tagsconcat = str(tagsconcat[0:-2])     #remove trailing comma space

    return str(tagsconcat)
    #-----------------------------------------------------------------------------------------------------------------
def misc9_update_custom_column_13(my_db, my_cursor, notifications, log, new_book_dict):

    n = len(new_book_dict)
    if n == 0:
        log("[4] Nothing Needs to Be Changed (Verify Configuration Settings).  Finished.")
        return

    mysql = str("UPDATE custom_column_13 SET value = ? WHERE id = ? ")
    for book, tagsconcat in new_book_dict.iteritems():
        b = str(book)
        t = str(tagsconcat)
        t = str(apply_tag_capitalization_rules(str(t),log))
        __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, t, b)
    #END FOR
    #-----------------------------------------------------------------------------------------------------------------
def misc10_control(my_db, my_cursor, notifications, log):

    from calibre_plugins.quarantine_and_scrub.config import prefs

    global current_library_path

    qs_library_best_rules = prefs['qs_library_best_rules']
    qs_library_1 = prefs['qs_library_1']
    qs_library_2 = prefs['qs_library_2']
    qs_library_3 = prefs['qs_library_3']
    qs_library_4 = prefs['qs_library_4']


    log(" ")

    log("Q&S Library with the Best Tag, Title & Series Rules and Web Source Series Validation Data (WSSVD) is:   " + str(qs_library_best_rules) )
    log(" ")
    log("Q&S Target Library [1] for Tag, Title & Series Rules and WSSVD is:   " + str(qs_library_1))
    log(" ")
    log("Q&S Target Library [2] for Tag, Title & Series Rules and WSSVD is:   " + str(qs_library_2))
    log(" ")
    log("Q&S Target Library [3] for Tag, Title & Series Rules and WSSVD is:   " + str(qs_library_3))
    log(" ")
    log("Q&S Target Library [4] for Tag, Title & Series Rules and WSSVD is:   " + str(qs_library_4))
    log(" ")

    nb= len(qs_library_best_rules)
    n1 = len(qs_library_1)
    n2 = len(qs_library_2)
    n3 = len(qs_library_3)
    n4 = len(qs_library_4)

    if nb < 16:
        log("Incorrect Q&S Configuration for the Q&S Library with the Best Tag, Title & Series Rules:   " + str(qs_library_best_rules) )
        return

    if n1 < 16 and n2 < 16 and n3 < 16 and n4 < 16:
        log("Incorrect Q&S Configuration for the 4 possible Q&S Target Libraries.  At least one (1) of them must be valid.")
        return

    if (qs_library_best_rules == qs_library_1 or \
         qs_library_best_rules == qs_library_2 or \
         qs_library_best_rules == qs_library_3 or \
         qs_library_best_rules == qs_library_4):
        log("Your Best Q&S Library Cannot Be a Target Q&S Library!  Invalid Configuration.  Terminating.")
        return


    if qs_library_best_rules <> current_library_path :
        path1 = str(qs_library_best_rules)
        path1 = str(path1.replace(os.sep, '/'))
        path2 = str(current_library_path)
        path2 = str(path2.replace(os.sep, '/'))
        if path1 <> path2:
            log("This job MUST be run from your Best Q&S Library.  Terminating.")
            log("If you feel this message is in error, verify  '/'s and '\\'s in your configuration of the path of your Best Q&S Library.")
            return


    status = True
    status_best = False
    status_t1 = False
    status_t2 = False
    status_t3 = False
    status_t4 = False

    name = str("Best")
    path = qs_library_best_rules
    status = misc10_attach_qs_metadata_db(my_db, my_cursor, path, name, log)
    if not status:
        return
    else:
        status_best = True

    if n1 >= 16:
        name = str("T1")
        path = qs_library_1
        status = misc10_attach_qs_metadata_db(my_db, my_cursor, path, name, log)
        if not status:
            return
        else:
            status_t1 = True

    if n2 >= 16:
        name = str("T2")
        path = qs_library_2
        status = misc10_attach_qs_metadata_db(my_db, my_cursor, path, name, log)
        if not status:
            return
        else:
            status_t2 = True

    if n3 >= 16:
        name = str("T3")
        path = qs_library_3
        status = misc10_attach_qs_metadata_db(my_db, my_cursor, path, name, log)
        if not status:
            return
        else:
            status_t3 = True

    if n4 >= 16:
        name = str("T4")
        path = qs_library_4
        status = misc10_attach_qs_metadata_db(my_db, my_cursor, path, name, log)
        if not status:
            return
        else:
            status_t4 = True

    if status_best and (status_t1 or status_t2 or status_t3 or status_t4):
        misc10_create_in_memory_db(my_db, my_cursor, log)
        misc10_copy_rules_among_qs_databases(my_db, my_cursor, log, status_t1, status_t2, status_t3, status_t4)
        misc10_log_results(my_db, my_cursor, notifications, log)
        misc10_copy_tags_to_best_db(my_db, my_cursor, log)
        return
    else:
        log("Incorrect Q&S Configuration for the 4 possible Q&S Target Libraries.  At least one of them must be valid.")
        return
  #----------------------------------------------------------------------------------------------------------------
def misc10_create_in_memory_db(my_db, my_cursor, log):

    try:
        sleep(0.5)
        mysql = "ATTACH DATABASE ':memory:' AS 'Mem1'   ;"
        my_cursor.execute(mysql)
        my_cursor.execute("begin")  #apsw
        mysql = 'CREATE TABLE IF NOT EXISTS Mem1._tags_copied_from_other_libraries ( id   INTEGER PRIMARY KEY,name TEXT NOT NULL COLLATE NOCASE,UNIQUE (name) )'
        my_cursor.execute (mysql)
        mysql = 'CREATE TABLE IF NOT EXISTS Mem1._tags_work_single (tag TEXT NOT NULL , subject TEXT NOT NULL , PRIMARY KEY (tag, subject))'
        my_cursor.execute (mysql)
        mysql = "CREATE TABLE IF NOT EXISTS Mem1._global_web_author_series (authname TEXT NOT NULL , seriesname TEXT NOT NULL , source_url TEXT NOT NULL , \
                        book_count INTEGER NOT NULL  DEFAULT 0, date_added DATETIME NOT NULL  DEFAULT CURRENT_TIMESTAMP, \
                        PRIMARY KEY (authname, seriesname)  UNIQUE (authname, seriesname) ON CONFLICT REPLACE )"
        my_cursor.execute (mysql)
        mysql = "CREATE TABLE IF NOT EXISTS Mem1._global_web_series_detail (authname TEXT NOT NULL, seriesname TEXT NOT NULL, booktitle TEXT NOT NULL ,\
                        seriesindex TEXT NOT NULL , source_url TEXT NOT NULL , date_added DATETIME NOT NULL  DEFAULT CURRENT_TIMESTAMP,\
                        PRIMARY KEY (authname, seriesname, booktitle) UNIQUE (authname, seriesname, booktitle) ON CONFLICT REPLACE )"
        my_cursor.execute (mysql)
        mysql = "CREATE TABLE IF NOT EXISTS Mem1._pristine_authors  ( id   INTEGER PRIMARY KEY, \
                              name TEXT NOT NULL COLLATE NOCASE, \
                              sort TEXT COLLATE NOCASE, \
                              link TEXT NOT NULL DEFAULT ' ' ,  \
                              UNIQUE(name) )"
        my_cursor.execute (mysql)
        my_cursor.execute("commit")
        print("In Memory Temporary DB Created")
        sleep(0.02)
    except Exception as e:
        log(str(e))
        log("FAILED:  CREATE TABLE IF NOT EXISTS Mem1._tags_copied_from_other_libraries ")
        pass   #do nothing if it already exists, including adding any new seed rows
  #----------------------------------------------------------------------------------------------------------------
def misc10_copy_tags_to_best_db(my_db, my_cursor, log):
    #since this job can only be run from "the best", "main" is really "best".  need to detach all attached databases to avoid a db lock error.

    try:
        mysql = "DETACH 'T1'"
        my_cursor.execute (mysql)
    except:
        pass

    try:
        mysql = "DETACH 'T2'"
        my_cursor.execute (mysql)
    except:
        pass

    try:
        mysql = "DETACH 'T3'"
        my_cursor.execute (mysql)
    except:
        pass

    try:
        mysql = "DETACH 'T4'"
        my_cursor.execute (mysql)
    except:
        pass

    try:
        mysql = "DETACH 'Best'"
        my_cursor.execute (mysql)
    except:
        pass

    try:
        my_cursor.execute("begin")  #apsw
        mysql = "DELETE FROM main._tags_copied_from_other_libraries"
        my_cursor.execute (mysql)
        my_cursor.execute ("commit")
        sleep(0.5)

        my_cursor.execute("begin")  #apsw
        mysql = "INSERT OR IGNORE INTO main._tags_copied_from_other_libraries  SELECT null, name FROM Mem1._tags_copied_from_other_libraries \
                                                                        WHERE name NOT IN(SELECT oldtag FROM main._tag_rules )"
        my_cursor.execute (mysql)
        my_cursor.execute ("commit")
        sleep(0.5)

        my_cursor.execute("begin")  #apsw
        mysql = "INSERT OR IGNORE INTO main._tags_work_single  SELECT tag, subject FROM Mem1._tags_work_single \
                                                                        WHERE tag NOT IN(SELECT oldtag FROM main._tag_rules )"
        my_cursor.execute (mysql)
        my_cursor.execute ("commit")
        sleep(0.5)

        my_cursor.execute("begin")  #apsw
        mysql = "INSERT OR IGNORE INTO main._global_web_author_series SELECT * FROM Mem1._global_web_author_series;"
        my_cursor.execute (mysql)
        my_cursor.execute ("commit")
        sleep(0.5)

        my_cursor.execute("begin")  #apsw
        mysql = "INSERT OR IGNORE INTO main._global_web_series_detail SELECT * FROM Mem1._global_web_series_detail;"
        my_cursor.execute (mysql)
        my_cursor.execute ("commit")
        sleep(0.5)

        my_cursor.execute("begin")  #apsw
        mysql = "INSERT OR IGNORE INTO main._pristine_authors SELECT * FROM Mem1._pristine_authors;"
        my_cursor.execute (mysql)
        my_cursor.execute ("commit")
        sleep(0.5)


    except Exception as e:
        log(str(e))
        log("FAILED: INSERT OR IGNORE INTO main._tags_copied_from_other_libraries  SELECT * FROM Mem1._tags_copied_from_other_libraries")
        pass
  #----------------------------------------------------------------------------------------------------------------
def misc10_log_results(my_db, my_cursor, notifications, log):

    log(" ")
    log("===================================")
    log(" ")

    mysql = "SELECT Count(*) FROM Best._tag_rules"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Tag Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._tag_priorities"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Tag Priorities Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._title_rules"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Title Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._series_rules"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Series Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._pristine_authors"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Pristine Authors Copied from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._global_web_author_series"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Web Source Series Validation Author/Series Records Copied from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._global_web_series_detail"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Web Source Series Validation Series/Book/Index Records Copied from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._tags_by_author_default"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Author-Tag Defaults Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._tags_by_comment"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Tags by Comment Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._tag_combination_rules"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Tag Combination Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")


    mysql = "SELECT Count(*) FROM Best._tag_capitalization_rules"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Tag Capitalization Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")


    mysql = "SELECT Count(*) FROM Best._tag_string_replacement_rules"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Tag String Replacement Rules Copied from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._dg_genre_author_rules"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Derive Genres Author Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._dg_genre_tag_rules_factual"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Derive Genres Factual Tag Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._dg_genre_tag_rules_fiction"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Derive Genres Fiction Tag Rules Cloned from Best to Target(s):  ", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM Best._dg_nf_language_keywords"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log("Number of Derive Genres Non-Fiction Language Keywords Cloned from Best to Target(s):  ", str(count1))
    log(" ")


    log(" ")
    log("Cloned:  the Target table was emptied, and then all new records from the Best library were copied to the Target")
    log("Copied:  the Target table had new records added, or existing records changed to the Best library values, but no deletes.")
    log(" ")
    log("Real Tags from all Target Libraries were copied to a Best Q&S Table used to offer choices for the 'Easy-Add' Tags-to-Purge Rules.")
    log("When you once again run this job to copy the Best tag rules back to the Targets, they will inherit any Best Tags-to-Purge Rules created using their own Tags.")
    log(" ")
    log("Work Tags from all Target Libraries were copied to a Best Q&S Table used to offer choices for the 'Easy-Add' Work Tags-to-NEWTAG Mapping Rules.")
    log("When you once again run this job to copy the Best tag rules back to the Targets, they will inherit any Best Work Tags-to-NEWTAG Mapping Rules created using their own Work Tags.")
    log(" ")
    log("Any Target Web Source Series Validation Series/Book/Index Records have been added to the Best Q&S Library for its own use plus future copying to all Targets.")
    log(" ")
    log("===================================")
    log(" ")
  #----------------------------------------------------------------------------------------------------------------
def misc10_copy_rules_among_qs_databases(my_db, my_cursor, log, status_t1, status_t2, status_t3, status_t4):

    try:
        my_cursor.execute("commit")    #to avoid any job failures due to an outstanding BEGIN TRANSACTION created elsewhere
    except:
        pass

    if status_t1:
        try:
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION;\
                DELETE FROM T1._tag_priorities;\
                DELETE FROM T1._tag_rules;\
                DELETE FROM T1._title_rules;\
                DELETE FROM T1._series_rules;\
                DELETE FROM T1._tags_by_author_default;\
                DELETE FROM T1._tags_by_comment;\
                DELETE FROM T1._tag_combination_rules;\
                DELETE FROM T1._tag_capitalization_rules;\
                DELETE FROM T1._dg_genre_author_rules;\
                DELETE FROM T1._dg_genre_tag_rules_factual;\
                DELETE FROM T1._dg_genre_tag_rules_fiction;\
                DELETE FROM T1._dg_nf_language_keywords;\
                COMMIT; ")
            print("T1: after deletes.....")
        except Exception as e:
            log(str(e))
            my_db.close()
            raise e
            return

        try:
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION; \
                INSERT OR IGNORE INTO T1._tags_by_author_default  SELECT * FROM Best._tags_by_author_default; \
                COMMIT; ")
        except:
            try:
                my_cursor.execute("commit")    #to avoid any job failures due to an outstanding BEGIN TRANSACTION created elsewhere
                print("potential database lock error; sleeping for 6 seconds")
                sleep(6.0)    #avoid database lock error just below
            except:
                print("potential database lock error; sleeping for 6 seconds")
                sleep(6.0)    #avoid database lock error just below
                pass

        try:
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION; \
                INSERT OR REPLACE INTO T1._tags_by_author_default  SELECT * FROM Best._tags_by_author_default; \
                INSERT OR REPLACE INTO T1._tags_by_comment  SELECT * FROM Best._tags_by_comment; \
                INSERT OR REPLACE INTO T1._tag_priorities    SELECT * FROM Best._tag_priorities;\
                COMMIT; ")
            print("T1: after inserts.[1]....")
            sleep(0.02)
        except Exception as e:
            log(str(e))
            my_db.close()
            raise e
            return

        try:
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION; \
                INSERT OR REPLACE INTO T1._tag_rules          SELECT * FROM Best._tag_rules;\
                INSERT OR REPLACE INTO T1._tag_combination_rules          SELECT * FROM Best._tag_combination_rules;\
                INSERT OR REPLACE INTO T1._tag_capitalization_rules          SELECT * FROM Best._tag_capitalization_rules;\
                INSERT OR REPLACE INTO T1._tag_string_replacement_rules          SELECT * FROM Best._tag_string_replacement_rules;\
                INSERT OR REPLACE INTO T1._title_rules         SELECT * FROM Best._title_rules;\
                INSERT OR REPLACE INTO T1._series_rules         SELECT * FROM Best._series_rules;\
                INSERT OR IGNORE INTO T1._book_awards         SELECT * FROM Best._book_awards;\
                INSERT OR IGNORE INTO T1._global_authors         SELECT * FROM Best._global_authors;\
                INSERT OR REPLACE INTO T1._dg_genre_author_rules               SELECT * FROM Best._dg_genre_author_rules ;\
                INSERT OR REPLACE INTO T1._dg_genre_tag_rules_factual        SELECT * FROM Best._dg_genre_tag_rules_factual ;\
                INSERT OR REPLACE INTO T1._dg_genre_tag_rules_fiction         SELECT * FROM Best._dg_genre_tag_rules_fiction ;\
                INSERT OR REPLACE INTO T1._dg_nf_language_keywords         SELECT * FROM Best._dg_nf_language_keywords ;\
                COMMIT; ")
            print("T1: after inserts.[1]....")
            sleep(0.02)
        except Exception as e:
            log(str(e))
            my_db.close()
            raise e
            return

        try:
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION;\
                INSERT OR REPLACE INTO T1._global_web_author_series        SELECT * FROM Best._global_web_author_series;\
                INSERT OR REPLACE INTO T1._global_web_series_detail          SELECT * FROM Best._global_web_series_detail;\
                INSERT OR IGNORE INTO T1._pristine_authors                         SELECT * FROM Best._pristine_authors;\
                INSERT OR IGNORE INTO Mem1._tags_copied_from_other_libraries  SELECT null, name FROM T1.tags WHERE name NOT LIKE '%978%' AND name NOT LIKE '%isbn%'; \
                INSERT OR IGNORE INTO Mem1._tags_work_single SELECT tag, subject FROM T1._tags_work_single  ;\
                INSERT OR IGNORE INTO Mem1._global_web_author_series SELECT * FROM T1._global_web_author_series;\
                INSERT OR IGNORE INTO Mem1._global_web_series_detail SELECT * FROM T1._global_web_series_detail;\
                INSERT OR IGNORE INTO Mem1._pristine_authors SELECT * FROM T1._pristine_authors;\
                COMMIT; ")
            print("T1: after inserts.[2]....")
            sleep(0.02)
        except Exception as e:
            log(str(e))
            my_db.close()
            raise e
            return

    if status_t2:
        try:
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION;\
                DELETE FROM T2._tag_priorities;\
                DELETE FROM T2._tag_rules;\
                DELETE FROM T2._title_rules;\
                DELETE FROM T2._series_rules;\
                DELETE FROM T2._tags_by_author_default;\
                DELETE FROM T2._tags_by_comment;\
                DELETE FROM T2._tag_combination_rules;\
                DELETE FROM T2._tag_capitalization_rules;\
                DELETE FROM T2._dg_genre_author_rules;\
                DELETE FROM T2._dg_genre_tag_rules_factual;\
                DELETE FROM T2._dg_genre_tag_rules_fiction;\
                DELETE FROM T2._dg_nf_language_keywords;\
                COMMIT; ")
            print("t2: after deletes.....")
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION;\
                INSERT OR REPLACE INTO T2._tags_by_author_default  SELECT * FROM Best._tags_by_author_default; \
                INSERT OR REPLACE INTO T2._tags_by_comment  SELECT * FROM Best._tags_by_comment; \
                INSERT OR REPLACE INTO T2._tag_priorities    SELECT * FROM Best._tag_priorities;\
                INSERT OR REPLACE INTO T2._tag_rules          SELECT * FROM Best._tag_rules;\
                INSERT OR REPLACE INTO T2._tag_combination_rules          SELECT * FROM Best._tag_combination_rules;\
                INSERT OR REPLACE INTO T2._tag_capitalization_rules          SELECT * FROM Best._tag_capitalization_rules;\
                INSERT OR REPLACE INTO T2._tag_string_replacement_rules          SELECT * FROM Best._tag_string_replacement_rules;\
                INSERT OR REPLACE INTO T2._title_rules         SELECT * FROM Best._title_rules;\
                INSERT OR REPLACE INTO T2._series_rules         SELECT * FROM Best._series_rules;\
                INSERT OR IGNORE INTO T2._book_awards         SELECT * FROM Best._book_awards;\
                INSERT OR IGNORE INTO T2._global_authors         SELECT * FROM Best._global_authors;\
                INSERT OR REPLACE INTO T2._dg_genre_author_rules               SELECT * FROM Best._dg_genre_author_rules ;\
                INSERT OR REPLACE INTO T2._dg_genre_tag_rules_factual        SELECT * FROM Best._dg_genre_tag_rules_factual ;\
                INSERT OR REPLACE INTO T2._dg_genre_tag_rules_fiction         SELECT * FROM Best._dg_genre_tag_rules_fiction ;\
                INSERT OR REPLACE INTO T2._dg_nf_language_keywords         SELECT * FROM Best._dg_nf_language_keywords ;\
                INSERT OR REPLACE INTO T2._global_web_author_series        SELECT * FROM Best._global_web_author_series;\
                INSERT OR REPLACE INTO T2._global_web_series_detail          SELECT * FROM Best._global_web_series_detail;\
                INSERT OR IGNORE INTO T2._pristine_authors                         SELECT * FROM Best._pristine_authors;\
                INSERT OR IGNORE INTO Mem1._tags_copied_from_other_libraries  SELECT null, name FROM T2.tags WHERE name NOT LIKE '%978%' AND name NOT LIKE '%isbn%'; \
                INSERT OR IGNORE INTO Mem1._tags_work_single SELECT tag, subject FROM T2._tags_work_single  ;\
                INSERT OR IGNORE INTO Mem1._global_web_author_series SELECT * FROM T2._global_web_author_series;\
                INSERT OR IGNORE INTO Mem1._global_web_series_detail SELECT * FROM T2._global_web_series_detail;\
                INSERT OR IGNORE INTO Mem1._pristine_authors SELECT * FROM T2._pristine_authors;\
                COMMIT; ")
            print("t2: after inserts.....")
            sleep(0.02)
        except Exception as e:
            log(str(e))
            my_db.close()
            raise e
            return

    if status_t3:
        try:
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION;\
                DELETE FROM T3._tag_priorities;\
                DELETE FROM T3._tag_rules;\
                DELETE FROM T3._title_rules;\
                DELETE FROM T3._series_rules;\
                DELETE FROM T3._tags_by_author_default;\
                DELETE FROM T3._tags_by_comment;\
                DELETE FROM T3._tag_combination_rules;\
                DELETE FROM T3._tag_capitalization_rules;\
                DELETE FROM T3._dg_genre_author_rules;\
                DELETE FROM T3._dg_genre_tag_rules_factual;\
                DELETE FROM T3._dg_genre_tag_rules_fiction;\
                DELETE FROM T3._dg_nf_language_keywords;\
                COMMIT; ")
            print("T3: after deletes.....")
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION;\
                INSERT OR REPLACE INTO T3._tags_by_author_default  SELECT * FROM Best._tags_by_author_default; \
                INSERT OR REPLACE INTO T3._tags_by_comment  SELECT * FROM Best._tags_by_comment; \
                INSERT OR REPLACE INTO T3._tag_priorities    SELECT * FROM Best._tag_priorities;\
                INSERT OR REPLACE INTO T3._tag_rules          SELECT * FROM Best._tag_rules;\
                INSERT OR REPLACE INTO T3._tag_combination_rules          SELECT * FROM Best._tag_combination_rules;\
                INSERT OR REPLACE INTO T3._tag_capitalization_rules          SELECT * FROM Best._tag_capitalization_rules;\
                INSERT OR REPLACE INTO T3._tag_string_replacement_rules          SELECT * FROM Best._tag_string_replacement_rules;\
                INSERT OR REPLACE INTO T3._title_rules         SELECT * FROM Best._title_rules;\
                INSERT OR REPLACE INTO T3._series_rules         SELECT * FROM Best._series_rules;\
                INSERT OR IGNORE INTO T3._book_awards         SELECT * FROM Best._book_awards;\
                INSERT OR IGNORE INTO T3._global_authors         SELECT * FROM Best._global_authors;\
                INSERT OR REPLACE INTO T3._dg_genre_author_rules               SELECT * FROM Best._dg_genre_author_rules ;\
                INSERT OR REPLACE INTO T3._dg_genre_tag_rules_factual        SELECT * FROM Best._dg_genre_tag_rules_factual ;\
                INSERT OR REPLACE INTO T3._dg_genre_tag_rules_fiction         SELECT * FROM Best._dg_genre_tag_rules_fiction ;\
                INSERT OR REPLACE INTO T3._dg_nf_language_keywords         SELECT * FROM Best._dg_nf_language_keywords ;\
                INSERT OR REPLACE INTO T3._global_web_author_series        SELECT * FROM Best._global_web_author_series;\
                INSERT OR REPLACE INTO T3._global_web_series_detail          SELECT * FROM Best._global_web_series_detail;\
                INSERT OR IGNORE INTO T3._pristine_authors                         SELECT * FROM Best._pristine_authors;\
                INSERT OR IGNORE INTO Mem1._tags_copied_from_other_libraries  SELECT null, name FROM T3.tags WHERE name NOT LIKE '%978%' AND name NOT LIKE '%isbn%'; \
                INSERT OR IGNORE INTO Mem1._tags_work_single SELECT tag, subject FROM T3._tags_work_single  ;\
                INSERT OR IGNORE INTO Mem1._global_web_author_series SELECT * FROM T3._global_web_author_series;\
                INSERT OR IGNORE INTO Mem1._global_web_series_detail SELECT * FROM T3._global_web_series_detail;\
                INSERT OR IGNORE INTO Mem1._pristine_authors SELECT * FROM T3._pristine_authors;\
                COMMIT; ")
            print("T3: after inserts....")
            sleep(0.02)
        except Exception as e:
            log(str(e))
            my_db.close()
            raise e
            return

    if status_t4:
        try:
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION;\
                DELETE FROM T4._tag_priorities;\
                DELETE FROM T4._tag_rules;\
                DELETE FROM T4._title_rules;\
                DELETE FROM T4._series_rules;\
                DELETE FROM T4._tags_by_author_default;\
                DELETE FROM T4._tags_by_comment;\
                DELETE FROM T4._tag_combination_rules;\
                DELETE FROM T4._tag_capitalization_rules;\
                DELETE FROM T4._dg_genre_author_rules;\
                DELETE FROM T4._dg_genre_tag_rules_factual;\
                DELETE FROM T4._dg_genre_tag_rules_fiction;\
                DELETE FROM T4._dg_nf_language_keywords;\
                COMMIT; ")
            print("T4: after deletes.....")
            sleep(0.02)
            my_cursor.execute("\
                BEGIN TRANSACTION;\
                INSERT OR REPLACE INTO T4._tags_by_author_default  SELECT * FROM Best._tags_by_author_default; \
                INSERT OR REPLACE INTO T4._tags_by_comment  SELECT * FROM Best._tags_by_comment; \
                INSERT OR REPLACE INTO T4._tag_priorities    SELECT * FROM Best._tag_priorities;\
                INSERT OR REPLACE INTO T4._tag_rules          SELECT * FROM Best._tag_rules;\
                INSERT OR REPLACE INTO T4._tag_combination_rules          SELECT * FROM Best._tag_combination_rules;\
                INSERT OR REPLACE INTO T4._tag_capitalization_rules          SELECT * FROM Best._tag_capitalization_rules;\
                INSERT OR REPLACE INTO T4._tag_string_replacement_rules          SELECT * FROM Best._tag_string_replacement_rules;\
                INSERT OR REPLACE INTO T4._title_rules         SELECT * FROM Best._title_rules;\
                INSERT OR REPLACE INTO T4._series_rules         SELECT * FROM Best._series_rules;\
                INSERT OR IGNORE INTO T4._book_awards         SELECT * FROM Best._book_awards;\
                INSERT OR IGNORE INTO T4._global_authors         SELECT * FROM Best._global_authors;\
                INSERT OR REPLACE INTO T4._dg_genre_author_rules               SELECT * FROM Best._dg_genre_author_rules ;\
                INSERT OR REPLACE INTO T4._dg_genre_tag_rules_factual        SELECT * FROM Best._dg_genre_tag_rules_factual ;\
                INSERT OR REPLACE INTO T4._dg_genre_tag_rules_fiction         SELECT * FROM Best._dg_genre_tag_rules_fiction ;\
                INSERT OR REPLACE INTO T4._dg_nf_language_keywords         SELECT * FROM Best._dg_nf_language_keywords ;\
                INSERT OR REPLACE INTO T4._global_web_author_series        SELECT * FROM Best._global_web_author_series;\
                INSERT OR REPLACE INTO T4._global_web_series_detail          SELECT * FROM Best._global_web_series_detail;\
                INSERT OR IGNORE INTO T4._pristine_authors                         SELECT * FROM Best._pristine_authors;\
                INSERT OR IGNORE INTO Mem1._tags_copied_from_other_libraries  SELECT null, name FROM T4.tags WHERE name NOT LIKE '%978%' AND name NOT LIKE '%isbn%'; \
                INSERT OR IGNORE INTO Mem1._tags_work_single SELECT tag, subject FROM T4._tags_work_single  ;\
                INSERT OR IGNORE INTO Mem1._global_web_author_series SELECT * FROM T4._global_web_author_series;\
                INSERT OR IGNORE INTO Mem1._global_web_series_detail SELECT * FROM T4._global_web_series_detail;\
                INSERT OR IGNORE INTO Mem1._pristine_authors SELECT * FROM T4._pristine_authors;\
                COMMIT; ")
            print("T4: after inserts.....")
            sleep(0.02)
        except Exception as e:
            log(str(e))
            my_db.close()
            raise e
            return

    log(" ")
    log(" ")
    log("Tag Rules, Title Rules, Series Rules, Web Source Series Validation Data, Author-Tag Defaults, DG Rules, ")
    log("Global Authors and Pristine Authors were copied from your Best Q&S library to your other configured libraries.")
  #----------------------------------------------------------------------------------------------------------------
def misc10_attach_qs_metadata_db(my_db, my_cursor, path, name, log):

    status = True

    try:
        s1 = "ATTACH DATABASE '"
        s2 =  "'  As '[NAME]';"
        s2 = str(s2.replace("[NAME]", str(name), 1))
        path = str(path)
        path = str(path.replace(os.sep, '/'))          #version 3.2.8
        mysql = s1 + path + s2
        if isbytestring(mysql):
            mysql = mysql.decode(filesystem_encoding)
        my_cursor.execute(mysql)
        log("Q&S Library metadata.db has been properly attached:     " + str(path))
        return True
    except Exception as e:
        log("Q&S Library metadata.db has NOT been properly attached")
        log(mysql)
        print(str(e))
        s = str(e)
        log(s)
        my_db.close()
        print("terminating early...")
        log("terminating early...")
        return False
    #-----------------------------------------------------------------------------------------------------------------
def misc14_control(my_db, my_cursor, notifications, log):

    global mynothing
    global my_run_type
    global this_date
    global my_selected_book_list

    tmp_books_list = []

    for row in my_selected_book_list:
        books = str(row)
        books = str(__strip_numerics(books))
        tmp_books_list.append(books)
        book = books
        break  #only one(1) book since only one(1) author
    #END FOR
    del my_selected_book_list

    n_total = len(tmp_books_list)
    if n_total == 0:
        log("No Authors Selected.")
        return

    socket.setdefaulttimeout(15)

    text_list = []

    n_counter = 0

    my_url_domain = str("www.fictiondb.com")
    my_url = str("http://www.fictiondb.com/series/author-series~[LETTER].htm")    #not really; just a placeholder.

    try:

        #-------------------------------------------------------
        #-------------------------------------------------------

        authname = mynothing

        mysql = "SELECT name FROM __book_author_name_sort WHERE book = [BOOK] "
        mysql = mysql.replace("[BOOK]", book, 1)
        tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
        if not tmp_rows:
            log("Invalid Author")
            return
        else:
            n = len(tmp_rows)
            if n == 0:
                log("Invalid Author")
                return
            else:
                for row in tmp_rows:
                    for col in row:
                        authname = col
                        break
                    break
                #END FOR

        if authname == mynothing or (not authname > " "):
            log("Invalid Author")
            return

        try:
            authname = unicodedata.normalize('NFKD', authname).encode('ascii', 'ignore')
        except:
            pass

        firstname = ""
        lastname = ""

        if "-" in authname:      #e.g. Cindy Holby - Wind 01 - Chase the Wind
            n = authname.find("-")
            if n > 0:    #not a leading - in authors name!
                print("authname with a dash in it:", str(authname))
                authname = str(authname[0:n])
                if authname.endswith("-"):
                    authname = str(authname[0:-1])
                authname = str(authname.strip())
        if not "," in authname:
            if " " in authname:
                auth_split = authname.split(" ")
                firstname = str(auth_split[0])
                lastname = str(auth_split[1])
            else:
                firstname = str(" ")
                lastname = str(authname)
        else:
            auth_split = authname.split(",")
            firstname = str(auth_split[1])
            lastname = str(auth_split[0])
        firstname = str(__change_tuple_to_bytestring(str(firstname)))
        lastname = str(__change_tuple_to_bytestring(str(lastname)))

        log("Author Selected Was: " + authname)
        print("Firstname: " + firstname + "   Lastname: " + lastname)

        series_list = misc14_get_all_series_for_single_author(my_db, my_cursor, firstname,lastname, notifications, log)
        n = len(series_list)
        if n == 0:
            log("<font size='3'><b><font color='#FF0000'>Web Source Found No Series For This Author</b></font>")
            return
        else:
            notifications.put((0.10, 'Retrieving Web Booktitles/Indexes for Series of Selected Author'))
            misc14_update_temp_web_tables(my_db, my_cursor, authname,firstname,lastname,series_list, notifications, log)
    except Exception as e:
        log(str(e))
        my_db.close()
        return


    sleep(0.02)
    mysql = str('INSERT OR REPLACE INTO _web_source_url_control (source_url) VALUES( "' + my_url_domain + '") ')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.02)
    #keep everything downloaded in other tables available for searching without re-downloading
    mysql = str('INSERT OR REPLACE INTO _global_web_author_series SELECT * FROM _web_author_series')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.02)
    mysql = str('INSERT OR REPLACE INTO _global_web_series_detail SELECT * FROM _web_series_detail')
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.02)
    mysql = str("DELETE FROM _global_web_author_series WHERE authname LIKE '%-%' OR authname like '%0%' or  authname like '%1%' \
            OR authname like '%2%' or  authname like '%3%' OR authname like '%4%' OR  authname like '%5%' OR authname like '%6%' \
            OR  authname like '%7%' OR authname like '%8%' OR  authname like '%9%' OR  authname like '%\%' \
            OR authname LIKE '%</span>%' OR authname LIKE '%</a>%' OR authname LIKE '%~%'  OR authname LIKE '%unknown%' \
            OR seriesname LIKE '%</span>%'   ")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.02)   #Note: 3.6.69 deletion of index == 0 was removed
    mysql = str("DELETE FROM _global_web_series_detail WHERE authname LIKE '%-%' OR authname like '%0%' or  authname like '%1%' \
            OR authname like '%2%' or  authname like '%3%' OR authname like '%4%' OR  authname like '%5%' OR authname like '%6%' \
            OR  authname like '%7%' OR authname like '%8%' OR  authname like '%9%' OR  authname like '%\%' \
            OR authname LIKE '%</span>%' OR authname LIKE '%</a>%' OR authname LIKE '%~%'  OR authname LIKE '%unknown%'  \
            OR seriesname LIKE '%</span>%' \
            OR booktitle LIKE '%</span>%' OR booktitle LIKE '%</a>%' OR booktitle LIKE '%~%' \
            OR seriesname LIKE '%</span>%' OR booktitle LIKE '%</a>%' OR booktitle LIKE '%~%'    ")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.02)
    mysql = str("INSERT OR IGNORE INTO _global_authors SELECT null,authname,'none',' ' FROM _web_author_series")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    sleep(0.02)
    mysql = str("INSERT OR IGNORE INTO _global_series SELECT null,seriesname,seriesname FROM _web_author_series")
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    log(".")
    sleep(0.02)


    for row in my_cursor.execute('SELECT authname,seriesname,booktitle,seriesindex FROM _web_series_detail ORDER BY seriesname,seriesindex'):
        authname,seriesname,booktitle,seriesindex = row
        log(str(authname) + "  " + str(seriesname) + "  " + str(booktitle) + "  " + str(seriesindex))

    my_db.close()

    log(" ")
    log("The Rename Work Series Name job now will have new WSSVD to use, as will Book Level Scrubbing for Work Title and Work Series Renaming.")
    log(" ")
    #-----------------------------------------------------------------------------------------------------------------
def misc14_get_all_series_for_single_author(my_db, my_cursor, firstname,lastname, notifications, log):

    global mynothing

    series_list = []      #accumulate the series names

    try:
        firstname = firstname.lower()
        lastname = lastname.lower()
    except:
        pass

    firstname = firstname.replace(".",mynothing,4)    #  J.D. Robb should be JD Robb for this purpose.


    '''
        http://www.fictiondb.com/series/author-series~a.htm        returns:

        tr class="g" ><td><span class="srtkey">Bad Boys o</span><a class="hghlt" href="../author/bella-andre~series~bad-boys-of-football~21385.htm">Bad Boys of Football</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
		<tr class="g" ><td><span class="srtkey">Hot Shots</span><a class="hghlt" href="../author/bella-andre~series~hot-shots~15693.htm">Hot Shots</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
		<tr class="g" ><td><span class="srtkey">Morrisons</span><a class="hghlt" href="../author/bella-andre~series~the-morrisons~36567.htm">The Morrisons</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
		<tr class="g" ><td><span class="srtkey">Sullivans</span><a class="hghlt" href="../author/bella-andre~series~the-sullivans~21394.htm">The Sullivans</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
        <tr class="g" ><td><span class="srtkey">Take Me</span><a class="hghlt" href="../author/bella-andre~series~take-me~21392.htm">Take Me</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
    '''

    #http://www.fictiondb.com/series/author-series~[?].htm
    my_url = str('http://www.fictiondb.com/series/author-series~[?].htm')
    my_url = str(my_url.replace("[?]",str(lastname[0:1])))
    print(my_url)

    #href="../author/bella-andre~series
    search_series_row = str('href="../author/[FIRSTNAME]-[LASTNAME]~series')
    search_series_row = str(search_series_row.replace("[FIRSTNAME]",firstname,1))
    search_series_row = str(search_series_row.replace("[LASTNAME]",lastname,1))
    print(search_series_row)

    try:
        req = urllib2.Request(my_url)
        response = urllib2.urlopen(req)
        soup = BeautifulSoup(response)
        response.close()
    except Exception as e:
        print(str(e))
        return series_list
    if not soup:
        print("no soup...")
        return series_list
    else:
        print("soup was found")
        pass

    results = soup.tbody
    if not results:
        print("no results; returning")
        return series_list

    n = len(results)
    print("results length:" + str(n))
    if n == 0:
        return series_list

    re_x = "class\=\"hghlt\"[ ]*[~][a-z \-'&]+[~][0-9]+[.htm]+"
    try:
        p1 = re.compile(re_x, re.IGNORECASE)
    except:
        log("REGEX COMPILE ERROR in Parsing Web Page for: " + re_x)
        return series_list

    log(" ")

    for row in results:
        s = str(row)
        n = s.find(str(search_series_row))
        if not n >= 0:
            continue
        else:
            #print("results row: " + str(s))
            s = s.replace(search_series_row,mynothing,1)
            match1 = p1.search(str(s))
            if match1:        #   class="hghlt" ~bad-boys-of-football~21385.htm
                s = str(match1.group(0))
                s = str(s.replace('class="hghlt"',mynothing,1))
                s = str(s.strip())
                if s.startswith("~"):   #~bad-boys-of-football~21385.htm
                    s = str(s[1: ])     #  bad-boys-of-football~21385.htm
                s_split = s.split("~")
                s = str(s_split[0])    #  bad-boys-of-football
                s = str(s.replace('-',' ',20))   #   bad boys of football
                s = str(s.strip())
                series_list.append(str(s))
                s = s.title()
                print("Web Series for Author: " + str(s))
                log("Web Series for Author: " + str(s))
    #END FOR


    '''
    Author Selected Was: Bella Andre
    Firstname: Bella   Lastname: Andre
    http://www.fictiondb.com/series/author-series~a.htm
    href="../author/bella-andre~series
    soup was found
    results row: <tr class="g"><td><span class="srtkey">Bad Boys o</span><a class="hghlt" ~bad-boys-of-football~21385.htm">Bad Boys of Football</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
    results row: <tr class="g"><td><span class="srtkey">Hot Shots</span><a class="hghlt" ~hot-shots~15693.htm">Hot Shots</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
    results row: <tr class="g"><td><span class="srtkey">Morrisons</span><a class="hghlt" ~the-morrisons~36567.htm">The Morrisons</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
    results row: <tr class="g"><td><span class="srtkey">Sullivans</span><a class="hghlt" ~the-sullivans~21394.htm">The Sullivans</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
    results row: <tr class="g"><td><span class="srtkey">Take Me</span><a class="hghlt" ~take-me~21392.htm">Take Me</a></td><td><a href="../author/bella-andre~40263.htm">Andre, Bella</a></td></tr>
    '''


    return series_list
    #-----------------------------------------------------------------------------------------------------------------
def misc14_update_temp_web_tables(my_db, my_cursor,authname,firstname,lastname,series_list, notifications, log):

    misc4_purge_old_web_table_data(my_db, my_cursor, log)  #shared with misc4

    firstname = firstname.replace(".",mynothing,4)    #  J.D. Robb should be JD Robb for this purpose.

    for row in series_list:
        seriesname = str(row)
        misc14_unpack_soup_www_fictiondb_com(my_db, my_cursor,authname,seriesname,firstname, lastname, log)
    #END FOR


    misc4_delete_web_table_duplicates(my_db, my_cursor, log)  #shared with misc4

    mysql = "SELECT Count(*) FROM _web_author_series"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log(" ")
    log("Number of Series Downloaded as WSSVD for Selected Author:", str(count1))
    log(" ")

    mysql = "SELECT Count(*) FROM _web_series_detail"
    my_cursor.execute(mysql)
    count1 = my_cursor.fetchall()
    count1 = __strip_numerics(count1)
    log(" ")
    log("Number of Series/Book Title Combinations Downloaded as WSSVD for Selected Author:", str(count1))
    log(" ")
    log(" ")
    log("This new WSSVD is now available to all Series Level Scrubbing functions.")
    log(" ")
    #-----------------------------------------------------------------------------------------------------------------
def misc14_unpack_soup_www_fictiondb_com(my_db, my_cursor, authname,seriesname, firstname, lastname, log):
    # for www.fictiondb.com only
    # my_url = str("http://www.fictiondb.com/series/author-series~[LETTER].htm")    #not really; just a placeholder.

    print("entering: misc14_unpack_soup_www_fictiondb_com")

    my_list1 = []
    results = []
    results1 = []
    results2 = []
    results3 = []
    results_final = []

    try:
        my_letter = str(lastname[0].lower())
    except:
        my_letter = str(lastname[0])

    my_author_link = str(str(firstname) + "-" + str(lastname))       #e.g.  rachel-caine
    try:
        my_author_link = str(my_author_link.lower())
    except:
        pass

    my_series_link = str(seriesname.replace(" ", "-", 20))            #e.g.  The-Morganville-Vampires
    try:
        my_series_link = str(my_series_link.lower())
    except:
        pass

    print("my_author_link: " + my_author_link)
    print("my_series_link: " + my_series_link)


    #<../author/amanda-cabot~series
    my_find_link = str("<../author/[FIRST-LAST]~series")
    my_find_link = str(my_find_link.replace("[FIRST-LAST]", my_author_link, 1))
    print(str(my_find_link))

    log(" ")

    my_url = str("http://www.fictiondb.com/series/author-series~[LETTER].htm")
    my_url = str(my_url.replace("[LETTER]", str(my_letter), 1))
    print(str(my_url))

    try:
        req = urllib2.Request(my_url)
        response = urllib2.urlopen(req)
        soup = BeautifulSoup(response)
        response.close()
    except Exception as e:
        print(str(e))
        return

    if not soup:
        print("no soup...")
        return
    else:
        print("soup was found")
        pass

    results = soup.tbody
    if not results:
        print("no results; returning")
        return

    n = len(results)
    print("results length:" + str(n))
    if n == 0:
        return
    print("now searching results to append to my_list1...")
    for row in results:
        s = str(row)
        if my_author_link in s:
            print("s: " + s)
            my_list1.append(str(s))
     #END FOR
    '''       now have (single line):
            <tr class="g"><td><span class="srtkey">Morganvill</span><a class="hghlt" href=".
            ./author/rachel-caine~series~the-morganville-vampires~9655.htm">The Morganville
            Vampires</a></td><td><a href="../author/rachel-caine~39124.htm">Caine, Rachel</a
            ></td></tr>
    '''


    n = len(my_list1)
    print("my_list1 length:" + str(n))
    if n == 0:
        print("no results in my_list1")
        return

    print("second part begins...............................................................")

    '''
                regex will search: <tr class="g"><td><span class="srtkey">Morganvill</span><a clas
                s="hghlt" href="../author/rachel-caine~series~the-morganville-vampires~9655.htm"
                >The Morganville Vampires</a></td><td><a href="../author/rachel-caine~39124.htm"
                >Caine, Rachel</a></td></tr>
    '''
    my_re_auth_series = str('href=["][.][.][/]author[/][a-z -]*[~]series[~][a-z -]*[SERIES-NAME][a-z -]*[~][0-9]+[.]htm["]')
    my_re_auth_series = str(my_re_auth_series.replace("[SERIES-NAME]", str(my_series_link), 1))
    print(str(my_re_auth_series))
    try:
        p = re.compile(my_re_auth_series, re.IGNORECASE)
    except:
        log("REGEX COMPILE ERROR in Parsing Web Page for: " + str(my_series_link) )
        log("REGEX COMPILE ERROR in misc4_unpack_soup_www_fictiondb_com for: "  + str(my_re_auth_series) )
        log("REGEX COMPILE ERROR: Please Forward This Log to the Developer.  Thank you.")
        return
    s_new_url = mynothing
    print("now searching my_list1")
    for row in my_list1:
        s = str(row)
        print("regex searching:", str(s))
        match1 = p.search(str(s))
        if not match1:
            print("not match1...continuing")
            continue
        else:
            print("yes match1")
            s_new_url = str(match1.group(0))
            print(str(s_new_url))
            break
    #END FOR

    if s_new_url == mynothing:
        #sys.exit("1 Terminated Purposefully")
        print("s_new_url == mynothing")
        return
    # now have:                href="../author/rachel-caine~series~the-morganville-vampires~9655.htm"
    s_new_url = str(s_new_url.replace('href="..', 'http://www.fictiondb.com', 1))
    # now have:                http://www.fictiondb.com/author/rachel-caine~series~the-morganville-vampires~9655.htm"
    s_new_url = str(s_new_url.replace('.htm"', '.htm', 1))
    # now have:                http://www.fictiondb.com/author/rachel-caine~series~the-morganville-vampires~9655.htm
    s_web_series_name = misc14_parse_web_series_name(s_new_url, log)
    print("Retrieved Web Series Name:     " + str(s_web_series_name))
    try:
        req = urllib2.Request(s_new_url)   #http://www.fictiondb.com/author/rachel-caine~series~the-morganville-vampires~9655.htm
        response = urllib2.urlopen(req)
        soup = BeautifulSoup(response)
        response.close()
    except Exception as e:
        print(str(e))
        return
    if not soup:
        print("no second soup...")
        return
    else:
        pass
        print("second soup was found")

    results2 = soup.tbody
    for row in results2:
        s = str(row)
        n = s.find("td>")     #<td>1</td> [...and...] <td><a itemprop="url" href="../author/rachel-caine~glass-houses~159550~b.htm"><span itemprop='name'>Glass Houses</span></a>
        if n >= 0:
            results3.append(str(s))
    #END FOR
    n = len(results3)
    print("results3 length:" + str(n))
    s_total_books = str(n)
    if n == 0:
        print("results3 has no rows")
        return
    '''   now have as a row:
        <td>14</td> <td><a itemprop="url" href="../author/rachel-caine~fall-of-night~398
        650~b.htm"><span itemprop="name">Fall of Night</span></a><span title=""><a class
        ="anth" href=""></a></span><span class="reissue" title="reissue"></span></td>
        <td class="nowrap"></td><td class=""><span class="visible-xs" itemprop="genre">U
        rban Fantasy</span><span class="hghlt hidden-xs" title="Urban Fantasy">UF</span>
        </td> (plus some junk)
    '''

    book_dict = {}

    my_re_index       = str("<td>[0-9]+</td>\s")
    my_re_booktitle = str("~.+[~][0-9]")

    try:
        p1 = re.compile(my_re_index, re.IGNORECASE)
        p2 = re.compile(my_re_booktitle, re.IGNORECASE)
    except:
        log("REGEX COMPILE ERROR in Parsing Web Page for: " + str(my_re_index) + " and: " + str(my_re_booktitle) )
        log("REGEX COMPILE ERROR in misc4_unpack_soup_www_fictiondb_com for: "  + str(s_new_url) )
        log("REGEX COMPILE ERROR: Please Forward This Log to the Developer.  Thank you.")
        return
    #now clean it up, and put index and booktitle into a dict
    tmp_index = "0"
    tmp_title = ""
    for row in results3:
        s = str(row)
        s = str(__change_tuple_to_bytestring(str(row)))
        print("Raw Row:", str(s))
        match1 = p1.search(str(s))
        match2 = p2.search(str(s))
        if match1:        #   <td>1</td>
            s = str(match1.group())
            s = str(s.replace("<td>", mynothing, 2))
            s = str(s.replace("</td>", mynothing, 1))
            tmp_index = str(s.strip())
            print("[match1]:", str(tmp_index), str(tmp_title))
        if match2 :       #    ~fall-of-night~3
            s = str(match2.group())
            s = str(s[1:-2])   #   fall-of-night
            s = str(s.replace("-", " ", 1))
            tmp_title = str(s.strip())
            tmp_title = str(tmp_title.replace("-", " ", 30))
            tmp_title = str(titlecase(tmp_title))
            book_dict[tmp_title] = str(tmp_index)
            print("[match2]:",str(tmp_index), str(tmp_title))
            tmp_index = "0"
            tmp_title = "NOTHINGTITLE"
        continue
    #END FOR


    n = len(book_dict)  #web books in series
    if n >= 60  or n == 0:
        log("==========================================================")
        log("ERROR[2].  Web URL May Have Changed (Or Not). Notify Developer If This Continues.")
        log("==========================================================")
        return

    authname = str(authname.title())
    tmp_auth = str(authname)

    seriesname = str(seriesname.title())
    s_web_series_name  = str(seriesname)

    my_url = str(my_url)   #used when inserting new rows into web tables

    orig_series = str(seriesname)   #fake work series

    if not s_total_books[0].isdigit():
        s_total_books = "0"
    for title,index in book_dict.iteritems():
        tmp_title = str(title)
        n = tmp_title.find("~")
        if n > 0:
            tmp_title = str(tmp_title[0:n])
            tmp_title = str(tmp_title.replace("~", mynothing, 1))
        tmp_title = str(tmp_title.replace("it's", "it is", 2))  #because booklevel scrubbing does this to avoid loss of the apostrophe due to utf8 to bytestring conversion
        tmp_title = str(titlecase(str(tmp_title)))  #titlecase to aid in comparison
        tmp_index = str(index)
        tmp_index = str(tmp_index.replace("&nbsp;", mynothing, 2))
        tmp_index = str(tmp_index.strip())
        if tmp_title > " ":
            if (("Search by This Series Name" or "&nbsp") in tmp_title) or  (not tmp_index[0].isdigit()):
                print("passing to avoid update for:" + str(tmp_index) + str(tmp_title))
                pass
            else:
                print(tmp_auth + " <> " + s_web_series_name +  " <> " + my_url +  " <> " + s_total_books)
                sleep(0.02)
                mysql = str('INSERT OR REPLACE INTO _web_author_series (authname,seriesname,source_url,book_count) VALUES (?,?,?,?) ')
                __execute_mysql_for_custom_column_generic_4_args(my_db, my_cursor, log, mysql, tmp_auth, s_web_series_name, my_url, s_total_books)
                sleep(0.02)
                mysql = "INSERT OR  REPLACE INTO _web_series_detail (authname, seriesname, booktitle, seriesindex, source_url) VALUES (?,?,?,?,?) "
                __execute_mysql_for_custom_column_generic_5_args(my_db, my_cursor, log, mysql, tmp_auth, s_web_series_name, tmp_title, tmp_index, my_url)
                sleep(0.02)
                #mysql = str('INSERT OR REPLACE INTO _web_series_rename_detail (work_series,work_author,web_series,web_author) VALUES (?,?,?,?) ')
                #__execute_mysql_for_custom_column_generic_4_args(my_db, my_cursor, log, mysql, orig_series, tmp_auth, s_web_series_name, tmp_auth)
                #sleep(0.05)
    #END FOR

    #mysql = str("INSERT OR REPLACE INTO _web_series_rename_detail_cumulative SELECT * FROM _web_series_rename_detail ")
    #__execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    #mysql = str("DELETE FROM _web_series_rename_detail WHERE rowid NOT NULL")
    #__execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)

    mysql = "DELETE FROM _web_series_rename_detail_cumulative WHERE work_series = web_series"
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.05)
    #-----------------------------------------------------------------------------------------------------------------
def misc14_parse_web_series_name(s_new_url, log):
    # now have:   http://www.fictiondb.com/author/rachel-caine~series~the-morganville-vampires~9655.htm

    #log("0 parsing:" + s_new_url)

    s_new_url = str(s_new_url)

    n = s_new_url.find("~series~")           #   ~series~scimitar-seas~
    n = n + 8
    s_web_series_name = str(s_new_url[n: ])
    #log("1:" + str(s_web_series_name))
    n = s_web_series_name.find("~")
    s_web_series_name = str(s_web_series_name[0:n])
    #log("2:" + str(s_web_series_name))
    s_web_series_name = str(s_web_series_name.replace("-", " ", 30))
    #log("3:" + str(s_web_series_name))
    s_web_series_name = str(s_web_series_name.strip())
    s_web_series_name = str(titlecase(s_web_series_name))

    #log("returning with:" + str(s_web_series_name))

    return str(s_web_series_name)
    #-----------------------------------------------------------------------------------------------------------------
def misc15_control(my_db, my_cursor, notifications, log):
    # Copy Work Tags to Custom Columns(s) Per Table "Tags CC Mapping Control" [Selected Books]

    # Rule T1:  use the Work Tag in the table as the Value for the custom column, and leave the Work Tag alone
    # Rule T2:  use the Work Tag in the table as the Value for the custom column, but then delete the Work Tag
    # Rule V1:  use the Value in the table as the Value for the custom column, and leave the Work Tag alone
    # Rule V2:  use the Value in the table as the Value for the custom column, but then delete the Work Tag

    #__build_regex_list_from_tag_capitalization_rules(my_db, my_cursor, log)

    # do NOT change any capitalization of any tags at all in misc15_control; should have been scrubbed already using normal work tag scrubbing jobs

    global my_selected_book_list

    mysql = "SELECT tag,cc_number,datatype,rule,cc_value FROM _tag_cc_mapping_control WHERE active = '1'  ORDER BY rule DESC, tag ASC, cc_number ASC ;"
    rules_list = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
    if not rules_list:
        log("No Active Rules in Table _tag_cc_mapping_control.  ")
        my_db.close()
        return
    else:
        n = len(rules_list)
        if n == 0:
            log("No Active Rules in Table _tag_cc_mapping_control.  ")
            my_db.close()
            return

    __explode_custom_column_13_if_needed(my_db, my_cursor, log)
    sleep(0.02)

    log(" ")
    log("Table _tags_cc_mapping_control entries to be used by this job in the sort sequence shown (by rule, descending, then all else, ascending): ")

    for row in rules_list:
        tag,cc_number,datatype,rule,cc_value = row
        if not cc_value:
            cc_value = "NULL"
        log(tag,cc_number,datatype,rule,cc_value)
    #END FOR
    log(" ")
    log(" ")
    log("Note:  if your table rules have the incorrect Custom Column Datatype for the specified Custom Column Number, this job will fail with odd error messages.")
    log(" ")
    log(" ")

    my_selected_book_list.sort()

    log("Selected books that have Work Tags: ")
    tagsall_list = []
    for row in my_selected_book_list:
        book = str(row)
        mysql = "SELECT book,tagsall FROM __books_work_populate WHERE book = ? AND tagsall NOT NULL AND tagsall > ' '  "
        my_cursor.execute(mysql,([book]))
        tmp_list = my_cursor.fetchall()
        if not tmp_list:
            continue
        else:
            n = len(tmp_list)
            if n == 0:
                continue
            else:
                for item in tmp_list:
                    tagsall_list.append(item)
                    book,tagsall = item
                    log(str(book),tagsall)
    #END FOR
    log(" ")
    log(" ")

    if len(tagsall_list) == 0:
        log("No Work Tags Exist for Selected Books")
        my_db.close()
        return

    n = len(tagsall_list)
    log("Number of books to be further processed: " + str(n))
    log(" ")

    tagsall_list.sort()

    new_tagsall_dict = {}   # key = book     value = new tagsall   only if new tagsall <> old tagsall

    deleted_tags = []
    for row in tagsall_list:   # 1 unique book per row
        #log("----------------------------------------------------------------------")
        book,tagsall = row
        new_tagsall = ""
        tagsall = tagsall + ", DUMMY"    #guarantees a split is possible
        tag_list = tagsall.split(",")
        tag_list.sort()
        rules_found = False
        del deleted_tags
        deleted_tags = []
        for tag in tag_list:
            tag = tag.strip()
            if tag == "DUMMY":
                continue
            else:
                try:
                    l_tag = tag.lower()
                except:
                    pass
                for item in rules_list:        #a single work tag can have multiple rules...
                    tabletag,cc_number,datatype,rule,cc_value = item
                    try:
                        l_tabletag = tabletag.lower()
                    except:
                        pass
                    if l_tag == l_tabletag:
                        rules_found = True
                        new_tag = misc15_custom_column_control(my_db, my_cursor, notifications, log,book,item,tag)
                        if new_tag <> "DELETE" and new_tag not in deleted_tags:
                            new_tag = new_tag.strip()
                            #log("new_tag kept: " + new_tag)
                            new_tagsall = new_tagsall + ", " + new_tag
                        else:
                            if new_tag == "DELETE":    #so original tag must be deleted...
                                #log("new_tag added to deleted_tags: " + tag)
                                tag = tag.strip()
                                deleted_tags.append(tag)   # multiple rules for same tag can cause the first rule to delete the tag, but the next one to not delete it implicitly
                    else:
                        if tag not in deleted_tags and tag > " ":
                            new_tagsall = new_tagsall + ", " + tag
                        else:
                            pass
                #END FOR
        #END FOR
        if not rules_found:
            continue
        new_tagsall = new_tagsall.strip()
        if new_tagsall == "" or new_tagsall == "," :
            new_tagsall = "DELETETAGSALL"
        new_tagsall = misc15_finalize_book_tagsall(new_tagsall, deleted_tags)
        if new_tagsall <> tagsall:
            new_tagsall_dict[book] = new_tagsall
    #END FOR

    if len(new_tagsall_dict) > 0:
        misc15_create_final_tagsall(my_db, my_cursor, notifications, log,new_tagsall_dict)

    my_db.close()
    #-----------------------------------------------------------------------------------------------------------------
def misc15_finalize_book_tagsall(new_tagsall,deleted_tags):

    #global tag_capitalization_regex_rules

    new_tagsall = new_tagsall + ", DUMMY"    #guarantees a split is possible
    tag_list = new_tagsall.split(",")
    tag_list.sort()
    final_tagsall = ""
    for tag in tag_list:
        tag = tag.strip()
        if tag == "DUMMY":
            continue
        else:
            if tag in deleted_tags:
                continue
            else:
                if tag > " ":
                    #tag = titlecase(tag)
                    #apply_tag_capitalization_rules(tag,log)
                    tag = tag.strip()
                    final_tagsall = final_tagsall + ", " + tag
    #END FOR
    new_tagsall = final_tagsall
    new_tagsall = new_tagsall.strip()
    if new_tagsall.startswith(","):
        new_tagsall = new_tagsall[1: ]
    if new_tagsall.endswith(","):
        new_tagsall = new_tagsall[0:-1 ]
    new_tagsall = new_tagsall.strip()

    return new_tagsall
    #-----------------------------------------------------------------------------------------------------------------
def misc15_custom_column_control(my_db, my_cursor, notifications, log,book,item,tag):
    # Rule T1:  use the Work Tag in the table as the Value for the custom column, and leave the Work Tag alone
    # Rule T2:  use the Work Tag in the table as the Value for the custom column, but then delete the Work Tag
    # Rule V1:  use the Value in the table as the Value for the custom column, and leave the Work Tag alone
    # Rule V2:  use the Value in the table as the Value for the custom column, but then delete the Work Tag

    t1 = "T1"
    t2 = "T2"
    v1 = "V1"
    v2 = "V2"

    tabletag,cc_number,datatype,rule,cc_value = item

    rule = rule.upper()

    if rule == t2 or rule == v2:
        new_tag = "DELETE"
    else:
        new_tag = tag

    if rule == t1 or rule == t2:
        cc_value = tabletag    # which == tag
    else:
        pass

    if cc_number > 19 and cc_value > " ":
        misc15_update_non_qs_custom_columns_generic(my_db, my_cursor, log,cc_number,datatype,cc_value,book)

    return new_tag
    #-----------------------------------------------------------------------------------------------------------------
def misc15_create_final_tagsall(my_db, my_cursor, notifications, log,new_tagsall_dict):
    #duplicates can occur if there are multiple rules for the same tag, so split, make unique, then rebuild tagsall...
    log(" ")
    for k,v in new_tagsall_dict.iteritems():
        book = k
        tagsall = v
        if not tagsall > " ":
            tagsall = "DELETETAGSALL"
        tagsall = tagsall + ", DUMMY"
        #log("k,v:" + str(book) + "=" + tagsall)
        tag_list = tagsall.split(", ")   #space important
        tag_set = set(tag_list)
        tag_list = list(tag_set)
        del tag_set
        tag_list.sort()
        final_tagsall = ""
        for tag in tag_list:
            tag = tag.strip()
            if tag == "DUMMY":
                continue
            else:
                if tag > " ":
                    final_tagsall = final_tagsall + ", " + tag
        #END FOR
        final_tagsall = final_tagsall.strip()
        if final_tagsall.startswith(","):
            final_tagsall = final_tagsall[1: ]
        if final_tagsall.endswith(","):
            final_tagsall = final_tagsall[0:-1 ]
        final_tagsall = final_tagsall.strip()
        misc15_update_final_tagsall(my_db, my_cursor, notifications, log,final_tagsall,book)
    #END FOR
#----------------------------------------------------------------------------------------------------------------
def misc15_update_final_tagsall(my_db, my_cursor, notifications, log,final_tagsall,book):
    #cc13 was previously exploded

    if final_tagsall <> "DELETETAGSALL":
        log("Final Work Tags for Book: " + str(book) + " - " + final_tagsall)

        mysql = "INSERT OR REPLACE INTO custom_column_13 (id,value) VALUES(?,?) "
        __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, book, final_tagsall)

        mysql = "INSERT OR REPLACE INTO books_custom_column_13_link VALUES(?,?,?) "
        __execute_mysql_for_custom_column_generic_3_args(my_db, my_cursor, log, mysql, book, book, book)
    else:
        log("No Remaining Work Tags Now Exist for Book: " + str(book))
        mysql = "DELETE FROM books_custom_column_13_link WHERE book = ? AND book = ?"
        __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, book, book)
#----------------------------------------------------------------------------------------------------------------
def misc15_update_non_qs_custom_columns_generic(my_db, my_cursor, log,cc_number,datatype,new_cc_value,book):
    #copied from booklevel.py
    global mynothing

    text_cc_table = str("custom_column_ZZ")
    text_cc_link_table = str("books_custom_column_ZZ_link")
    comment_cc_table = str("custom_column_ZZ")

    #log("new_cc_value is: " + new_cc_value)
    #log("datatype is: " + datatype)
    #log("cc_number is: " + str(cc_number))

    if datatype <> 'text' and datatype <> 'comments' :
        log("ERROR: table _title_cc_control row has a datatype value error: " + str(datatype))
        return
    else:
        if datatype == "text":
            #log("datatype is: " + datatype)
            text_cc_table = str(text_cc_table.replace("ZZ", str(cc_number), 1))
            text_cc_link_table = str(text_cc_link_table.replace("ZZ", str(cc_number), 1))
            normalized = True
        else:
            #log("datatype is: " + datatype)
            comment_cc_table = str(comment_cc_table.replace("ZZ", str(cc_number), 1))
            normalized = False

    #log("normalized is: " + str(normalized))
    sleep(0.02)
    if not normalized:
        #log("not normalized...")
        mysql = str("DELETE FROM  [TABLE]  WHERE book = [BOOK]  ")
        mysql = str(mysql.replace("[TABLE]", str(comment_cc_table),1))
        mysql = str(mysql.replace("[BOOK]", str(book),1))
        #log(str(mysql))
        __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
        sleep(0.02)
        mysql = str("INSERT OR REPLACE INTO  [TABLE]  (id,book,value) VALUES (?,?,?) ")
        mysql = str(mysql.replace("[TABLE]", str(comment_cc_table),1))
        #log(str(mysql))
        __execute_mysql_for_custom_column_generic_3_args(my_db, my_cursor, log, mysql, book, book, new_cc_value)
        return
    if normalized:
        #log("normalized")
        mysql = "SELECT id FROM [TABLE]  WHERE value = ?"
        mysql = str(mysql.replace("[TABLE]", str(text_cc_table),1))
        #log(str(mysql))
        tmp_rows = []
        del tmp_rows
        my_cursor.execute(mysql, (new_cc_value,))
        tmp_rows = my_cursor.fetchall()  #will get all of the rows from the query
        if not tmp_rows:
            id = "0"
        else:
            if len(tmp_rows) == 0:
                id = "0"
            else:
                for row in tmp_rows:
                    for col in row:
                        id = col
                        #log("id is: " + str(id))

        if str(id) == "0":      #value does not yet exist in base cc table...
            #log("id is 0")
            mysql = str("INSERT OR REPLACE INTO  [TABLE]  (id,value) VALUES (?,?) ")
            mysql = str(mysql.replace("[TABLE]", str(text_cc_table),1))
            #log("[1]" + str(mysql))
            my_null = None
            __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, my_null, new_cc_value)
            sleep(0.02)
        mysql = str("DELETE FROM  [TABLE]  WHERE book = [BOOK]  ")
        mysql = str(mysql.replace("[TABLE]", str(text_cc_link_table),1))
        mysql = str(mysql.replace("[BOOK]", str(book),1))
        #log("[2]" + str(mysql))
        __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
        sleep(0.02)
        mysql = "SELECT id FROM [TABLE]  WHERE value = ?"
        mysql = str(mysql.replace("[TABLE]", str(text_cc_table),1))
        #log(str(mysql))
        tmp_rows = []
        del tmp_rows
        my_cursor.execute(mysql, (new_cc_value,))
        tmp_rows = my_cursor.fetchall()  #will get all of the rows from the query
        if not tmp_rows:
            id = "0"
        else:
            if len(tmp_rows) == 0:
                id = "0"
            else:
                for row in tmp_rows:
                    for col in row:
                        id = col
                        #log("id is: " + str(id))

        if str(id) == "0":
            log("ERROR: base custom table value does not exist although it was just added; continuing.")
            log("Book: " + str(book) + "  Table: " + str(text_cc_table))
            return
        else:
            mysql = str("INSERT OR REPLACE INTO [LINKTABLE]  (id,book,value) VALUES ( null,?,?) ")
        mysql = str(mysql.replace("[LINKTABLE]", str(text_cc_link_table),1))
        #log(str(mysql))
        __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, book, id)
        sleep(0.02)
        mysql = str("DELETE FROM  [TABLE]  WHERE id NOT IN(SELECT value FROM [LINKTABLE] WHERE book NOT NULL )")         #delete orphans, if any
        mysql = str(mysql.replace("[TABLE]", str(text_cc_table),1))
        mysql = str(mysql.replace("[LINKTABLE]", str(text_cc_link_table),1))
        #log(str(mysql))
        __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
        return
#----------------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------------
def misc16_control(my_db, my_cursor, notifications, log):
    # Option 16: Update Pseudonym Custom Column [Selected Books]

    global my_selected_book_list
    global mynothing

    # Determine what the custom column Pseudonyms has as a physical field name
    from calibre_plugins.quarantine_and_scrub.config import prefs
    custom_field = prefs['pseudonyms']
    if not custom_field:
        custom_field = "none"
    custom_field = custom_field.strip()
    if custom_field == "none" or custom_field == "#none" or custom_field == "#" or (not custom_field > ' '):
        log("Error: No Custom Column Exists for Configured Pseudonyms: " + custom_field)
        return

    log(" ")
    log("-----------------------------------------------------------------------------------------------")
    log(" ")
    log("Q&S Configured Pseudonym Custom Column #Name to be used is:  " + str(custom_field))
    log(" ")
    log("-----------------------------------------------------------------------------------------------")
    log(" ")
    log(" ")
    log("This job uses Real Author, not Work Author.")
    log(" ")
    log(" ")

    custom_field = str(custom_field.replace("#", mynothing, 1))

    mysql = str("SELECT id,datatype,display,is_multiple,normalized FROM custom_columns WHERE label = '[pseudonyms]' ")
    mysql = str(mysql.replace("[pseudonyms]", str(custom_field)))
    my_cursor.execute (mysql)
    rows = my_cursor.fetchall()
    if not rows:
        log("ERROR: Configured Pseudonym Custom Column does NOT Exist")
        return
    if len(rows) == 0:
        log("ERROR: Configured Pseudonym Custom Column does NOT Exist")
        return
    for row in rows:
        custom_id,datatype,display,is_multiple,normalized = row
        #~ log(" ")
        #~ log("Custom Column Table Entry: " + str(row))
        #~ log(" ")

    #"28","pseudonyms","Pseudonyms","text","0","1","{""description"": ""Author Pseudonyms"", ""is_names"": true}","1","1"

    if datatype <> "text" or normalized <> 1:
        log(" ")
        log("ERROR: Datatype of Pseudonym Custom Column is NOT 'text'")
        log(" ")
        return
    if is_multiple <> 1:
        log(" ")
        log("ERROR: Pseudonym Custom Column was NOT configured as 'People' ")
        log(" ")
        return
    if not '"is_names": true' in display:
        log(" ")
        log("ERROR: Pseudonym Custom Column was NOT configured as 'People' ")
        log(" ")
        return
    if custom_id < 20:
        log(" ")
        log("ERROR: Pseudonym Custom Column ID Is Invalid.  It must be >= 20' ")
        log(" ")
        return

    custom_id = str(custom_id)

    custom_table = str("custom_column_X")
    custom_table = str(custom_table.replace("X",custom_id,1))

    books_custom_column_link = str("books_custom_column_X_link")
    books_custom_column_link = str(books_custom_column_link.replace("X",custom_id,1))

    #~ log(" ")
    #~ log("custom column number: " + custom_id)
    #~ log("custom column table: " + custom_table)
    #~ log("books custom column link table: " + books_custom_column_link)
    #~ log(" ")

    my_selected_book_list.sort()

    for row in my_selected_book_list:
        book = str(row)
        mysql = "SELECT pseudonym,author FROM _global_pseudonyms \
                            WHERE author IN(SELECT name FROM __book_author_name_sort WHERE book = ? ) ORDER BY pseudonym "
        my_cursor.execute(mysql,([book]))
        tmp_rows = my_cursor.fetchall()
        new_value = ""
        for item in tmp_rows:
            pseudonym,author = item
            new_value = new_value + ", " + pseudonym
        #END FOR
        new_value = new_value.strip()
        if new_value.startswith(","):
            new_value = new_value[1: ]
        if new_value.endswith(","):
            new_value = new_value[0:-1]
        new_value = new_value.strip()
        if new_value > " ":
            log("New Pseudonym(s) for Author and Book: " + author + " - " + str(book) + " - " + new_value)
            misc16_update_pseudonyms_custom_column(my_db, my_cursor, notifications, log,custom_table, books_custom_column_link, book, new_value)
    #END FOR
    #-----------------------------------------------------------------------------------------------------------------
def misc16_update_pseudonyms_custom_column(my_db, my_cursor, notifications, log,custom_table, books_custom_column_link, book, new_value):

    global mynothing

    mysql = "SELECT id FROM [TABLE]  WHERE value = ?"
    mysql = str(mysql.replace("[TABLE]", str(custom_table),1))
    tmp_rows = []
    del tmp_rows
    my_cursor.execute(mysql,(new_value,))
    tmp_rows = my_cursor.fetchall()  #will get all of the rows from the query
    if not tmp_rows:
        id = "0"
    else:
        if len(tmp_rows) == 0:
            id = "0"
        else:
            for row in tmp_rows:
                for col in row:
                    id = col
                    #log("id is: " + str(id))

    if str(id) == "0":      #value does not yet exist in base cc table...
        mysql = str("INSERT OR REPLACE INTO  [TABLE]  (id,value) VALUES (NULL,?) ")
        mysql = str(mysql.replace("[TABLE]", str(custom_table),1))
        __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, [new_value], mynothing)
        sleep(0.02)
    mysql = str("DELETE FROM  [TABLE]  WHERE book = [BOOK]  ")
    mysql = str(mysql.replace("[TABLE]", str(books_custom_column_link),1))
    mysql = str(mysql.replace("[BOOK]", str(book),1))
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    sleep(0.02)
    mysql = "SELECT id FROM [TABLE]  WHERE value = ?"
    mysql = str(mysql.replace("[TABLE]", str(custom_table),1))
    tmp_rows = []
    del tmp_rows
    my_cursor.execute(mysql, (new_value,))
    tmp_rows = my_cursor.fetchall()  #will get all of the rows from the query
    if not tmp_rows:
        id = "0"
    else:
        if len(tmp_rows) == 0:
            id = "0"
        else:
            for row in tmp_rows:
                for col in row:
                    id = col
                    #log("id is: " + str(id))

    if str(id) == "0":
        log("ERROR: base custom table value does not exist although it was just added; continuing.")
        log("Book: " + str(book) + "  Table: " + str(custom_table))
        return

    mysql = str("INSERT OR REPLACE INTO [LINKTABLE]  (id,book,value) VALUES ( null,?,?) ")
    mysql = str(mysql.replace("[LINKTABLE]", str(books_custom_column_link),1))
    __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, book, id)
    sleep(0.02)
    mysql = str("DELETE FROM  [TABLE]  WHERE id NOT IN(SELECT value FROM [LINKTABLE] WHERE book NOT NULL )")         #delete orphans, if any
    mysql = str(mysql.replace("[TABLE]", str(custom_table),1))
    mysql = str(mysql.replace("[LINKTABLE]", str(books_custom_column_link),1))
    __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql)
    #-----------------------------------------------------------------------------------------------------------------
    #-----------------------------------------------------------------------------------------------------------------
    #-----------------------------------------------------------------------------------------------------------------
def __similar(a,b):
    #returns the probability the string a is the same as string b

    return SequenceMatcher(None, a, b).ratio()
    #-----------------------------------------------------------------------------------------------------------------
def __printsafe(*arg):

    try:
        print(arg)
        pass
    except:
        pass
        print("...cannot be printed...")
    #-----------------------------------------------------------------------------------------------------------------
    #-----------------------------------------------------------------------------------------------------------------
    #-----------------------------------------------------------------------------------------------------------------
    #--------------------------------------------------------------------------------------------------------------------------------------------------------------
    #----------------------------------------------------------------------------------------------------------------------------------------------
def __count_global_series(my_db, my_cursor, log):

    log("===================================")
    log("For Your Reference Only")
    log("===================================")

    mysql = "SELECT Count(*) FROM _global_series"
    my_cursor.execute(mysql)
    count2 = my_cursor.fetchall()
    count2 = __strip_numerics(count2)
    __printsafe("Number of Global Series:", str(count2))
    log("Number of Global Series:    ", str(count2))

    mysql = "SELECT Count(*) FROM _pristine_series"
    my_cursor.execute(mysql)
    count4 = my_cursor.fetchall()
    count4 = __strip_numerics(count4)
    __printsafe("Number of Pristine Series:", str(count4))
    log("Number of Pristine Series:  ", str(count4))

    log("===================================")
    log("===================================")
  #----------------------------------------------------------------------------------------------------------------------------------------------
def __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql):

    sleep(0.02) #avoid db locks

    tmp_rows = []
    del tmp_rows
    tmp_rows = []

    mysql = str(mysql)
    try:
        my_cursor.execute(mysql)
        tmp_rows = my_cursor.fetchall() #will get all of the rows from the query
        if tmp_rows:
            return tmp_rows
        else:
            tmp_rows = []
            return tmp_rows
    except:
        #likely due to database lock; sleep then try again...
        __printsafe("database LOCK error; sleeping and will then try again")
        sleep(5.0)
        tmp_rows = []
        del tmp_rows
        tmp_rows = []
        try:
            my_db, my_cursor = __reset_cursor(my_db,log)
            my_cursor.execute(mysql)
            tmp_rows = my_cursor.fetchall() #will get all of the rows from the query
            if tmp_rows:
                return tmp_rows
            else:
                tmp_rows = []
                return tmp_rows
        except Exception as e:
            log(str(e))
            __printsafe(str(e))
            __printsafe(mysql)
            __printsafe("__execute_mysql_fetchall_generic:   try-except failure")
            my_db.close()
            __printsafe("database has been CLOSED")
            raise e
            sys.exit("ERROR in __execute_mysql_fetchall_generic....")
    #---------------------------------------------------------------------------------------------------------------------------------------------
def __execute_mysql_with_commit_generic(my_db, my_cursor, log, mysql):


    mysql = str(mysql)
    try:
        #my_cursor.execute ("BEGIN TRANSACTION")
        my_cursor.execute("begin") #apsw
        my_cursor.execute(mysql)  #apsw
        my_cursor.execute("commit") #apsw
        #my_cursor.execute (mysql)
        #my_cursor.execute ("commit")
    except:
        __printsafe("database locked; sleeping and will retry")
        sleep(5.0)
        try:
            my_db, my_cursor = __reset_cursor(my_db,log)
            my_cursor.execute("begin") #apsw  will already have been executed
            my_cursor.execute(mysql)  #apsw
            my_cursor.execute("commit") #apsw
        except Exception as e:
            __printsafe(str(mysql))
            log(str(e))
            __printsafe(str(e))
            my_db.close()
            __printsafe("database has been CLOSED")
            raise e
            sys.exit("ERROR in __execute_mysql_with_commit_generic....")

    sleep(0.03) #avoid db locks
#---------------------------------------------------------------------------------------------------------------------------------------------
def __execute_mysql_for_custom_column_generic_1_arg(my_db, my_cursor, log, mysql, v1):

    global mynothing

    try:
        my_cursor.execute("begin")  #apsw
        my_cursor.execute(mysql,(v1))  #apsw
        my_cursor.execute("commit")  #apsw
    except:
        __printsafe("database locked; sleeping and then will retry")
        log("database locked; sleeping and then will retry")
        sleep(10.0)
        try:
            my_db, my_cursor = __reset_cursor(my_db,log)
            my_cursor.execute("begin") #apsw
            my_cursor.execute(mysql,(v1))  #apsw
            my_cursor.execute("commit")  #apsw
        except Exception as e:
            log(str(e))
            __printsafe(str(e))
            my_db.close()
            log("database has been CLOSED")
            raise e
            sys.exit("ERROR in __execute_mysql_for_custom_column_generic_1_arg")

    sleep(0.05)  #avoid db locks
#---------------------------------------------------------------------------------------------------------------------------------------------
def __execute_mysql_for_custom_column_generic(my_db, my_cursor, log, mysql, v1, v2):

    global mynothing


    s2 = str(v2)  #second parameter is optional
    s2 = str(__change_tuple_to_bytestring(s2))

    if s2 == mynothing:
        s2 = str("IGNORE")

    try:
        my_cursor.execute("begin") #apsw
        if s2 <> "IGNORE":
            my_cursor.execute(mysql,(v1,v2))  #apsw
        else:
            my_cursor.execute(mysql,(v1))  #apsw
        my_cursor.execute("commit") #apsw
    except:
        __printsafe("database locked; sleeping and then will retry")
        log("database locked; sleeping and then will retry")
        sleep(5.0)
        try:
            my_db, my_cursor = __reset_cursor(my_db,log)
            my_cursor.execute("begin") #apsw  will already have been executed
            if s2 <> "IGNORE":
                my_cursor.execute(mysql,(v1,v2))  #apsw
            else:
                my_cursor.execute(mysql,(v1))  #apsw
            my_cursor.execute("commit") #apsw
        except Exception as e:
            __printsafe(str(mysql))
            __printsafe("v1 is: ", str(v1), " v2 is: ", str(v2))
            log(str(e))
            __printsafe(str(e))
            my_db.close()
            __printsafe("database has been CLOSED")
            raise e
            sys.exit("ERROR in __execute_mysql_for_custom_column_generic")

    sleep(0.03) #avoid db locks
#---------------------------------------------------------------------------------------------------------------------------------------------
def __execute_mysql_for_custom_column_generic_3_args(my_db, my_cursor, log, mysql, v1, v2, v3):

    global mynothing


    try:
        my_cursor.execute("begin") #apsw
        my_cursor.execute(mysql,(v1,v2,v3))  #apsw
        my_cursor.execute("commit") #apsw
    except:
        __printsafe("database locked; sleeping and then will retry")
        log("database locked; sleeping and then will retry")
        sleep(5.0)
        try:
            my_db, my_cursor = __reset_cursor(my_db,log)
            my_cursor.execute("begin") #apsw
            my_cursor.execute(mysql,(v1,v2,v3))  #apsw
            my_cursor.execute("commit") #apsw
        except Exception as e:
            __printsafe(str(mysql))
            __printsafe(v1,v2,v3)
            log(str(e))
            __printsafe(str(e))
            my_db.close()
            __printsafe("database has been CLOSED")
            raise e
            sys.exit("ERROR in __execute_mysql_for_custom_column_generic_3_args")

    sleep(0.03) #avoid db locks
#---------------------------------------------------------------------------------------------------------------------------------------------
def __execute_mysql_for_custom_column_generic_4_args(my_db, my_cursor, log, mysql, v1, v2, v3, v4):

    global mynothing


    try:
        my_cursor.execute("begin")   #apsw
        my_cursor.execute(mysql,(v1,v2,v3,v4))   #apsw
        my_cursor.execute("commit")   #apsw
    except:
        __printsafe("database locked; sleeping and then will retry")
        log("database locked; sleeping and then will retry")
        sleep(5.0)
        try:
            my_db, my_cursor = __reset_cursor(my_db,log)
            my_cursor.execute("begin")   #apsw
            my_cursor.execute(mysql,(v1,v2,v3,v4))   #apsw
            my_cursor.execute("commit")   #apsw
        except Exception as e:
            __printsafe(v1,v2,v3,v4)
            __printsafe(str(mysql))
            log(str(e))
            __printsafe(str(e))
            my_db.close()
            __printsafe("database has been CLOSED")
            raise e
            sys.exit("ERROR in __execute_mysql_for_custom_column_generic_4_args")

    sleep(0.03)   #avoid db locks
#---------------------------------------------------------------------------------------------------------------------------------------------
def __execute_mysql_for_custom_column_generic_5_args(my_db, my_cursor, log, mysql, v1, v2, v3, v4, v5):

    global mynothing

    try:
        my_cursor.execute("begin")   #apsw
        my_cursor.execute(mysql,(v1,v2,v3,v4, v5))   #apsw
        my_cursor.execute("commit")   #apsw
    except:
        __printsafe("database locked; sleeping and then will retry")
        log("database locked; sleeping and then will retry")
        sleep(5.0)
        try:
            my_db, my_cursor = __reset_cursor(my_db,log)
            my_cursor.execute("begin")   #apsw
            my_cursor.execute(mysql,(v1,v2,v3,v4, v5))   #apsw
            my_cursor.execute("commit")   #apsw
        except Exception as e:
            __printsafe(v1,v2,v3,v4, v5)
            __printsafe(str(mysql))
            log(str(e))
            __printsafe(str(e))
            my_db.close()
            __printsafe("database has been CLOSED")
            raise e
            sys.exit("ERROR in __execute_mysql_for_custom_column_generic_4_args")

    sleep(0.03)   #avoid db locks
#----------------------------------------------------------------------------------------------------------------
def __strip_numerics(s):

    global mynothing

    s = str(s)

    s = __change_tuple_to_bytestring(s)

    s = s.replace(',', mynothing, 4)
    s = s.replace("(", mynothing, 4)
    s = s.replace(")", mynothing, 4)
    s = s.replace(" ", mynothing, 10)
    s = s.strip()

    return s
#---------------------------------------------------------------------------------------------------------------------------------------------
def __change_tuple_to_bytestring(tuple):
    #do not ever remove spaces here;  for commas,  only remove unicode special character combinations that may contain a comma.

    #example:   (u'Microsoft Word - New Microsoft Office Word Document',)
    #example:   (u"Saurabh's Account",)
    #example:   (u"Saurabh's Account", u"Account, Saurabh's")  <<=== keep the possessive 's
    #example:   '(199)' should be simply  '199'  in mysql
    #example:   (199) should be simply 199
    #example:   Sarah Miles'  should be simply  Sarah Miles
    #example:   Matt Scudder)  should be simply Matt Scudder
    #example:  (u'Hope Over Fear (Over #1)',)) should be simply  Hope Over Fear (Over #1)
    #example:   u"The Wizard King: Heart's Desire Book 3") should be simply   The Wizard King: Heart's Desire Book 3

    #but:  do NOT break this:  Follow the Queen: A Menage, Sexy and Short Romance)  <<< trailing ) that is not matched at all on the left.

    global mynothing

    s_string = str(tuple)

    s_string = s_string.replace('(u"', mynothing, 100)
    s_string = s_string.replace("(u'", mynothing, 100)  #example: (u'Hope Over Fear (Over #1)',)) becomes Hope Over Fear (Over #1)',))

    s_string = s_string.replace("',))", mynothing, 100)  #example: Hope Over Fear (Over #1)',)) becomes Hope Over Fear (Over #1)
    s_string = s_string.replace("',)", mynothing, 100)  #example:   (u"Saurabh's Account",)
    s_string = s_string.replace('",)', mynothing, 100)

    s_string = s_string.replace('u"', mynothing, 100)   #example:   (u"Saurabh's Account", u"Account, Saurabh's")  <<=== keep the possessive 's
    s_string = s_string.replace("u'", mynothing, 100)

    s_string = s_string.replace("'(", "'", 100)  #example:   '(199)' should be simply  '199'  in mysql
    s_string = s_string.replace(")'", "'", 100)

    s_string = s_string.replace('u"', mynothing, 100)  #example:   u"The Wizard King: Heart's Desire Book 3")
    s_string = s_string.replace('")', mynothing, 100)

    #return if string is good at this point.  example:  Hope Over Fear (Over #1)
    #__printsafe("in strip 1: ", s_string)
    s_string = str(s_string)
    n1 = s_string.find("(")  #left
    n2 = s_string.find(")")  #right
    __printsafe("left:", str(n1), " right:", str(n2))
    if n1 >= 0:
        if n2 >= 0 and n2 > n1:   #example:  Hope Over Fear (Over #1)     ... matched set ...
            #__printsafe("in strip 2 returning: ", s_string)
            return s_string
        else:  #example:  Hope Over Fear ) (Over #1                                        ... not a matched set ...
            pass
    n1 = s_string.count("(")  #left
    n2 = s_string.count(")")  #right
    __printsafe("left:", str(n1), " right:", str(n2))
    if n2 == 2 and n1 == 1:   # don't break:  Follow the Queen: A Menage, Sexy and Short Romance)   within the unicode    ('xxxxxxxxxxxxxxxx)',)
        __printsafe(str(s_string))
        return s_string  #otherwise, the following logic would strip  e) leaving Romanc
    if n2 == 1 and n1 == 0:   # don't break:  Follow the Queen: A Menage, Sexy and Short Romance)   within the unicode    ('xxxxxxxxxxxxxxxx)',)
        __printsafe(str(s_string))
        return s_string  #otherwise, the following logic would strip  e) leaving Romanc

    #__printsafe("in strip 3 NOT returning: ", s_string)

    n = s_string.find("(")
    if n == 0: #example:  is (199)
        s_string = s_string.replace("(", mynothing, 1)  #now is 199)
        c = s_string.count(")")  #example:  is 199)
        if c == 1:
            s_string = s_string.replace(")", mynothing, 1)  #example:  now 199
        else:
            pass

    #must do each of these twice
    n = s_string.rfind("'")  #s_string =  Sarah Miles', so n == 11
    l = len(s_string)  #l = 12
    if n == (l - 1):  # 11 = 12 - 1, so True
        z = (n - 1)  # 11 - 1 = 10
        s_string = s_string[0:z]  # now  Sarah Miles not Sarah Miles'
    else:
        pass

    n = s_string.rfind(")")  #s_string =  Matt Scudder) , so n == 13
    l = len(s_string)  #l = 14
    if n == (l - 1):  # 13 = 14 - 1, so True
        z = (n - 1)  # 11 - 1 = 10
        s_string = s_string[0:z]  # now  Matt Scudder not Matt Scudder)
        pass

    #must do each of these twice
    n = s_string.rfind("'")  #s_string =  Sarah Miles', so n == 11
    l = len(s_string)  #l = 12
    if n == (l - 1):  # 11 = 12 - 1, so True
        z = (n - 1)  # 11 - 1 = 10
        s_string = s_string[0:z]  # now  Sarah Miles not Sarah Miles'
    else:
        pass

    n = s_string.rfind(")")  #s_string =  Matt Scudder) , so n == 13
    l = len(s_string)  #l = 14
    if n == (l - 1):  # 13 = 14 - 1, so True
        z = (n - 1)  # 11 - 1 = 10
        s_string = s_string[0:z]  # now  Matt Scudder not Matt Scudder)
        pass

    s_string = s_string.strip()

    if isinstance(s_string, unicode):
        s_string = str(s_string)
        if isinstance(s_string, unicode):
            pass
            __printsafe("s_string is still a unicode object! [2]", s_string)

    return s_string
#---------------------------------------------------------------------------------------------------------------------------------------------
def __reset_cursor(my_db,log):

    global my_guidb

    my_db.close()

    db = my_guidb
    path = db.library_path
    __printsafe(path)
    if isbytestring(path):
        path = path.decode(filesystem_encoding)
    path = path.replace(os.sep, '/')
    path = os.path.join(path, 'metadata.db')
    path = path.replace(os.sep, '/')
    __printsafe(path)
    log(path)
    __printsafe(path)
    try:
        my_db =apsw.Connection(path)
        #my_db = sqlite3.connect(path)
    except Exception as e:
        __printsafe(str(e))
        log(str(e))
        raise e
        return

    my_cursor = my_db.cursor()

    mysql = str("PRAGMA main.locking_mode=EXCLUSIVE;")
    my_cursor.execute(mysql)  #apsw

    mysql = "PRAGMA main.busy_timeout = 5000;"      #PRAGMA busy_timeout = milliseconds;  (obviously just for this connection)
    my_cursor.execute(mysql)

    sleep(0.5)

    Create_SQLite_User_Functions(my_db, my_cursor, log)

    return my_db, my_cursor
    #-----------------------------------------------------------------------------------------------------------------
def Create_SQLite_User_Functions(my_db, my_cursor, log):

        global my_terminate_early

        #parameters are: function name for SQLite; number of parameters; Python callable function
        try:
            #this is apsw:
            my_db.createscalarfunction("update_utf8_for_display", SQLite_User_Function_1)
            # this is sqlite:  my_db.create_function("update_utf8_for_display", 1, SQLite_User_Function_1)
            __printsafe("Create_SQLite_User_Function 1 was successful...")
        except Exception as e:
            log("Create_SQLite_User_Function 1 failed...cannot proceed...")
            print(str(e))
            my_db.close()
            raise e


        sleep(0.5)  #avoid db locks
    #-----------------------------------------------------------------------------------------------------------------
def SQLite_User_Function_1(s):
    #------------------------------------------------------------------------------------------------------------
    #mysql = "UPDATE custom_column_4 SET value = (update_utf8_for_display(value)  );"
    #------------------------------------------------------------------------------------------------------------

    mynothing = ""

    t = s
    if t is None: #ignore Nulls being passed by SQLite
        return s
    else:
        try:
            if not isinstance(t, unicode):
                t = "u'" + str(t) + "'"
            if isinstance(t, unicode):
                t = eval(t)    #example:  unicode such as  u'n\xe3o-fic\xe7\xe3o'   that user wants to see in gui custom column as:  não-ficção
                return t
            else:
                __printsafe("SQLite_User_Function_1...........not unicode:", str(t))
                pass
        except Exception as e:
            log(str(e))
            log("SQLite User Function 1 UTF8 failed")
            pass

    return s
 #----------------------------------------------------------------------------------------------------------------
def remove_duplicate_work_tags(my_db, my_cursor, log):

    try:
        mysql = "SELECT id,value FROM custom_column_13 WHERE value LIKE '%, %'  "
        tmp_rows = __execute_mysql_fetchall_generic(my_db, my_cursor, log, mysql)
        if not tmp_rows:
            return
        else:
            if len(tmp_rows) == 0:
                return
            else:
                pass
        my_cursor.execute("begin")
        for row in tmp_rows:
            id,value = row
            if value.count(", ") == 0:     #not likely, given the SQL above...
                continue
            s_list = value.split(", ")
            s_set = set(s_list)
            s_list = list(s_set)   #now no duplicates
            s_list.sort()
            new_tagsall = ""
            for item in s_list:
                item = item.strip()
                new_tagsall = new_tagsall + ", " + item
            if new_tagsall.startswith(","):
                new_tagsall = new_tagsall[1: ]
                new_tagsall = new_tagsall.strip()
            mysql = "UPDATE custom_column_13 SET value = ? WHERE id = ? "
            my_cursor.execute(mysql,(new_tagsall,id))
            sleep(0.02)
        #END FOR
        try:
            my_cursor.execute("commit")
            sleep(0.5)
        except Exception as e:
            log(str(e))
            pass
    except Exception as e:
        log(str(e))
        try:
            my_cursor.execute("commit")
        except:
            pass
 #----------------------------------------------------------------------------------------------------------------
 #----------------------------------------------------------------------------------------------------------------
 #----------------------------------------------------------------------------------------------------------------
 #----------------------------------------------------------------------------------------------------------------
#END of miscellany.py
#=================================================================================

