[capink]

Originally Posted by bigwoof

Hi,

I created a chain to fix all the various tags in my library. it has 4 different single field edit templates all with default scope (whatever is selected).

I selected all the books and ran the chain. it crashed with an out of memory error.

Is there a way to get the chains to run book by book over an entire library? I don't mind waiting a few/many hours for it to be done.

Thanks!

Bigwoof

Here is a modified version of the Chain Caller action (called Modified Chain Caller) that can call a target chain multiple times to process chunks of books for each iteration.

#!/usr/bin/env python
# ~*~ coding: utf-8 ~*~

from qt.core import (QApplication, Qt, QWidget, QGridLayout, QHBoxLayout, QVBoxLayout,
                      QGroupBox, QComboBox, QCheckBox, QSpinBox)

import time

from calibre import prints
from calibre.constants import DEBUG

from calibre_plugins.action_chains.chains import Chain
import calibre_plugins.action_chains.config as cfg


from calibre_plugins.action_chains.actions.chain_caller import ChainCallerAction


tooltip = '''
This option splits the books into multiple chunks and run the target chain multiple times,
each time processing a chunk of books. The chunk size can be selected in the spin box.
The book ids in each chunk is passed to the target chain in a variabe called _book_ids.
You must adjust the scope of the actions in the target chains using a template scope to
act on this variable.
'''

class ConfigWidget(QWidget):
    def __init__(self, plugin_action, chain_name, chains_config, *args, **kwargs):
        QWidget.__init__(self)
        self.plugin_action = plugin_action
        self.gui = plugin_action.gui
        self.db = self.gui.current_db
        self.chain_name = chain_name
        self.chains_config = chains_config
        self._init_controls()

    def _init_controls(self):

        l = self.l = QVBoxLayout()
        self.setLayout(l)

        chain_groupbox = QGroupBox(_('Select chain'))
        l.addWidget(chain_groupbox)
        chain_groupbox_l = QVBoxLayout()
        chain_groupbox.setLayout(chain_groupbox_l)
        self.chain_combo = QComboBox()
        chain_groupbox_l.addWidget(self.chain_combo)

        opts_groupbox = QGroupBox(_('Options'))
        opts_groupbox_l = QVBoxLayout()
        opts_groupbox.setLayout(opts_groupbox_l)
        l.addWidget(opts_groupbox)

        vars_chk = self.vars_chk = QCheckBox(_('Pass variables from calling chain'))
        vars_chk.setChecked(False)
        opts_groupbox_l.addWidget(vars_chk)

        #all_chain_names = [chain_config['menuText'] for chain_config in cfg.get_chains_config()]
        all_chain_names = [chain_config['menuText'] for chain_config in self.chains_config if chain_config['menuText']]
        try:
            all_chain_names.remove(self.chain_name)
        except:
            pass
        self.chain_combo.addItems(all_chain_names)
        self.chain_combo.setCurrentIndex(-1)

        chunk_group_box = self.chunk_group_box = QGroupBox('Split books into chunks')
        chunk_group_box.setCheckable(True)
        chunk_group_box.setChecked(False)
        chunk_group_box_l = QVBoxLayout()
        chunk_group_box.setLayout(chunk_group_box_l)
        l.addWidget(chunk_group_box)
        chunk_group_box.setToolTip(tooltip)
        self.chunk_spinbox = QSpinBox()
        self.chunk_spinbox.setMaximum(100000)
        self.chunk_spinbox.setMinimum(1)
        self.chunk_spinbox.setSingleStep(1)
        self.chunk_spinbox.setValue(10)
        chunk_group_box_l.addWidget(self.chunk_spinbox)

        delay_group_box = self.delay_group_box = QGroupBox('Delay')
        delay_group_box_l = QVBoxLayout()
        delay_group_box.setLayout(delay_group_box_l)
        l.addWidget(delay_group_box)
        delay_group_box.setToolTip(_('Delay in seconds between calling chunks'))
        self.delay_spinbox = QSpinBox()
        self.delay_spinbox.setMaximum(100000)
        self.delay_spinbox.setMinimum(0)
        self.delay_spinbox.setSingleStep(1)
        self.delay_spinbox.setValue(0)
        delay_group_box_l.addWidget(self.delay_spinbox)

        l.addStretch(1)

        self.setMinimumSize(300,300)

    def load_settings(self, settings):
        if settings:
            idx = self.chain_combo.findText(settings['chain_name'])
            self.chain_combo.setCurrentIndex(idx)
            self.vars_chk.setChecked(settings.get('pass_vars', False))
            self.chunk_group_box.setChecked(settings.get('split_to_chunks', False))
            self.chunk_spinbox.setValue(settings.get('chunk_size', 10))
            self.delay_spinbox.setValue(settings.get('delay', 0))

    def save_settings(self):
        settings = {}
        settings['chain_name'] = self.chain_combo.currentText()
        settings['pass_vars'] = self.vars_chk.isChecked()
        settings['split_to_chunks'] = self.chunk_group_box.isChecked()
        settings['chunk_size'] = self.chunk_spinbox.value()
        settings['delay'] = self.delay_spinbox.value()
        return settings

class ModifiedChainCallerAction(ChainCallerAction):

    name = 'Modified Chain Caller'
    is_experimental = False

    def run(self, gui, settings, chain):
        # Chain variables {
        add_to_vars = {}
        if settings.get('pass_vars'):
            add_to_vars = self.sanitize_chain_vars(chain)
        add_to_vars['_caller'] = chain.chain_name
        # }
        book_ids = chain.scope().get_book_ids()
        if settings.get('split_to_chunks', False):
            chunk_size = settings['chunk_size']
            chunks = [book_ids[x:x+chunk_size] for x in range(0, len(book_ids), chunk_size)]
        else:
            chunks = [book_ids]
        chunks_no = len(chunks)
        delay = settings.get('delay', 0)
        for i, chunk in enumerate(chunks):
            prints('Action Chains: Chain Caller: running chunk no {} of {}'.format(i, chunks_no))
            add_to_vars['_book_ids'] = ','.join([str(x) for x in chunk])
            target_chain_config = cfg.get_chain_config(settings['chain_name'])
            target_chain = Chain(self.plugin_action, target_chain_config, show_progress=False,
                                chain_vars=add_to_vars, chain_caller=True)
            target_chain.run()
            del target_chain
            if delay:
                if DEBUG:
                    prints(f'sleeping for {delay} seconds')
                time.sleep(delay)

    def config_widget(self):
        return ConfigWidget

You should use this Modified Chain Caller to call your target chain as follows:

1. Add the Modified Chain Caller as a custom action using the module editor (Action Chains > manage modules > create module > copy/paste the code above)
2. Create a new chain that consists of a single action > Modified Chain Caller.
3. In the settings of the Modified Chain Caller, choose your target chain from the dropdown menu.
4. In the settings of the Modified Chain Caller, tick the box for splitting books into chunks, and choose the chunk size (default is 10 books)
5. In your target chain, all the actions must have a template scope with the following template:
   

   program:
       globals(_book_ids)

6. Now select the books you want to process, and run the Modified Chain Caller chain.

I am not sure whether this can help you with the out of memory problem, give it a try and see.