From 48eb507f984353044bac8729ca7788bb47fa7d55 Mon Sep 17 00:00:00 2001 From: prculley Date: Thu, 15 Jun 2017 16:24:41 -0500 Subject: [PATCH] Guarantee order on db emits Partial fix #10068 --- gramps/gen/db/generic.py | 60 +++++++++------- gramps/plugins/db/bsddb/undoredo.py | 103 +++++++++++++++------------- gramps/plugins/db/bsddb/write.py | 24 ++++--- gramps/plugins/db/dbapi/dbapi.py | 32 +++++---- 4 files changed, 123 insertions(+), 96 deletions(-) diff --git a/gramps/gen/db/generic.py b/gramps/gen/db/generic.py index 9f958a0c9..06593dfdc 100644 --- a/gramps/gen/db/generic.py +++ b/gramps/gen/db/generic.py @@ -44,7 +44,8 @@ import glob from . import (DbReadBase, DbWriteBase, DbUndo, DBLOGNAME, DBUNDOFN, KEY_TO_CLASS_MAP, REFERENCE_KEY, PERSON_KEY, FAMILY_KEY, CITATION_KEY, SOURCE_KEY, EVENT_KEY, MEDIA_KEY, PLACE_KEY, - REPOSITORY_KEY, NOTE_KEY, TAG_KEY, TXNADD, TXNDEL) + REPOSITORY_KEY, NOTE_KEY, TAG_KEY, TXNADD, TXNUPD, TXNDEL, + KEY_TO_NAME_MAP) from ..errors import HandleError from ..utils.callback import Callback from ..updatecallback import UpdateCallback @@ -132,6 +133,8 @@ class DbGenericUndo(DbUndo): transaction = txn db = self.db subitems = transaction.get_recnos() + # sigs[obj_type][trans_type] + sigs = [[[] for trans_type in range(3)] for key in range(11)] # Process all records in the transaction try: @@ -144,14 +147,10 @@ class DbGenericUndo(DbUndo): self.undo_reference(new_data, handle) else: self.undo_data(new_data, handle, key) + sigs[key][trans_type].append(handle) # now emit the signals - for record_id in subitems: - (key, trans_type, handle, old_data, new_data) = \ - pickle.loads(self.undodb[record_id]) + self.undo_sigs(sigs, False) - if key != REFERENCE_KEY: - self.undo_signals(trans_type, handle, - db.emit, SIGBASE[key], False) self.db._txn_commit() except: self.db._txn_abort() @@ -183,6 +182,8 @@ class DbGenericUndo(DbUndo): transaction = txn db = self.db subitems = transaction.get_recnos(reverse=True) + # sigs[obj_type][trans_type] + sigs = [[[] for trans_type in range(3)] for key in range(11)] # Process all records in the transaction try: @@ -195,14 +196,10 @@ class DbGenericUndo(DbUndo): self.undo_reference(old_data, handle) else: self.undo_data(old_data, handle, key) + sigs[key][trans_type].append(handle) # now emit the signals - for record_id in subitems: - (key, trans_type, handle, old_data, new_data) = \ - pickle.loads(self.undodb[record_id]) + self.undo_sigs(sigs, True) - if key != REFERENCE_KEY: - self.undo_signals(trans_type, handle, - db.emit, SIGBASE[key], True) self.db._txn_commit() except: self.db._txn_abort() @@ -257,19 +254,34 @@ class DbGenericUndo(DbUndo): obj = self.db._get_table_func(cls)["class_func"].create(data) self.db._update_secondary_values(obj) - def undo_signals(self, trans_type, handle, emit, signal_root, reverse): + def undo_sigs(self, sigs, undo): """ - Helper method to undo/redo the changes made + Helper method to undo/redo the signals for changes made + We want to do deletes and adds first + Note that if 'undo' we swap emits """ - if ((not reverse) and trans_type == TXNADD) \ - or (reverse and trans_type == TXNDEL): - typ = '-add' - elif not reverse and trans_type == TXNDEL \ - or reverse and trans_type == TXNADD: - typ = '-delete' - else: # TXNUPD - typ = '-update' - emit(signal_root + typ, ([handle],)) + for trans_type in [TXNDEL, TXNADD, TXNUPD]: + for obj_type in range(11): + handles = sigs[obj_type][trans_type] + if handles: + if not undo and trans_type == TXNDEL \ + or undo and trans_type == TXNADD: + typ = '-delete' + else: + # don't update a handle if its been deleted, and note + # that 'deleted' handles are in the 'add' list if we + # are undoing + handles = [handle for handle in handles + if handle not in + sigs[obj_type][TXNADD if undo else TXNDEL]] + if ((not undo) and trans_type == TXNADD) \ + or (undo and trans_type == TXNDEL): + typ = '-add' + else: # TXNUPD + typ = '-update' + if handles: + self.db.emit(KEY_TO_NAME_MAP[obj_type] + typ, + (handles,)) class Cursor: def __init__(self, iterator): diff --git a/gramps/plugins/db/bsddb/undoredo.py b/gramps/plugins/db/bsddb/undoredo.py index b69ac56a8..b4488164f 100644 --- a/gramps/plugins/db/bsddb/undoredo.py +++ b/gramps/plugins/db/bsddb/undoredo.py @@ -51,7 +51,8 @@ _ = glocale.translation.gettext # Gramps modules # #------------------------------------------------------------------------- -from gramps.gen.db.dbconst import * +from gramps.gen.db.dbconst import (REFERENCE_KEY, KEY_TO_NAME_MAP, TXNDEL, + TXNADD, TXNUPD) from . import BSDDBTxn from gramps.gen.errors import DbError @@ -226,8 +227,9 @@ class DbUndo: txn = self.undoq.pop() self.redoq.append(txn) transaction = txn - db = self.db subitems = transaction.get_recnos(reverse=True) + # sigs[obj_type][trans_type] + sigs = [[[] for trans_type in range(3)] for key in range(11)] # Process all records in the transaction for record_id in subitems: @@ -238,29 +240,25 @@ class DbUndo: self.undo_reference(old_data, handle, self.mapbase[key]) else: self.undo_data(old_data, handle, self.mapbase[key]) + handle = handle.decode('utf-8') + sigs[key][trans_type].append(handle) # now emit the signals - for record_id in subitems: - (key, trans_type, handle, old_data, new_data) = \ - pickle.loads(self.undodb[record_id]) - - if key != REFERENCE_KEY: - self.undo_signals(trans_type, handle, - db.emit, _SIGBASE[key], True) + self.undo_sigs(sigs, True) # Notify listeners - if db.undo_callback: + if self.db.undo_callback: if self.undo_count > 0: - db.undo_callback(_("_Undo %s") - % self.undoq[-1].get_description()) + self.db.undo_callback(_("_Undo %s") + % self.undoq[-1].get_description()) else: - db.undo_callback(None) + self.db.undo_callback(None) - if db.redo_callback: - db.redo_callback(_("_Redo %s") - % transaction.get_description()) + if self.db.redo_callback: + self.db.redo_callback(_("_Redo %s") + % transaction.get_description()) - if update_history and db.undo_history_callback: - db.undo_history_callback() + if update_history and self.db.undo_history_callback: + self.db.undo_history_callback() return True @undoredo @@ -272,8 +270,9 @@ class DbUndo: txn = self.redoq.pop() self.undoq.append(txn) transaction = txn - db = self.db subitems = transaction.get_recnos() + # sigs[obj_type][trans_type] + sigs = [[[] for trans_type in range(3)] for key in range(11)] # Process all records in the transaction for record_id in subitems: @@ -284,29 +283,26 @@ class DbUndo: self.undo_reference(new_data, handle, self.mapbase[key]) else: self.undo_data(new_data, handle, self.mapbase[key]) - # Process all signals in the transaction - for record_id in subitems: - (key, trans_type, handle, old_data, new_data) = \ - pickle.loads(self.undodb[record_id]) + handle = handle.decode('utf-8') + sigs[key][trans_type].append(handle) + # now emit the signals + self.undo_sigs(sigs, False) - if key != REFERENCE_KEY: - self.undo_signals(trans_type, handle, - db.emit, _SIGBASE[key], False) # Notify listeners - if db.undo_callback: - db.undo_callback(_("_Undo %s") - % transaction.get_description()) + if self.db.undo_callback: + self.db.undo_callback(_("_Undo %s") + % transaction.get_description()) - if db.redo_callback: + if self.db.redo_callback: if self.redo_count > 1: new_transaction = self.redoq[-2] - db.redo_callback(_("_Redo %s") - % new_transaction.get_description()) + self.db.redo_callback(_("_Redo %s") + % new_transaction.get_description()) else: - db.redo_callback(None) + self.db.redo_callback(None) - if update_history and db.undo_history_callback: - db.undo_history_callback() + if update_history and self.db.undo_history_callback: + self.db.undo_history_callback() return True def undo_reference(self, data, handle, db_map): @@ -337,19 +333,34 @@ class DbUndo: self.db._log_error() raise DbError(msg) - def undo_signals(self, trans_type, handle, emit, signal_root, reverse): + def undo_sigs(self, sigs, undo): """ - Helper method to undo/redo the changes made + Helper method to undo/redo the signals for changes made + We want to do deletes and adds first + Note that if 'undo' we swap emits """ - if ((not reverse) and trans_type == TXNADD) \ - or (reverse and trans_type == TXNDEL): - typ = '-add' - elif not reverse and trans_type == TXNDEL \ - or reverse and trans_type == TXNADD: - typ = '-delete' - else: # TXNUPD - typ = '-update' - emit(signal_root + typ, ([handle.decode('utf-8')],)) + for trans_type in [TXNDEL, TXNADD, TXNUPD]: + for obj_type in range(11): + handles = sigs[obj_type][trans_type] + if handles: + if not undo and trans_type == TXNDEL \ + or undo and trans_type == TXNADD: + typ = '-delete' + else: + # don't update a handle if its been deleted, and note + # that 'deleted' handles are in the 'add' list if we + # are undoing + handles = [handle for handle in handles + if handle not in + sigs[obj_type][TXNADD if undo else TXNDEL]] + if ((not undo) and trans_type == TXNADD) \ + or (undo and trans_type == TXNDEL): + typ = '-add' + else: # TXNUPD + typ = '-update' + if handles: + self.db.emit(KEY_TO_NAME_MAP[obj_type] + typ, + (handles,)) undo_count = property(lambda self:len(self.undoq)) redo_count = property(lambda self:len(self.redoq)) diff --git a/gramps/plugins/db/bsddb/write.py b/gramps/plugins/db/bsddb/write.py index fa56dd960..ac4c999e5 100644 --- a/gramps/plugins/db/bsddb/write.py +++ b/gramps/plugins/db/bsddb/write.py @@ -2034,11 +2034,11 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): self.txn = None self.env.log_flush() if not transaction.batch: - emit = self.__emit - for obj_type, obj_name in KEY_TO_NAME_MAP.items(): - emit(transaction, obj_type, TXNADD, obj_name, '-add') - emit(transaction, obj_type, TXNUPD, obj_name, '-update') - emit(transaction, obj_type, TXNDEL, obj_name, '-delete') + # do deletes and adds first + for trans_type in [TXNDEL, TXNADD, TXNUPD]: + for obj_type in range(11): + if obj_type != REFERENCE_KEY: + self.__emit(transaction, obj_type, trans_type) self.transaction = None transaction.clear() self.undodb.commit(transaction, msg) @@ -2051,21 +2051,23 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): hex(id(self)), transaction.get_description())) - def __emit(self, transaction, obj_type, trans_type, obj, suffix): + def __emit(self, transaction, obj_type, trans_type): """ Define helper function to do the actual emits """ if (obj_type, trans_type) in transaction: if trans_type == TXNDEL: handles = [handle.decode('utf-8') for handle, data in - transaction[(obj_type, trans_type)]] + transaction[(obj_type, trans_type)]] else: handles = [handle.decode('utf-8') for handle, data in - transaction[(obj_type, trans_type)] - if (handle, None) not in transaction[(obj_type, - TXNDEL)]] + transaction[(obj_type, trans_type)] + if (handle, None) not in transaction[(obj_type, + TXNDEL)]] if handles: - self.emit(obj + suffix, (handles, )) + self.emit(KEY_TO_NAME_MAP[obj_type] + + ['-add', '-update', '-delete'][trans_type], + (handles, )) def transaction_abort(self, transaction): """ diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py index a5eb090b5..56f9e9715 100644 --- a/gramps/plugins/db/dbapi/dbapi.py +++ b/gramps/plugins/db/dbapi/dbapi.py @@ -308,21 +308,23 @@ class DBAPI(DbGeneric): self.dbapi.commit() if not txn.batch: # Now, emit signals: - for (obj_type_val, txn_type_val) in list(txn): - if obj_type_val == REFERENCE_KEY: - continue - if txn_type_val == TXNDEL: - handles = [handle for (handle, data) in - txn[(obj_type_val, txn_type_val)]] - else: - handles = [handle for (handle, data) in - txn[(obj_type_val, txn_type_val)] - if (handle, None) - not in txn[(obj_type_val, TXNDEL)]] - if handles: - signal = KEY_TO_NAME_MAP[ - obj_type_val] + action[txn_type_val] - self.emit(signal, (handles, )) + # do deletes and adds first + for trans_type in [TXNDEL, TXNADD, TXNUPD]: + for obj_type in range(11): + if obj_type != REFERENCE_KEY and \ + (obj_type, trans_type) in txn: + if trans_type == TXNDEL: + handles = [handle for (handle, data) in + txn[(obj_type, trans_type)]] + else: + handles = [handle for (handle, data) in + txn[(obj_type, trans_type)] + if (handle, None) + not in txn[(obj_type, TXNDEL)]] + if handles: + signal = KEY_TO_NAME_MAP[ + obj_type] + action[trans_type] + self.emit(signal, (handles, )) self.transaction = None msg = txn.get_description() self.undodb.commit(txn, msg)