From a2028ba0f633e60a73ddace2386408f7dd77ccee Mon Sep 17 00:00:00 2001 From: Michiel Nauta Date: Mon, 18 Apr 2011 21:09:49 +0000 Subject: [PATCH] Speedup merging svn: r17146 --- src/Merge/mergeevent.py | 19 ++++++++---- src/Merge/mergemedia.py | 37 +++++++++++++---------- src/Merge/mergenote.py | 51 ++++++++++++++++++-------------- src/Merge/mergeplace.py | 25 ++++++++++------ src/Merge/mergerepository.py | 12 ++++++-- src/Merge/mergesource.py | 50 ++++++++++++++++++------------- src/Merge/test/merge_ref_test.py | 12 +++++--- 7 files changed, 127 insertions(+), 79 deletions(-) diff --git a/src/Merge/mergeevent.py b/src/Merge/mergeevent.py index 04e76be26..63fa0bdcb 100644 --- a/src/Merge/mergeevent.py +++ b/src/Merge/mergeevent.py @@ -29,6 +29,7 @@ Provide merge capabilities for events. # Gramps modules # #------------------------------------------------------------------------- +from gen.lib import Person, Family from gen.db import DbTxn from gen.ggettext import sgettext as _ import const @@ -36,6 +37,7 @@ import GrampsDisplay import ManagedWindow import DateHandler import Utils +from Errors import MergeError #------------------------------------------------------------------------- # @@ -213,8 +215,11 @@ class MergeEventQuery(object): with DbTxn(_("Merge Event Objects"), self.database) as trans: self.database.commit_event(self.phoenix, trans) - for person in self.database.iter_people(): - if person.has_handle_reference("Event", old_handle): + for (class_name, handle) in self.database.find_backlink_handles( + old_handle): + if class_name == Person.__name__: + person = self.database.get_person_from_handle(handle) + assert(person.has_handle_reference("Event", old_handle)) bri = person.birth_ref_index dri = person.death_ref_index person.replace_handle_reference("Event", old_handle, @@ -234,11 +239,13 @@ class MergeEventQuery(object): person.death_ref_index = index break self.database.commit_person(person, trans) - - for family in self.database.iter_families(): - if family.has_handle_reference("Event", old_handle): + elif class_name == Family.__name__: + family = self.database.get_family_from_handle(handle) + assert(family.has_handle_reference("Event", old_handle)) family.replace_handle_reference("Event", old_handle, new_handle) self.database.commit_family(family, trans) - + else: + raise MergeError("Encounter an object of type %s that has " + "an event reference." % class_name) self.database.remove_event(old_handle, trans) diff --git a/src/Merge/mergemedia.py b/src/Merge/mergemedia.py index 2a4780020..56ee6fb94 100644 --- a/src/Merge/mergemedia.py +++ b/src/Merge/mergemedia.py @@ -29,12 +29,14 @@ Provide merge capabilities for media objects. # Gramps modules # #------------------------------------------------------------------------- +from gen.lib import Person, Family, Event, Source, Place from gen.db import DbTxn from gen.ggettext import sgettext as _ import const import GrampsDisplay import ManagedWindow import DateHandler +from Errors import MergeError #------------------------------------------------------------------------- # @@ -187,29 +189,34 @@ class MergeMediaQuery(object): with DbTxn(_("Merge Media Objects"), self.database) as trans: self.database.commit_media_object(self.phoenix, trans) - for person in self.database.iter_people(): - if person.has_media_reference(old_handle): + for (class_name, handle) in self.database.find_backlink_handles( + old_handle): + if class_name == Person.__name__: + person = self.database.get_person_from_handle(handle) + assert(person.has_media_reference(old_handle)) person.replace_media_references(old_handle, new_handle) self.database.commit_person(person, trans) - - for family in self.database.iter_families(): - if family.has_media_reference(old_handle): + elif class_name == Family.__name__: + family = self.database.get_family_from_handle(handle) + assert(family.has_media_reference(old_handle)) family.replace_media_references(old_handle, new_handle) self.database.commit_family(family, trans) - - for event in self.database.iter_events(): - if event.has_media_reference(old_handle): + elif class_name == Event.__name__: + event = self.database.get_event_from_handle(handle) + assert(event.has_media_reference(old_handle)) event.replace_media_references(old_handle, new_handle) self.database.commit_event(event, trans) - - for source in self.database.iter_sources(): - if source.has_media_reference(old_handle): + elif class_name == Source.__name__: + source = self.database.get_source_from_handle(handle) + assert(source.has_media_reference(old_handle)) source.replace_media_references(old_handle, new_handle) self.database.commit_source(source, trans) - - for place in self.database.iter_places(): - if place.has_media_reference(old_handle): + elif class_name == Place.__name__: + place = self.database.get_place_from_handle(handle) + assert(place.has_media_reference(old_handle)) place.replace_media_references(old_handle, new_handle) self.database.commit_place(place, trans) - + else: + raise MergeError("Encounter an object of type % s that has " + "a media object reference." % class_name) self.database.remove_object(old_handle, trans) diff --git a/src/Merge/mergenote.py b/src/Merge/mergenote.py index 46a91d237..4eef7f970 100644 --- a/src/Merge/mergenote.py +++ b/src/Merge/mergenote.py @@ -29,12 +29,15 @@ Provide merge capabilities for notes. # Gramps modules # #------------------------------------------------------------------------- +from gen.lib import (Person, Family, Event, Place, Source, Repository, + MediaObject) from gen.db import DbTxn from gen.ggettext import sgettext as _ import const import GrampsDisplay import ManagedWindow from gui.widgets.styledtextbuffer import StyledTextBuffer +from Errors import MergeError #------------------------------------------------------------------------- # @@ -199,40 +202,44 @@ class MergeNoteQuery(object): self.phoenix.merge(self.titanic) with DbTxn(_("Merge Notes"), self.database) as trans: self.database.commit_note(self.phoenix, trans) - - for person in self.database.iter_people(): - if person.has_note_reference(old_handle): + for (class_name, handle) in self.database.find_backlink_handles( + old_handle): + if class_name == Person.__name__: + person = self.database.get_person_from_handle(handle) + assert(person.has_note_reference(old_handle)) person.replace_note_references(old_handle, new_handle) self.database.commit_person(person, trans) - - for family in self.database.iter_families(): - if family.has_note_reference(old_handle): + elif class_name == Family.__name__: + family = self.database.get_family_from_handle(handle) + assert(family.has_note_reference(old_handle)) family.replace_note_references(old_handle, new_handle) self.database.commit_family(family, trans) - - for event in self.database.iter_events(): - if event.has_note_reference(old_handle): + elif class_name == Event.__name__: + event = self.database.get_event_from_handle(handle) + assert(event.has_note_reference(old_handle)) event.replace_note_references(old_handle, new_handle) self.database.commit_event(event, trans) - - for source in self.database.iter_sources(): - if source.has_note_reference(old_handle): + elif class_name == Source.__name__: + source = self.database.get_source_from_handle(handle) + assert(source.has_note_reference(old_handle)) source.replace_note_references(old_handle, new_handle) self.database.commit_source(source, trans) - - for place in self.database.iter_places(): - if place.has_note_reference(old_handle): + elif class_name == Place.__name__: + place = self.database.get_place_from_handle(handle) + assert(place.has_note_reference(old_handle)) place.replace_note_references(old_handle, new_handle) self.database.commit_place(place, trans) - - for obj in self.database.iter_media_objects(): - if obj.has_note_reference(old_handle): + elif class_name == MediaObject.__name__: + obj = self.database.get_object_from_handle(handle) + assert(obj.has_note_reference(old_handle)) obj.replace_note_references(old_handle, new_handle) self.database.commit_media_object(obj, trans) - - for repo in self.database.iter_repositories(): - if repo.has_note_reference(old_handle): + elif class_name == Repository.__name__: + repo = self.database.get_repository_from_handle(handle) + assert(repo.has_note_reference(old_handle)) repo.replace_note_references(old_handle, new_handle) self.database.commit_repository(repo, trans) - + else: + raise MergeError("Encounter object of type %s that has " + "a note reference." % class_name) self.database.remove_note(old_handle, trans) diff --git a/src/Merge/mergeplace.py b/src/Merge/mergeplace.py index d464659dc..d2b7afb65 100644 --- a/src/Merge/mergeplace.py +++ b/src/Merge/mergeplace.py @@ -37,11 +37,13 @@ import gtk # Gramps modules # #------------------------------------------------------------------------- +from gen.lib import Person, Family, Event from gen.db import DbTxn from gen.ggettext import sgettext as _ import const import GrampsDisplay import ManagedWindow +from Errors import MergeError #------------------------------------------------------------------------- # @@ -213,22 +215,27 @@ class MergePlaceQuery(object): with DbTxn(_("Merge Places"), self.database) as trans: self.database.commit_place(self.phoenix, trans) - for person in self.database.iter_people(): - if person.has_handle_reference('Place', old_handle): + for (class_name, handle) in self.database.find_backlink_handles( + old_handle): + if class_name == Person.__name__: + person = self.database.get_person_from_handle(handle) + assert(person.has_handle_reference('Place', old_handle)) person.replace_handle_reference('Place', old_handle, new_handle) self.database.commit_person(person, trans) - - for family in self.database.iter_families(): - if family.has_handle_reference('Place', old_handle): + elif class_name == Family.__name__: + family = self.database.get_family_from_handle(handle) + assert(family.has_handle_reference('Place', old_handle)) family.replace_handle_reference('Place', old_handle, new_handle) self.database.commit_family(family, trans) - - for event in self.database.iter_events(): - if event.has_handle_reference('Place', old_handle): + elif class_name == Event.__name__: + event = self.database.get_event_from_handle(handle) + assert(event.has_handle_reference('Place', old_handle)) event.replace_handle_reference('Place', old_handle, new_handle) self.database.commit_event(event, trans) - + else: + raise MergeError("Encounter an object of type %s that has " + "a place reference." % class_name) self.database.remove_place(old_handle, trans) diff --git a/src/Merge/mergerepository.py b/src/Merge/mergerepository.py index 1429b330e..bbde017f8 100644 --- a/src/Merge/mergerepository.py +++ b/src/Merge/mergerepository.py @@ -29,10 +29,12 @@ Provide merge capabilities for repositories. # Gramps modules # #------------------------------------------------------------------------- +from gen.lib import Source from gen.db import DbTxn from gen.ggettext import sgettext as _ import const import GrampsDisplay +from Errors import MergeError import ManagedWindow #------------------------------------------------------------------------- @@ -174,8 +176,14 @@ class MergeRepoQuery(object): with DbTxn(_("Merge Repositories"), self.database) as trans: self.database.commit_repository(self.phoenix, trans) - for source in self.database.iter_sources(): - if source.has_repo_reference(old_handle): + for (class_name, handle) in self.database.find_backlink_handles( + old_handle): + if class_name == Source.__name__: + source = self.database.get_source_from_handle(handle) + assert source.has_handle_reference('Repository', old_handle) source.replace_repo_references(old_handle, new_handle) self.database.commit_source(source, trans) + else: + raise MergeError("Encounter an object of type %s that has " + "a repository reference." % class_name) self.database.remove_repository(old_handle, trans) diff --git a/src/Merge/mergesource.py b/src/Merge/mergesource.py index d62a045e7..db387ac20 100644 --- a/src/Merge/mergesource.py +++ b/src/Merge/mergesource.py @@ -30,11 +30,14 @@ Provide merge capabilities for sources. # Gramps modules # #------------------------------------------------------------------------- +from gen.lib import (Person, Family, Event, Place, Source, Repository, + MediaObject) from gen.db import DbTxn from gen.ggettext import sgettext as _ import const import GrampsDisplay import ManagedWindow +from Errors import MergeError #------------------------------------------------------------------------- # @@ -200,39 +203,44 @@ class MergeSourceQuery(object): with DbTxn(_("Merge Source"), self.database) as trans: self.database.commit_source(self.phoenix, trans) - for person in self.database.iter_people(): - if person.has_source_reference(old_handle): + for (class_name, handle) in self.database.find_backlink_handles( + old_handle): + if class_name == Person.__name__: + person = self.database.get_person_from_handle(handle) + assert(person.has_source_reference(old_handle)) person.replace_source_references(old_handle, new_handle) self.database.commit_person(person, trans) - - for family in self.database.iter_families(): - if family.has_source_reference(old_handle): + elif class_name == Family.__name__: + family = self.database.get_family_from_handle(handle) + assert(family.has_source_reference(old_handle)) family.replace_source_references(old_handle, new_handle) self.database.commit_family(family, trans) - - for event in self.database.iter_events(): - if event.has_source_reference(old_handle): + elif class_name == Event.__name__: + event = self.database.get_event_from_handle(handle) + assert(event.has_source_reference(old_handle)) event.replace_source_references(old_handle, new_handle) self.database.commit_event(event, trans) - - for source in self.database.iter_sources(): - if source.has_source_reference(old_handle): + elif class_name == Source.__name__: + source = self.database.get_source_from_handle(handle) + assert(source.has_source_reference(old_handle)) source.replace_source_references(old_handle, new_handle) self.database.commit_source(source, trans) - - for place in self.database.iter_places(): - if place.has_source_reference(old_handle): + elif class_name == Place.__name__: + place = self.database.get_place_from_handle(handle) + assert(place.has_source_reference(old_handle)) place.replace_source_references(old_handle, new_handle) self.database.commit_place(place, trans) - - for obj in self.database.iter_media_objects(): - if obj.has_source_reference(old_handle): + elif class_name == MediaObject.__name__: + obj = self.database.get_object_from_handle(handle) + assert(obj.has_source_reference(old_handle)) obj.replace_source_references(old_handle, new_handle) self.database.commit_media_object(obj, trans) - - for repo in self.database.iter_repositories(): - if repo.has_source_reference(old_handle): + elif class_name == Repository.__name__: + repo = self.database.get_repository_from_handle(handle) + assert(repo.has_source_reference(old_handle)) repo.replace_source_references(old_handle, new_handle) self.database.commit_repository(repo, trans) - + else: + raise MergeError("Encounter an object of type %s that has " + "a source reference." % class_name) self.database.remove_source(old_handle, trans) diff --git a/src/Merge/test/merge_ref_test.py b/src/Merge/test/merge_ref_test.py index acd083cd9..aa1dc108f 100644 --- a/src/Merge/test/merge_ref_test.py +++ b/src/Merge/test/merge_ref_test.py @@ -127,7 +127,8 @@ class BaseMergeCheck(unittest.TestCase): '-p "name=climerge,primary=%s,secondary=%s" ' '-e - -f gramps' % (phoenix_id, titanic_id), stdin=subprocess.PIPE, - stdout=subprocess.PIPE, shell=True) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) result_str, err_str = process.communicate(str(input_doc)) if err_str: if test_error_str: @@ -163,17 +164,20 @@ class BaseMergeCheck(unittest.TestCase): self.canonicalize(expect_doc)) def raw_contains(self, phoenix_id, titanic_id, input_doc, expect_str, - debug=False): + test_error_str='', debug=False): process = subprocess.Popen('python gramps.py ' '--config=preferences.eprefix:DEFAULT ' '-i - -f gramps -a tool ' '-p "name=climerge,primary=%s,secondary=%s" ' '-e - -f raw' % (phoenix_id, titanic_id), stdin=subprocess.PIPE, - stdout=subprocess.PIPE, shell=True) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) result_str, err_str = process.communicate(str(input_doc)) if err_str: - print err_str + if test_error_str: + self.assert_(test_error_str in err_str) + return if debug: print 'input :', self.canonicalize(input_doc) print 'result:', result_str