From c867ab193762f2cf3fa6057ad168077b1080f418 Mon Sep 17 00:00:00 2001 From: Alex Roitman Date: Wed, 1 Feb 2006 06:25:51 +0000 Subject: [PATCH] * src/GrampsDb/_GrampsBSDDB.py (_find_by_handle): delegate key lookup to dbmap.db; (transaction_commit, transaction_begin): Enable removing/rebuilding of the secondary index. * src/GrampsDb/_ReadXML.py (LineParser): Improve first pass. svn: r5861 --- gramps2/ChangeLog | 5 ++ gramps2/src/GrampsDb/_GrampsBSDDB.py | 84 +++++++++++++++------------- gramps2/src/GrampsDb/_ReadXML.py | 18 +++--- 3 files changed, 56 insertions(+), 51 deletions(-) diff --git a/gramps2/ChangeLog b/gramps2/ChangeLog index 07ac9d9fb..df0d4fff0 100644 --- a/gramps2/ChangeLog +++ b/gramps2/ChangeLog @@ -4,6 +4,11 @@ * src/GrampsDb/_GrampsBSDDB.py (find_next_*_gramps_id): Remove methods, as there's no need to override ones the DbBase now. + * src/GrampsDb/_GrampsBSDDB.py (_find_by_handle): delegate key + lookup to dbmap.db; (transaction_commit, transaction_begin): + Enable removing/rebuilding of the secondary index. + * src/GrampsDb/_ReadXML.py (LineParser): Improve first pass. + 2006-01-31 Don Allingham * src/EditPerson.py: clean up building notebook pages * src/DisplayTabs.py: support for drag-n-drop diff --git a/gramps2/src/GrampsDb/_GrampsBSDDB.py b/gramps2/src/GrampsDb/_GrampsBSDDB.py index 917adb4fb..43ac7f43e 100644 --- a/gramps2/src/GrampsDb/_GrampsBSDDB.py +++ b/gramps2/src/GrampsDb/_GrampsBSDDB.py @@ -579,7 +579,7 @@ class GrampsBSDDB(GrampsDbBase): # Add references to the reference_map for all primary object referenced # from the primary object 'obj' or any of its secondary objects. - handle = obj.get_handle() + handle = obj.handle update = self.reference_map_primary_map.has_key(str(handle)) if update: @@ -589,10 +589,10 @@ class GrampsBSDDB(GrampsDbBase): # First thing to do is get hold of all rows in the reference_map - # table that hold a reference from this primary obj. This means finding - # all the rows that have this handle somewhere in the list of (class_name,handle) - # pairs. - # The primary_map secondary index allows us to look this up quickly. + # table that hold a reference from this primary obj. This means + # finding all the rows that have this handle somewhere in the + # list of (class_name,handle) pairs. + # The primary_map sec index allows us to look this up quickly. existing_references = set() @@ -610,25 +610,27 @@ class GrampsBSDDB(GrampsDbBase): # ((primary_object_class_name, primary_object_handle), # (referenced_object_class_name, referenced_object_handle)) # so we need the second tuple give us a reference that we can - # compare with what is returned from get_referenced_handles_recursively + # compare with what is returned from + # get_referenced_handles_recursively - # Looks like there is a bug in the set() and next_dup() methods - # because they do not run the data through cPickle.loads before - # returning it, so we have to here. + # secondary DBs are not DBShelf's, so we need to do pickling + # and unpicking ourselves here existing_reference = cPickle.loads(data)[1] - existing_references.add((KEY_TO_CLASS_MAP[existing_reference[0]], - existing_reference[1])) + existing_references.add( + (KEY_TO_CLASS_MAP[existing_reference[0]], + existing_reference[1])) ret = primary_cur.next_dup() primary_cur.close() - # Once we have the list of rows that already have a reference we need to compare - # it with the list of objects that are still references from the primary object. + # Once we have the list of rows that already have a reference + # we need to compare it with the list of objects that are + # still references from the primary object. current_references = set(obj.get_referenced_handles_recursively()) - no_longer_required_references = existing_references.difference(current_references) - + no_longer_required_references = existing_references.difference( + current_references) new_references = current_references.difference(existing_references) @@ -646,9 +648,11 @@ class GrampsBSDDB(GrampsDbBase): if update: # handle deletion of old references if len(no_longer_required_references) > 0: - for (ref_class_name,ref_handle) in no_longer_required_references: + for (ref_class_name,ref_handle) in \ + no_longer_required_references: try: - self._remove_reference((handle,ref_handle),transaction,txn) + self._remove_reference( + (handle,ref_handle),transaction,txn) #self.reference_map.delete(str((handle,ref_handle),), # txn=self.txn) except: # ignore missing old reference @@ -956,7 +960,7 @@ class GrampsBSDDB(GrampsDbBase): def _find_from_handle(self,handle,transaction,class_type,dmap,add_func): obj = class_type() handle = str(handle) - if dmap.has_key(handle): + if dmap.db.has_key(handle): data = dmap.get(handle,txn=self.txn) obj.unserialize(data) else: @@ -981,14 +985,14 @@ class GrampsBSDDB(GrampsDbBase): # but we can't help it. Disabling the secondary index # removal/rebuilding for batch transactions for now. -## # Disconnect unneeded secondary indices -## self.surnames.close() -## junk = db.DB(self.env) -## junk.remove(self.full_name,"surnames") + # Disconnect unneeded secondary indices + self.surnames.close() + junk = db.DB(self.env) + junk.remove(self.full_name,"surnames") -## self.reference_map_referenced_map.close() -## junk = db.DB(self.env) -## junk.remove(self.full_name,"reference_map_referenced_map") + self.reference_map_referenced_map.close() + junk = db.DB(self.env) + junk.remove(self.full_name,"reference_map_referenced_map") return transaction @@ -1017,22 +1021,22 @@ class GrampsBSDDB(GrampsDbBase): # but we can't help it. Disabling the secondary index # removal/rebuilding for batch transactions for now. -## open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT -## dupe_flags = db.DB_DUP|db.DB_DUPSORT -## # create new secondary indices to replace the ones removed -## self.surnames = db.DB(self.env) -## self.surnames.set_flags(dupe_flags) -## self.surnames.open(self.full_name,"surnames", -## db.DB_BTREE,flags=table_flags) -## self.person_map.associate(self.surnames,find_surname,open_flags) + open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT + dupe_flags = db.DB_DUP|db.DB_DUPSORT + # create new secondary indices to replace the ones removed + self.surnames = db.DB(self.env) + self.surnames.set_flags(dupe_flags) + self.surnames.open(self.full_name,"surnames", + db.DB_BTREE,flags=open_flags) + self.person_map.associate(self.surnames,find_surname,open_flags) -## self.reference_map_referenced_map = db.DB(self.env) -## self.reference_map_referenced_map.set_flags(dupe_flags) -## self.reference_map_referenced_map.open( -## self.full_name,"reference_map_referenced_map", -## db.DB_BTREE,flags=open_flags) -## self.reference_map.associate(self.reference_map_referenced_map, -## find_referenced_handle,open_flags) + self.reference_map_referenced_map = db.DB(self.env) + self.reference_map_referenced_map.set_flags(dupe_flags) + self.reference_map_referenced_map.open( + self.full_name,"reference_map_referenced_map", + db.DB_BTREE,flags=open_flags) + self.reference_map.associate(self.reference_map_referenced_map, + find_referenced_handle,open_flags) self.txn = None def undo(self): diff --git a/gramps2/src/GrampsDb/_ReadXML.py b/gramps2/src/GrampsDb/_ReadXML.py index 8439820a1..022758f75 100644 --- a/gramps2/src/GrampsDb/_ReadXML.py +++ b/gramps2/src/GrampsDb/_ReadXML.py @@ -84,11 +84,8 @@ def importData(database, filename, callback=None,cl=0,use_trans=False): change = os.path.getmtime(filename) parser = GrampsParser(database,callback,basefile,change,filename) - print "starting line parser" linecounter = LineParser(filename) - print "finished parser" lc = linecounter.get_count() - print "got line count:", lc ro = database.readonly database.readonly = False @@ -231,7 +228,9 @@ class LineParser: else: f = open(filename,"r") - self.count = len(f.readlines()) + for line in f: + self.count += 1 + f.close() except: self.count = 0 @@ -587,12 +586,12 @@ class GrampsParser: self.db.set_researcher(self.owner) if self.home != None: person = self.db.find_person_from_handle(self.home,self.trans) - self.db.set_default_person_handle(person.get_handle()) + self.db.set_default_person_handle(person.handle) if self.tempDefault != None: - handle = self.map_gid(self.tempDefault) - person = self.find_person_by_gramps_id(handle) + gramps_id = self.map_gid(self.tempDefault) + person = self.find_person_by_gramps_id(gramps_id) if person: - self.db.set_default_person_handle(person.get_handle()) + self.db.set_default_person_handle(person.handle) for key in self.func_map.keys(): del self.func_map[key] @@ -1649,11 +1648,8 @@ if __name__ == "__main__": parser = GrampsParser(database,callback,basefile,change,filename) - print "starting line parser" linecounter = LineParser(filename) - print "finished parser" lc = linecounter.get_count() - print "got line count:", lc xml_file = gzip.open(filename,"rb")