* src/GrampsDb/_GrampsBSDDB.py (_find_by_handle): delegate key

lookup to dbmap.db; (transaction_commit, transaction_begin):
Enable removing/rebuilding of the secondary index.
* src/GrampsDb/_ReadXML.py (LineParser): Improve first pass.


svn: r5861
This commit is contained in:
Alex Roitman 2006-02-01 06:25:51 +00:00
parent b13d42d960
commit c867ab1937
3 changed files with 56 additions and 51 deletions

View File

@ -4,6 +4,11 @@
* src/GrampsDb/_GrampsBSDDB.py (find_next_*_gramps_id): Remove * src/GrampsDb/_GrampsBSDDB.py (find_next_*_gramps_id): Remove
methods, as there's no need to override ones the DbBase now. methods, as there's no need to override ones the DbBase now.
* src/GrampsDb/_GrampsBSDDB.py (_find_by_handle): delegate key
lookup to dbmap.db; (transaction_commit, transaction_begin):
Enable removing/rebuilding of the secondary index.
* src/GrampsDb/_ReadXML.py (LineParser): Improve first pass.
2006-01-31 Don Allingham <don@gramps-project.org> 2006-01-31 Don Allingham <don@gramps-project.org>
* src/EditPerson.py: clean up building notebook pages * src/EditPerson.py: clean up building notebook pages
* src/DisplayTabs.py: support for drag-n-drop * src/DisplayTabs.py: support for drag-n-drop

View File

@ -579,7 +579,7 @@ class GrampsBSDDB(GrampsDbBase):
# Add references to the reference_map for all primary object referenced # Add references to the reference_map for all primary object referenced
# from the primary object 'obj' or any of its secondary objects. # from the primary object 'obj' or any of its secondary objects.
handle = obj.get_handle() handle = obj.handle
update = self.reference_map_primary_map.has_key(str(handle)) update = self.reference_map_primary_map.has_key(str(handle))
if update: if update:
@ -589,10 +589,10 @@ class GrampsBSDDB(GrampsDbBase):
# First thing to do is get hold of all rows in the reference_map # First thing to do is get hold of all rows in the reference_map
# table that hold a reference from this primary obj. This means finding # table that hold a reference from this primary obj. This means
# all the rows that have this handle somewhere in the list of (class_name,handle) # finding all the rows that have this handle somewhere in the
# pairs. # list of (class_name,handle) pairs.
# The primary_map secondary index allows us to look this up quickly. # The primary_map sec index allows us to look this up quickly.
existing_references = set() existing_references = set()
@ -610,25 +610,27 @@ class GrampsBSDDB(GrampsDbBase):
# ((primary_object_class_name, primary_object_handle), # ((primary_object_class_name, primary_object_handle),
# (referenced_object_class_name, referenced_object_handle)) # (referenced_object_class_name, referenced_object_handle))
# so we need the second tuple give us a reference that we can # so we need the second tuple give us a reference that we can
# compare with what is returned from get_referenced_handles_recursively # compare with what is returned from
# get_referenced_handles_recursively
# Looks like there is a bug in the set() and next_dup() methods # secondary DBs are not DBShelf's, so we need to do pickling
# because they do not run the data through cPickle.loads before # and unpicking ourselves here
# returning it, so we have to here.
existing_reference = cPickle.loads(data)[1] existing_reference = cPickle.loads(data)[1]
existing_references.add((KEY_TO_CLASS_MAP[existing_reference[0]], existing_references.add(
existing_reference[1])) (KEY_TO_CLASS_MAP[existing_reference[0]],
existing_reference[1]))
ret = primary_cur.next_dup() ret = primary_cur.next_dup()
primary_cur.close() primary_cur.close()
# Once we have the list of rows that already have a reference we need to compare # Once we have the list of rows that already have a reference
# it with the list of objects that are still references from the primary object. # we need to compare it with the list of objects that are
# still references from the primary object.
current_references = set(obj.get_referenced_handles_recursively()) current_references = set(obj.get_referenced_handles_recursively())
no_longer_required_references = existing_references.difference(current_references) no_longer_required_references = existing_references.difference(
current_references)
new_references = current_references.difference(existing_references) new_references = current_references.difference(existing_references)
@ -646,9 +648,11 @@ class GrampsBSDDB(GrampsDbBase):
if update: if update:
# handle deletion of old references # handle deletion of old references
if len(no_longer_required_references) > 0: if len(no_longer_required_references) > 0:
for (ref_class_name,ref_handle) in no_longer_required_references: for (ref_class_name,ref_handle) in \
no_longer_required_references:
try: try:
self._remove_reference((handle,ref_handle),transaction,txn) self._remove_reference(
(handle,ref_handle),transaction,txn)
#self.reference_map.delete(str((handle,ref_handle),), #self.reference_map.delete(str((handle,ref_handle),),
# txn=self.txn) # txn=self.txn)
except: # ignore missing old reference except: # ignore missing old reference
@ -956,7 +960,7 @@ class GrampsBSDDB(GrampsDbBase):
def _find_from_handle(self,handle,transaction,class_type,dmap,add_func): def _find_from_handle(self,handle,transaction,class_type,dmap,add_func):
obj = class_type() obj = class_type()
handle = str(handle) handle = str(handle)
if dmap.has_key(handle): if dmap.db.has_key(handle):
data = dmap.get(handle,txn=self.txn) data = dmap.get(handle,txn=self.txn)
obj.unserialize(data) obj.unserialize(data)
else: else:
@ -981,14 +985,14 @@ class GrampsBSDDB(GrampsDbBase):
# but we can't help it. Disabling the secondary index # but we can't help it. Disabling the secondary index
# removal/rebuilding for batch transactions for now. # removal/rebuilding for batch transactions for now.
## # Disconnect unneeded secondary indices # Disconnect unneeded secondary indices
## self.surnames.close() self.surnames.close()
## junk = db.DB(self.env) junk = db.DB(self.env)
## junk.remove(self.full_name,"surnames") junk.remove(self.full_name,"surnames")
## self.reference_map_referenced_map.close() self.reference_map_referenced_map.close()
## junk = db.DB(self.env) junk = db.DB(self.env)
## junk.remove(self.full_name,"reference_map_referenced_map") junk.remove(self.full_name,"reference_map_referenced_map")
return transaction return transaction
@ -1017,22 +1021,22 @@ class GrampsBSDDB(GrampsDbBase):
# but we can't help it. Disabling the secondary index # but we can't help it. Disabling the secondary index
# removal/rebuilding for batch transactions for now. # removal/rebuilding for batch transactions for now.
## open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT
## dupe_flags = db.DB_DUP|db.DB_DUPSORT dupe_flags = db.DB_DUP|db.DB_DUPSORT
## # create new secondary indices to replace the ones removed # create new secondary indices to replace the ones removed
## self.surnames = db.DB(self.env) self.surnames = db.DB(self.env)
## self.surnames.set_flags(dupe_flags) self.surnames.set_flags(dupe_flags)
## self.surnames.open(self.full_name,"surnames", self.surnames.open(self.full_name,"surnames",
## db.DB_BTREE,flags=table_flags) db.DB_BTREE,flags=open_flags)
## self.person_map.associate(self.surnames,find_surname,open_flags) self.person_map.associate(self.surnames,find_surname,open_flags)
## self.reference_map_referenced_map = db.DB(self.env) self.reference_map_referenced_map = db.DB(self.env)
## self.reference_map_referenced_map.set_flags(dupe_flags) self.reference_map_referenced_map.set_flags(dupe_flags)
## self.reference_map_referenced_map.open( self.reference_map_referenced_map.open(
## self.full_name,"reference_map_referenced_map", self.full_name,"reference_map_referenced_map",
## db.DB_BTREE,flags=open_flags) db.DB_BTREE,flags=open_flags)
## self.reference_map.associate(self.reference_map_referenced_map, self.reference_map.associate(self.reference_map_referenced_map,
## find_referenced_handle,open_flags) find_referenced_handle,open_flags)
self.txn = None self.txn = None
def undo(self): def undo(self):

View File

@ -84,11 +84,8 @@ def importData(database, filename, callback=None,cl=0,use_trans=False):
change = os.path.getmtime(filename) change = os.path.getmtime(filename)
parser = GrampsParser(database,callback,basefile,change,filename) parser = GrampsParser(database,callback,basefile,change,filename)
print "starting line parser"
linecounter = LineParser(filename) linecounter = LineParser(filename)
print "finished parser"
lc = linecounter.get_count() lc = linecounter.get_count()
print "got line count:", lc
ro = database.readonly ro = database.readonly
database.readonly = False database.readonly = False
@ -231,7 +228,9 @@ class LineParser:
else: else:
f = open(filename,"r") f = open(filename,"r")
self.count = len(f.readlines()) for line in f:
self.count += 1
f.close() f.close()
except: except:
self.count = 0 self.count = 0
@ -587,12 +586,12 @@ class GrampsParser:
self.db.set_researcher(self.owner) self.db.set_researcher(self.owner)
if self.home != None: if self.home != None:
person = self.db.find_person_from_handle(self.home,self.trans) person = self.db.find_person_from_handle(self.home,self.trans)
self.db.set_default_person_handle(person.get_handle()) self.db.set_default_person_handle(person.handle)
if self.tempDefault != None: if self.tempDefault != None:
handle = self.map_gid(self.tempDefault) gramps_id = self.map_gid(self.tempDefault)
person = self.find_person_by_gramps_id(handle) person = self.find_person_by_gramps_id(gramps_id)
if person: if person:
self.db.set_default_person_handle(person.get_handle()) self.db.set_default_person_handle(person.handle)
for key in self.func_map.keys(): for key in self.func_map.keys():
del self.func_map[key] del self.func_map[key]
@ -1649,11 +1648,8 @@ if __name__ == "__main__":
parser = GrampsParser(database,callback,basefile,change,filename) parser = GrampsParser(database,callback,basefile,change,filename)
print "starting line parser"
linecounter = LineParser(filename) linecounter = LineParser(filename)
print "finished parser"
lc = linecounter.get_count() lc = linecounter.get_count()
print "got line count:", lc
xml_file = gzip.open(filename,"rb") xml_file = gzip.open(filename,"rb")