diff --git a/gramps2/ChangeLog b/gramps2/ChangeLog index fc7136eb1..c0686a855 100644 --- a/gramps2/ChangeLog +++ b/gramps2/ChangeLog @@ -1,3 +1,17 @@ +2006-02-02 Alex Roitman + * src/GrampsDb/_GrampsBSDDB.py (transaction_begin, + transaction_commit): only do secondary index magic if not + instructde otherwise; (BdbTransaction.__init__): accept additional + argument, pass it on to Transaction class. + * src/GrampsDb/_GrampsDbBase.py (Transaction.__init__): accept + additional argument for no_magic. + * src/GrampsDb/_ReadGedcom.py (NoteParser): count people during + first pass; (parse_gedcom_file): start transaction with or without + secondary index magic. + * src/GrampsDb/_ReadXML.py (LineParser):count people during + first pass; (parse): start transaction with or without secondary + index magic. + 2006-02-01 Don Allingham * src/EditPerson.py: unused code * src/ImgManip.py: convienence code diff --git a/gramps2/src/GrampsDb/_GrampsBSDDB.py b/gramps2/src/GrampsDb/_GrampsBSDDB.py index 17aab0334..e017ecced 100644 --- a/gramps2/src/GrampsDb/_GrampsBSDDB.py +++ b/gramps2/src/GrampsDb/_GrampsBSDDB.py @@ -968,31 +968,27 @@ class GrampsBSDDB(GrampsDbBase): add_func(obj,transaction) return obj - def transaction_begin(self,msg="",batch=False): + def transaction_begin(self,msg="",batch=False,no_magic=False): """ Creates a new Transaction tied to the current UNDO database. The transaction has no effect until it is committed using the transaction_commit function of the this database object. """ - transaction = BdbTransaction(msg,self.undodb,batch) + transaction = BdbTransaction(msg,self.undodb,batch,no_magic) if transaction.batch: self.env.txn_checkpoint() self.env.set_flags(db.DB_TXN_NOSYNC,1) # async txn - # FIXME: There appears to have a bug in bsddb - # preventing us from rebuilding the secondary indices. - # In particular, the rebuild_secondary will also fail, - # but we can't help it. Disabling the secondary index - # removal/rebuilding for batch transactions for now. - # Disconnect unneeded secondary indices - self.surnames.close() - junk = db.DB(self.env) - junk.remove(self.full_name,"surnames") + if not transaction.no_magic: + # Disconnect unneeded secondary indices + self.surnames.close() + junk = db.DB(self.env) + junk.remove(self.full_name,"surnames") - self.reference_map_referenced_map.close() - junk = db.DB(self.env) - junk.remove(self.full_name,"reference_map_referenced_map") + self.reference_map_referenced_map.close() + junk = db.DB(self.env) + junk.remove(self.full_name,"reference_map_referenced_map") return transaction @@ -1015,28 +1011,25 @@ class GrampsBSDDB(GrampsDbBase): self.env.txn_checkpoint() self.env.set_flags(db.DB_TXN_NOSYNC,0) # sync txn - # FIXME: There appears to have a bug in bsddb - # preventing us from rebuilding the secondary indices. - # In particular, the rebuild_secondary will also fail, - # but we can't help it. Disabling the secondary index - # removal/rebuilding for batch transactions for now. + if not transaction.no_magic: + # create new secondary indices to replace the ones removed + open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT + dupe_flags = db.DB_DUP|db.DB_DUPSORT - open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT - dupe_flags = db.DB_DUP|db.DB_DUPSORT - # create new secondary indices to replace the ones removed - self.surnames = db.DB(self.env) - self.surnames.set_flags(dupe_flags) - self.surnames.open(self.full_name,"surnames", - db.DB_BTREE,flags=open_flags) - self.person_map.associate(self.surnames,find_surname,open_flags) + self.surnames = db.DB(self.env) + self.surnames.set_flags(dupe_flags) + self.surnames.open(self.full_name,"surnames", + db.DB_BTREE,flags=open_flags) + self.person_map.associate(self.surnames,find_surname, + open_flags) - self.reference_map_referenced_map = db.DB(self.env) - self.reference_map_referenced_map.set_flags(dupe_flags) - self.reference_map_referenced_map.open( - self.full_name,"reference_map_referenced_map", - db.DB_BTREE,flags=open_flags) - self.reference_map.associate(self.reference_map_referenced_map, - find_referenced_handle,open_flags) + self.reference_map_referenced_map = db.DB(self.env) + self.reference_map_referenced_map.set_flags(dupe_flags) + self.reference_map_referenced_map.open( + self.full_name,"reference_map_referenced_map", + db.DB_BTREE,flags=open_flags) + self.reference_map.associate(self.reference_map_referenced_map, + find_referenced_handle,open_flags) self.txn = None def undo(self): @@ -1499,8 +1492,8 @@ class GrampsBSDDB(GrampsDbBase): class BdbTransaction(Transaction): - def __init__(self,msg,db,batch=False): - Transaction.__init__(self,msg,db,batch) + def __init__(self,msg,db,batch=False,no_magic=False): + Transaction.__init__(self,msg,db,batch,no_magic) self.reference_del = [] self.reference_add = [] diff --git a/gramps2/src/GrampsDb/_GrampsDbBase.py b/gramps2/src/GrampsDb/_GrampsDbBase.py index f864d66b5..5273a5db3 100644 --- a/gramps2/src/GrampsDb/_GrampsDbBase.py +++ b/gramps2/src/GrampsDb/_GrampsDbBase.py @@ -1865,18 +1865,29 @@ class Transaction: Defines a group of database commits that define a single logical operation. """ - def __init__(self,msg,db,batch=False): + def __init__(self,msg,db,batch=False,no_magic=False): """ - Creates a new transaction. A Transaction instance should not be created - directly, but by the GrampsDbBase class or classes derived from - GrampsDbBase. The db parameter is a list-like interface that stores - the commit data. This could be a simple list, or a RECNO-style database - object. + Creates a new transaction. A Transaction instance should not be + created directly, but by the GrampsDbBase class or classes derived + from GrampsDbBase. The db parameter is a list-like interface that + stores the commit data. This could be a simple list, or a RECNO-style + database object. + + The batch parameter is set to True for large transactions. For such + transactions, the list of changes is not maintained, and no undo + is possible. + + The no_magic parameter is ignored for non-batch transactions, and + is also of no importance for DB backends other than BSD DB. For + the BSDDB, when this paramter is set to True, some secondary + indices will be removed at the beginning and then rebuilt at + the end of such transaction (only if it is batch). """ self.db = db self.first = None self.last = None self.batch = batch + self.no_magic = no_magic self.length = 0 self.person_add = [] diff --git a/gramps2/src/GrampsDb/_ReadGedcom.py b/gramps2/src/GrampsDb/_ReadGedcom.py index 7ff310938..ded9af5e9 100644 --- a/gramps2/src/GrampsDb/_ReadGedcom.py +++ b/gramps2/src/GrampsDb/_ReadGedcom.py @@ -1,7 +1,7 @@ # # Gramps - a GTK+/GNOME based genealogy program # -# Copyright (C) 2000-2005 Donald N. Allingham +# Copyright (C) 2000-2006 Donald N. Allingham # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -198,7 +198,7 @@ def import2(database, filename, callback, codeset, use_trans): try: np = NoteParser(filename, False) g = GedcomParser(database,filename, callback, codeset, np.get_map(), - np.get_lines()) + np.get_lines(),np.get_persons()) except IOError,msg: ErrorDialog(_("%s could not be opened\n") % filename,str(msg)) return @@ -275,6 +275,7 @@ class IdFinder: noteRE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+NOTE(.*)$") contRE = re.compile(r"\s*\d+\s+CONT\s(.*)$") concRE = re.compile(r"\s*\d+\s+CONC\s(.*)$") +personRE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+INDI(.*)$") #------------------------------------------------------------------------- # @@ -305,6 +306,7 @@ class NoteParser: self.name_map = {} self.count = 0 + self.person_count = 0 f = open(filename,"rU") innote = False @@ -333,6 +335,9 @@ class NoteParser: self.name_map["@%s@" % data] = noteobj noteobj.append(match.groups()[1]) innote = True + elif personRE.match(line): + self.person_count += 1 + f.close() def get_map(self): @@ -341,6 +346,9 @@ class NoteParser: def get_lines(self): return self.count + def get_persons(self): + return self.person_count + #------------------------------------------------------------------------- # # Reader - serves as the lexical analysis engine @@ -421,9 +429,10 @@ class GedcomParser: SyntaxError = "Syntax Error" BadFile = "Not a GEDCOM file" - def __init__(self, dbase, filename, callback, codeset, note_map, lines): + def __init__(self,dbase,filename,callback,codeset,note_map,lines,people): self.maxlines = lines + self.maxpeople = people self.interval = lines/100 self.percent = 0 self.callback = callback @@ -637,9 +646,12 @@ class GedcomParser: self.backoff = True def parse_gedcom_file(self,use_trans=False): + if self.maxpeople < 1000: + no_magic = True + else: + no_magic = False + self.trans = self.db.transaction_begin("",not use_trans,no_magic) - self.trans = self.db.transaction_begin("",not use_trans) - #self.trans.set_batch(not use_trans) self.db.disable_signals() t = time.time() self.fam_count = 0 diff --git a/gramps2/src/GrampsDb/_ReadXML.py b/gramps2/src/GrampsDb/_ReadXML.py index 022758f75..cf8fe45cd 100644 --- a/gramps2/src/GrampsDb/_ReadXML.py +++ b/gramps2/src/GrampsDb/_ReadXML.py @@ -30,6 +30,7 @@ import sets import shutil from xml.parsers.expat import ExpatError, ParserCreate from gettext import gettext as _ +import re #------------------------------------------------------------------------- # @@ -67,6 +68,8 @@ try: except: gzip_ok = 0 +personRE = re.compile(r"\s*\