Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* src/GrampsDb/_GrampsBSDDB.py (transaction_begin,

Browse Source 
transaction_commit): only do secondary index magic if not
instructde otherwise; (BdbTransaction.__init__): accept additional
argument, pass it on to Transaction class.
* src/GrampsDb/_GrampsDbBase.py (Transaction.__init__): accept
additional argument for no_magic.
* src/GrampsDb/_ReadGedcom.py (NoteParser): count people during
first pass; (parse_gedcom_file): start transaction with or without
secondary index magic.
* src/GrampsDb/_ReadXML.py (LineParser):count people during
first pass; (parse): start transaction with or without secondary
index magic.


svn: r5865
This commit is contained in:
Alex Roitman 2006-02-02 14:53:31 +00:00
parent da30e8d5fa
commit 487cd028ed
5 changed files with 95 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
ChangeLog
View File

@ -1,3 +1,17 @@
2006-02-02 Alex Roitman <shura@gramps-project.org>
* src/GrampsDb/_GrampsBSDDB.py (transaction_begin,
transaction_commit): only do secondary index magic if not
instructde otherwise; (BdbTransaction.__init__): accept additional
argument, pass it on to Transaction class.
* src/GrampsDb/_GrampsDbBase.py (Transaction.__init__): accept
additional argument for no_magic.
* src/GrampsDb/_ReadGedcom.py (NoteParser): count people during
first pass; (parse_gedcom_file): start transaction with or without
secondary index magic.
* src/GrampsDb/_ReadXML.py (LineParser):count people during
first pass; (parse): start transaction with or without secondary
index magic.
2006-02-01 Don Allingham <don@gramps-project.org>
* src/EditPerson.py: unused code
* src/ImgManip.py: convienence code

65
src/GrampsDb/_GrampsBSDDB.py
View File

@ -968,31 +968,27 @@ class GrampsBSDDB(GrampsDbBase):
add_func(obj,transaction)
return obj
def transaction_begin(self,msg="",batch=False):
def transaction_begin(self,msg="",batch=False,no_magic=False):
"""
Creates a new Transaction tied to the current UNDO database. The
transaction has no effect until it is committed using the
transaction_commit function of the this database object.
"""
transaction = BdbTransaction(msg,self.undodb,batch)
transaction = BdbTransaction(msg,self.undodb,batch,no_magic)
if transaction.batch:
self.env.txn_checkpoint()
self.env.set_flags(db.DB_TXN_NOSYNC,1) # async txn
# FIXME: There appears to have a bug in bsddb
# preventing us from rebuilding the secondary indices.
# In particular, the rebuild_secondary will also fail,
# but we can't help it. Disabling the secondary index
# removal/rebuilding for batch transactions for now.
# Disconnect unneeded secondary indices
self.surnames.close()
junk = db.DB(self.env)
junk.remove(self.full_name,"surnames")
if not transaction.no_magic:
# Disconnect unneeded secondary indices
self.surnames.close()
junk = db.DB(self.env)
junk.remove(self.full_name,"surnames")
self.reference_map_referenced_map.close()
junk = db.DB(self.env)
junk.remove(self.full_name,"reference_map_referenced_map")
self.reference_map_referenced_map.close()
junk = db.DB(self.env)
junk.remove(self.full_name,"reference_map_referenced_map")
return transaction
@ -1015,28 +1011,25 @@ class GrampsBSDDB(GrampsDbBase):
self.env.txn_checkpoint()
self.env.set_flags(db.DB_TXN_NOSYNC,0) # sync txn
# FIXME: There appears to have a bug in bsddb
# preventing us from rebuilding the secondary indices.
# In particular, the rebuild_secondary will also fail,
# but we can't help it. Disabling the secondary index
# removal/rebuilding for batch transactions for now.
if not transaction.no_magic:
# create new secondary indices to replace the ones removed
open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT
dupe_flags = db.DB_DUP|db.DB_DUPSORT
open_flags = db.DB_CREATE|db.DB_AUTO_COMMIT
dupe_flags = db.DB_DUP|db.DB_DUPSORT
# create new secondary indices to replace the ones removed
self.surnames = db.DB(self.env)
self.surnames.set_flags(dupe_flags)
self.surnames.open(self.full_name,"surnames",
db.DB_BTREE,flags=open_flags)
self.person_map.associate(self.surnames,find_surname,open_flags)
self.surnames = db.DB(self.env)
self.surnames.set_flags(dupe_flags)
self.surnames.open(self.full_name,"surnames",
db.DB_BTREE,flags=open_flags)
self.person_map.associate(self.surnames,find_surname,
open_flags)
self.reference_map_referenced_map = db.DB(self.env)
self.reference_map_referenced_map.set_flags(dupe_flags)
self.reference_map_referenced_map.open(
self.full_name,"reference_map_referenced_map",
db.DB_BTREE,flags=open_flags)
self.reference_map.associate(self.reference_map_referenced_map,
find_referenced_handle,open_flags)
self.reference_map_referenced_map = db.DB(self.env)
self.reference_map_referenced_map.set_flags(dupe_flags)
self.reference_map_referenced_map.open(
self.full_name,"reference_map_referenced_map",
db.DB_BTREE,flags=open_flags)
self.reference_map.associate(self.reference_map_referenced_map,
find_referenced_handle,open_flags)
self.txn = None
def undo(self):
@ -1499,8 +1492,8 @@ class GrampsBSDDB(GrampsDbBase):
class BdbTransaction(Transaction):
def __init__(self,msg,db,batch=False):
Transaction.__init__(self,msg,db,batch)
def __init__(self,msg,db,batch=False,no_magic=False):
Transaction.__init__(self,msg,db,batch,no_magic)
self.reference_del = []
self.reference_add = []

23
src/GrampsDb/_GrampsDbBase.py
View File

@ -1865,18 +1865,29 @@ class Transaction:
Defines a group of database commits that define a single logical
operation.
"""
def __init__(self,msg,db,batch=False):
def __init__(self,msg,db,batch=False,no_magic=False):
"""
Creates a new transaction. A Transaction instance should not be created
directly, but by the GrampsDbBase class or classes derived from
GrampsDbBase. The db parameter is a list-like interface that stores
the commit data. This could be a simple list, or a RECNO-style database
object.
Creates a new transaction. A Transaction instance should not be
created directly, but by the GrampsDbBase class or classes derived
from GrampsDbBase. The db parameter is a list-like interface that
stores the commit data. This could be a simple list, or a RECNO-style
database object.
The batch parameter is set to True for large transactions. For such
transactions, the list of changes is not maintained, and no undo
is possible.
The no_magic parameter is ignored for non-batch transactions, and
is also of no importance for DB backends other than BSD DB. For
the BSDDB, when this paramter is set to True, some secondary
indices will be removed at the beginning and then rebuilt at
the end of such transaction (only if it is batch).
"""
self.db = db
self.first = None
self.last = None
self.batch = batch
self.no_magic = no_magic
self.length = 0
self.person_add = []

22
src/GrampsDb/_ReadGedcom.py
View File

@ -1,7 +1,7 @@
#
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2000-2005 Donald N. Allingham
# Copyright (C) 2000-2006 Donald N. Allingham
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -198,7 +198,7 @@ def import2(database, filename, callback, codeset, use_trans):
try:
np = NoteParser(filename, False)
g = GedcomParser(database,filename, callback, codeset, np.get_map(),
np.get_lines())
np.get_lines(),np.get_persons())
except IOError,msg:
ErrorDialog(_("%s could not be opened\n") % filename,str(msg))
return
@ -275,6 +275,7 @@ class IdFinder:
noteRE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+NOTE(.*)$")
contRE = re.compile(r"\s*\d+\s+CONT\s(.*)$")
concRE = re.compile(r"\s*\d+\s+CONC\s(.*)$")
personRE = re.compile(r"\s*\d+\s+\@(\S+)\@\s+INDI(.*)$")
#-------------------------------------------------------------------------
#
@ -305,6 +306,7 @@ class NoteParser:
self.name_map = {}
self.count = 0
self.person_count = 0
f = open(filename,"rU")
innote = False
@ -333,6 +335,9 @@ class NoteParser:
self.name_map["@%s@" % data] = noteobj
noteobj.append(match.groups()[1])
innote = True
elif personRE.match(line):
self.person_count += 1
f.close()
def get_map(self):
@ -341,6 +346,9 @@ class NoteParser:
def get_lines(self):
return self.count
def get_persons(self):
return self.person_count
#-------------------------------------------------------------------------
#
# Reader - serves as the lexical analysis engine
@ -421,9 +429,10 @@ class GedcomParser:
SyntaxError = "Syntax Error"
BadFile = "Not a GEDCOM file"
def __init__(self, dbase, filename, callback, codeset, note_map, lines):
def __init__(self,dbase,filename,callback,codeset,note_map,lines,people):
self.maxlines = lines
self.maxpeople = people
self.interval = lines/100
self.percent = 0
self.callback = callback
@ -637,9 +646,12 @@ class GedcomParser:
self.backoff = True
def parse_gedcom_file(self,use_trans=False):
if self.maxpeople < 1000:
no_magic = True
else:
no_magic = False
self.trans = self.db.transaction_begin("",not use_trans,no_magic)
self.trans = self.db.transaction_begin("",not use_trans)
#self.trans.set_batch(not use_trans)
self.db.disable_signals()
t = time.time()
self.fam_count = 0

22
src/GrampsDb/_ReadXML.py
View File

@ -30,6 +30,7 @@ import sets
import shutil
from xml.parsers.expat import ExpatError, ParserCreate
from gettext import gettext as _
import re
#-------------------------------------------------------------------------
#
@ -67,6 +68,8 @@ try:
except:
gzip_ok = 0
personRE = re.compile(r"\s*\<person\s(.*)$")
#-------------------------------------------------------------------------
#
# Importing data into the currently open database.
@ -86,6 +89,7 @@ def importData(database, filename, callback=None,cl=0,use_trans=False):
linecounter = LineParser(filename)
lc = linecounter.get_count()
pc = linecounter.get_person_count()
ro = database.readonly
database.readonly = False
@ -124,7 +128,7 @@ def importData(database, filename, callback=None,cl=0,use_trans=False):
ErrorDialog(_("%s could not be opened") % filename)
return
try:
parser.parse(xml_file,use_trans,lc)
parser.parse(xml_file,use_trans,lc,pc)
except IOError,msg:
if cl:
print "Error reading %s" % filename
@ -208,6 +212,7 @@ class LineParser:
def __init__(self, filename):
self.count = 0
self.person_count = 0
if gzip_ok:
use_gzip = 1
@ -230,14 +235,20 @@ class LineParser:
for line in f:
self.count += 1
if personRE.match(line):
self.person_count += 1
f.close()
except:
self.count = 0
self.person_count = 0
def get_count(self):
return self.count
def get_person_count(self):
return self.person_count
#-------------------------------------------------------------------------
#
# Gramps database parsing class. Derived from SAX XML parser
@ -570,9 +581,12 @@ class GrampsParser:
self.ridswap[gramps_id] = gramps_id
return self.ridswap[gramps_id]
def parse(self,file,use_trans=False,linecount=0):
self.trans = self.db.transaction_begin("",batch=True)
def parse(self,file,use_trans=False,linecount=0,personcount=0):
if personcount < 1000:
no_magic = True
else:
no_magic = False
self.trans = self.db.transaction_begin("",batch=True,no_magic=no_magic)
self.linecount = linecount
self.db.disable_signals()