Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merged geps21 branch, changes r15866-16034, into trunk

Browse Source 
svn: r16035
This commit is contained in:
Benny Malengier
2010-10-24 14:43:47 +00:00
parent 36f70bed44 405572cbdc
commit e109e3d743
48 changed files with 3589 additions and 1548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plugins/import/ImportGrdb.py
+110 -14
View File
@@ -77,8 +77,9 @@ def find_surname(key, data):
"""
Return the surname from the data stream. Used for building a secondary
index.
This function is not needed, as we don't use the secondary index.
"""
return str(data[3][5])
return str("a")
def find_idmap(key, data):
"""
@@ -138,6 +139,10 @@ class GrampsBSDDB(DbGrdb, UpdateCallback):
This is replaced for internal use by gen/db/dbdir.py
However, this class is still used for import of the 2.2.x
GRDB format. In 3.0+ this format is no longer used.
We only need to upgrade the old main tables.
That will be used to append data to the database this GrampsBSDDB is
imported to.
"""
def __init__(self, use_txn = True):
@@ -1247,19 +1252,6 @@ class GrampsBSDDB(DbGrdb, UpdateCallback):
self.surname_list = list(set(self.surnames.keys()))
self.sort_surname_list()
def remove_from_surname_list(self, person):
"""
Check whether there are persons with the same surname left in
the database. If not then we need to remove the name from the list.
The function must be overridden in the derived class.
"""
name = str(person.get_primary_name().get_surname())
try:
if self.surnames.keys().count(name) == 1:
self.surname_list.remove(unicode(name))
except ValueError:
pass
def __get_obj_from_gramps_id(self, val, tbl, class_init, prim_tbl):
if tbl.has_key(str(val)):
#if str(val) in tbl:
@@ -1563,6 +1555,8 @@ class GrampsBSDDB(DbGrdb, UpdateCallback):
self.gramps_upgrade_13()
if version < 14:
self.gramps_upgrade_14()
if version < 15:
self.gramps_upgrade_15()
LOG.debug("Upgrade time: %s %s", int(time.time()-t), "seconds")
def gramps_upgrade_10(self):
@@ -2589,6 +2583,108 @@ class GrampsBSDDB(DbGrdb, UpdateCallback):
name_type, prefix, patronymic,
group_as, sort_as, display_as, call)
def gramps_upgrade_15(self):
"""Upgrade database from version 14 to 15. This upgrade adds:
* tagging
* surname list
"""
length = len(self.person_map)+10
self.set_total(length)
# ---------------------------------
# Modify Person
# ---------------------------------
for handle in self.person_map.keys():
person = self.person_map[handle]
(junk_handle, # 0
gramps_id, # 1
gender, # 2
primary_name, # 3
alternate_names, # 4
death_ref_index, # 5
birth_ref_index, # 6
event_ref_list, # 7
family_list, # 8
parent_family_list, # 9
media_list, # 10
address_list, # 11
attribute_list, # 12
urls, # 13
ord_list, # 14
psource_list, # 15
pnote_list, # 16
change, # 17
marker, # 18
pprivate, # 19
person_ref_list, # 20
) = person
new_primary_name = self.convert_name_15(primary_name)
new_alternate_names = [self.convert_name_15(altname) for altname in
alternate_names]
new_person = (junk_handle, # 0
gramps_id, # 1
gender, # 2
new_primary_name, # 3
new_alternate_names,# 4
death_ref_index, # 5
birth_ref_index, # 6
event_ref_list, # 7
family_list, # 8
parent_family_list, # 9
media_list, # 10
address_list, # 11
attribute_list, # 12
urls, # 13
ord_list, # 14
psource_list, # 15
pnote_list, # 16
change, # 17
marker, # 18
pprivate, # 19
person_ref_list, # 20
[] # 21, tags
)
the_txn = self.env.txn_begin()
self.person_map.put(str(handle), new_person, txn=the_txn)
the_txn.commit()
self.update(length)
#surname is now different, normally remove secondary index with names
#we skip this, as this database will not be used after the import
# Bump up database version. Separate transaction to save metadata.
the_txn = self.env.txn_begin()
self.metadata.put('version', 15, txn=the_txn)
the_txn.commit()
def convert_name_15(self, name):
(privacy, source_list, note_list, date,
first_name, surname, suffix, title,
name_type, prefix, patronymic,
group_as, sort_as, display_as, call) = name
connector = u""
origintype = (NameOriginType.NONE, u"")
patorigintype = (NameOriginType.PATRONYMIC, u"")
if patronymic.strip() == u"":
#no patronymic, create a single surname
surname_list = [(surname, prefix, True, origintype, connector)]
else:
#a patronymic, if no surname or equal as patronymic, a single surname
if (surname.strip() == u"") or (surname == patronymic and prefix == u""):
surname_list = [(patronymic, prefix, True, patorigintype, connector)]
else:
#two surnames, first patronymic, then surname which is primary
surname_list = [(patronymic, u"", False, patorigintype, u""),
(surname, prefix, True, origintype, connector)]
#return new value, add two empty strings for nick and family nick
return (privacy, source_list, note_list, date,
first_name, surname_list, suffix, title, name_type,
group_as, sort_as, display_as, call, u"", u"")
def set_auto_remove(self):
"""
BSDDB change log settings using new method with renamed attributes
src/plugins/import/ImportXml.py
+159 -64
View File
@@ -96,9 +96,23 @@ def importData(database, filename, callback=None, cl=0):
database.smap = {}
database.pmap = {}
database.fmap = {}
xml_file = open_file(filename, cl)
versionparser = VersionParser(xml_file)
if xml_file is None or \
version_is_valid(versionparser, cl) is False:
if cl:
sys.exit(1)
else:
return
version_string = versionparser.get_xmlns_version()
#reset file to the start
xml_file.seek(0)
change = os.path.getmtime(filename)
parser = GrampsParser(database, callback, change)
parser = GrampsParser(database, callback, change, version_string)
linecounter = LineParser(filename)
line_cnt = linecounter.get_count()
@@ -107,17 +121,7 @@ def importData(database, filename, callback=None, cl=0):
read_only = database.readonly
database.readonly = False
xml_file = open_file(filename, cl)
if xml_file is None or \
version_is_valid(xml_file, cl) is False:
if cl:
sys.exit(1)
else:
return
try:
xml_file.seek(0)
info = parser.parse(xml_file, line_cnt, person_cnt)
except IOError, msg:
if cl:
@@ -348,8 +352,10 @@ class LineParser(object):
#-------------------------------------------------------------------------
class GrampsParser(UpdateCallback):
def __init__(self, database, callback, change):
def __init__(self, database, callback, change, version_string):
UpdateCallback.__init__(self, callback)
#version of the xml file
self.version_string = version_string
self.stext_list = []
self.scomments_list = []
self.note_list = []
@@ -423,6 +429,8 @@ class GrampsParser(UpdateCallback):
self.childref = None
self.personref = None
self.name = None
self.surname = None
self.surnamepat = None
self.home = None
self.owner = gen.lib.Researcher()
self.func_list = [None]*50
@@ -440,17 +448,33 @@ class GrampsParser(UpdateCallback):
self.eidswap = {}
self.func_map = {
#name part
"name": (self.start_name, self.stop_name),
"first": (None, self.stop_first),
"call": (None, self.stop_call),
"aka": (self.start_name, self.stop_aka), #deprecated < 1.3.0
"last": (self.start_last, self.stop_last), #deprecated in 1.4.0
"nick": (None, self.stop_nick),
"title": (None, self.stop_title),
"suffix": (None, self.stop_suffix),
"patronymic": (self.start_patronymic, self.stop_patronymic), #deprecated in 1.4.0
"familynick": (None, self.stop_familynick), #new in 1.4.0
"group": (None, self.stop_group), #new in 1.4.0, replaces attribute
#new in 1.4.0
"surname": (self.start_surname, self.stop_surname),
#
"namemaps": (None, None),
"name-formats": (None, None),
#other
"address": (self.start_address, self.stop_address),
"addresses": (None, None),
"childlist": (None, None),
"aka": (self.start_name, self.stop_aka),
"childlist": (None, None),
"attribute": (self.start_attribute, self.stop_attribute),
"attr_type": (None, self.stop_attr_type),
"attr_value": (None, self.stop_attr_value),
"bookmark": (self.start_bmark, None),
"bookmarks": (None, None),
"format": (self.start_format, None),
"name-formats": (None, None),
"child": (self.start_child, None),
"childof": (self.start_childof, None),
"childref": (self.start_childref, self.stop_childref),
@@ -476,17 +500,11 @@ class GrampsParser(UpdateCallback):
"rel": (self.start_rel, None),
"region": (self.start_region, None),
"father": (self.start_father, None),
"first": (None, self.stop_first),
"call": (None, self.stop_call),
"gender": (None, self.stop_gender),
"header": (None, None),
"last": (self.start_last, self.stop_last),
"map": (self.start_namemap, None),
"mediapath": (None, self.stop_mediapath),
"mother": (self.start_mother, None),
"name": (self.start_name, self.stop_name),
"namemaps": (None, None),
"nick": (None, self.stop_nick),
"note": (self.start_note, self.stop_note),
"noteref": (self.start_noteref, None),
"p": (None, self.stop_ptag),
@@ -511,7 +529,6 @@ class GrampsParser(UpdateCallback):
"status": (self.start_status, None),
"sealed_to": (self.start_sealed_to, None),
"coord": (self.start_coord, None),
"patronymic": (None, self.stop_patronymic),
"pos": (self.start_pos, None),
"postal": (None, self.stop_postal),
"range": (self.start_range, None),
@@ -536,13 +553,11 @@ class GrampsParser(UpdateCallback):
"stext": (None, self.stop_stext),
"stitle": (None, self.stop_stitle),
"street": (None, self.stop_street),
"style": (self.start_style, None),
"suffix": (None, self.stop_suffix),
"style": (self.start_style, None),
"tag": (self.start_tag, None),
"tagref": (self.start_tagref, None),
"tags": (None, None),
"text": (None, self.stop_text),
"title": (None, self.stop_title),
"url": (self.start_url, None),
"repository": (self.start_repo, self.stop_repo),
"reporef": (self.start_reporef, self.stop_reporef),
@@ -743,6 +758,11 @@ class GrampsParser(UpdateCallback):
return self.nidswap[gramps_id]
def parse(self, ifile, linecount=0, personcount=0):
"""
Parse the xml file
:param ifile: must be a file handle that is already open, with position
at the start of the file
"""
if personcount < 1000:
no_magic = True
else:
@@ -1305,6 +1325,14 @@ class GrampsParser(UpdateCallback):
except KeyError:
pass
def start_surname(self, attrs):
self.surname = gen.lib.Surname()
self.surname.set_prefix(attrs.get("prefix", ""))
self.surname.set_primary(bool(attrs.get("primary",0)))
self.surname.set_connector(attrs.get("connector", ""))
origin_type = attrs.get("derivation", "")
self.surname.origintype.set_from_xml_str(origin_type)
def start_namemap(self, attrs):
type = attrs.get('type')
key = attrs['key']
@@ -1321,9 +1349,17 @@ class GrampsParser(UpdateCallback):
self.db.set_name_group_mapping(key, value)
def start_last(self, attrs):
self.name.prefix = attrs.get('prefix', '')
""" This is the element in version < 1.4.0 to do the surname"""
self.surname = gen.lib.Surname()
self.surname.prefix = attrs.get('prefix', '')
self.name.group_as = attrs.get('group', '')
def start_patronymic(self, attrs):
""" This is the element in version < 1.4.0 to do the patronymic"""
self.surnamepat = gen.lib.Surname()
self.surnamepat.set_origintype(gen.lib.NameTypeOrigin(
gen.lib.NameTypeOrigin.PATRONYMIC))
def start_style(self, attrs):
"""
Styled text tag in notes (v1.4.0 onwards).
@@ -1927,14 +1963,8 @@ class GrampsParser(UpdateCallback):
self.num_places = 0
def start_database(self, attrs):
try:
# This is a proper way to get the XML version
xmlns = attrs.get('xmlns')
self.version_string = xmlns.split('/')[4]
except:
# Before we had a proper DTD, the version was hard to determine
# so we're setting it to 1.0.0
self.version_string = '1.0.0'
# we already parsed xml once in VersionParser to obtain version
pass
def start_pos(self, attrs):
self.person.position = (int(attrs["x"]), int(attrs["y"]))
@@ -2078,15 +2108,62 @@ class GrampsParser(UpdateCallback):
self.db.commit_note(note, self.trans, self.change)
self.info.add('new-object', NOTE_KEY, note)
self.event.add_note(note.handle)
elif self.alt_name:
# former aka tag -- alternate name
if self.name.get_type() == "":
self.name.set_type(gen.lib.NameType.AKA)
self.person.add_alternate_name(self.name)
else:
#first correct old xml that has no nametype set
if self.alt_name:
# alternate name or former aka tag
if self.name.get_type() == "":
self.name.set_type(gen.lib.NameType.AKA)
else:
if self.name.get_type() == "":
self.name.set_type(gen.lib.NameType.BIRTH)
#same logic as bsddb upgrade for xml < 1.4.0 which will
#have a surnamepat and/or surname. From 1.4.0 surname has been
#added to name in self.stop_surname
if not self.surnamepat:
#no patronymic, only add surname if present
if self.surname:
self.name.add_surname(self.surname)
self.name.set_primary_surname(0)
else:
#a patronymic, if no surname, a single surname
if not self.surname:
self.name.add_surname(self.surnamepat)
self.name.set_primary_surname(0)
else:
#two surnames, first patronymic, then surname which is primary
self.name.add_surname(self.surnamepat)
self.name.add_surname(self.surname)
self.name.set_primary_surname(1)
if self.alt_name:
self.person.add_alternate_name(self.name)
else:
self.person.set_primary_name(self.name)
self.name = None
self.surname = None
self.surnamepat = None
def stop_aka(self, tag):
if self.name.get_type() == "":
self.name.set_type(gen.lib.NameType.AKA)
if not self.surnamepat:
#no patronymic, only add surname if present
if self.surname:
self.name.add_surname(self.surname)
self.name.set_primary_surname(0)
else:
if self.name.get_type() == "":
self.name.set_type(gen.lib.NameType.BIRTH)
self.person.set_primary_name (self.name)
#a patronymic, if no surname, a single surname
if not self.surname:
self.name.add_surname(self.surnamepat)
self.name.set_primary_surname(0)
else:
#two surnames, first patronymic, then surname which is primary
self.name.add_surname(self.surnamepat)
self.name.add_surname(self.surname)
self.name.set_primary_surname(1)
self.person.add_alternate_name(self.name)
self.name = None
def stop_rname(self, tag):
@@ -2256,28 +2333,53 @@ class GrampsParser(UpdateCallback):
self.source_ref.add_note(note.handle)
def stop_last(self, tag):
if self.surname:
self.surname.set_surname(tag)
if not tag.strip() and not self.surname.get_prefix().strip():
#consider empty surname as no surname
self.surname = None
def stop_surname(self, tag):
if self.name:
self.name.set_surname(tag)
self.surname.set_surname(tag)
self.name.add_surname(self.surname)
self.surname = None
def stop_group(self, tag):
""" group name of a name"""
if self.name:
self.name.set_group_as(tag)
def stop_suffix(self, tag):
if self.name:
self.name.set_suffix(tag)
def stop_patronymic(self, tag):
if self.name:
self.name.set_patronymic(tag)
if self.surnamepat:
self.surnamepat.set_surname(tag)
if not tag.strip():
self.surnamepat = None
def stop_title(self, tag):
if self.name:
self.name.set_title(tag)
def stop_nick(self, tag):
if self.person:
"""in < 1.3.0 nick is on person and mapped to attribute
from 1.4.0 it is a name element
"""
if self.name:
self.name.set_nick_name(tag)
elif self.person:
attr = gen.lib.Attribute()
attr.set_type(gen.lib.AttributeType.NICKNAME)
attr.set_value(tag)
self.person.add_attribute(attr)
def stop_familynick(self, tag):
if self.name:
self.name.set_family_nick_name(tag)
def stop_text(self, tag):
self.note_text = tag
@@ -2379,12 +2481,6 @@ class GrampsParser(UpdateCallback):
elif self.in_scomments:
self.scomments_list.append(tag)
def stop_aka(self, tag):
self.person.add_alternate_name(self.name)
if self.name.get_type() == "":
self.name.set_type(gen.lib.NameType.AKA)
self.name = None
def startElement(self, tag, attrs):
self.func_list[self.func_index] = (self.func, self.tlist)
self.func_index += 1
@@ -2469,7 +2565,6 @@ class VersionParser(object):
self.__p.StartElementHandler = self.__element_handler
self.__gramps_version = 'unknown'
self.__xml_version = '1.0.0'
xml_file.seek(0)
self.__p.ParseFile(xml_file)
@@ -2541,25 +2636,25 @@ def open_file(filename, cli):
return xml_file
def version_is_valid(filename, cli):
def version_is_valid(versionparser, cli):
"""
Validate the xml version.
:param versionparser: A VersionParser object to work with
"""
parser = VersionParser(filename)
if parser.get_xmlns_version() > libgrampsxml.GRAMPS_XML_VERSION:
if versionparser.get_xmlns_version() > libgrampsxml.GRAMPS_XML_VERSION:
msg = _("The .gramps file you are importing was made by version %(newer)s of "
"Gramps, while you are running an older version %(older)s. "
"The file will not be imported. Please upgrade to the latest "
"version of Gramps and try again." ) % {
'newer' : parser.get_gramps_version(), 'older' : const.VERSION }
'newer' : versionparser.get_gramps_version(), 'older' : const.VERSION }
if cli:
LOG.warn(msg)
return False
else:
ErrorDialog(msg)
return False
if parser.get_xmlns_version() < '1.0.0':
if versionparser.get_xmlns_version() < '1.0.0':
msg = _("The .gramps file you are importing was made by version "
"%(oldgramps)s of Gramps, while you are running a more "
"recent version %(newgramps)s.\n\n"
@@ -2567,9 +2662,9 @@ def version_is_valid(filename, cli):
" Gramps that supports version %(xmlversion)s of the xml.\nSee"
"\n http://gramps-project.org/wiki/index.php?title=GRAMPS_XML\n "
"for more info."
) % {'oldgramps': parser.get_gramps_version(),
) % {'oldgramps': versionparser.get_gramps_version(),
'newgramps': const.VERSION,
'xmlversion': parser.get_xmlns_version(),
'xmlversion': versionparser.get_xmlns_version(),
}
if cli:
LOG.warn(msg)
@@ -2577,7 +2672,7 @@ def version_is_valid(filename, cli):
else:
ErrorDialog(_('The file will not be imported'), msg)
return False
elif parser.get_xmlns_version() < '1.1.0':
elif versionparser.get_xmlns_version() < '1.1.0':
msg = _("The .gramps file you are importing was made by version "
"%(oldgramps)s of Gramps, while you are running a much "
"more recent version %(newgramps)s.\n\n"
@@ -2587,9 +2682,9 @@ def version_is_valid(filename, cli):
"is version %(xmlversion)s of the xml.\nSee"
"\n http://gramps-project.org/wiki/index.php?title=GRAMPS_XML\n"
"for more info."
) % {'oldgramps': parser.get_gramps_version(),
) % {'oldgramps': versionparser.get_gramps_version(),
'newgramps': const.VERSION,
'xmlversion': parser.get_xmlns_version(),
'xmlversion': versionparser.get_xmlns_version(),
}
if cli:
LOG.warn(msg)