Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modifications to upgrade code as suggested by Michiel Nauta; many thanks for his advice.

Browse Source 
svn: r18096
This commit is contained in:
Tim G L Lyons 2011-09-01 22:07:04 +00:00
parent 29cc4ee6e9
commit bad963e150
2 changed files with 164 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

364
src/gen/db/upgrade.py
View File

@ -36,7 +36,7 @@ if config.get('preferences.use-bsddb3'):
from bsddb3 import db from bsddb3 import db
else: else:
from bsddb import db from bsddb import db
from gen.db import BSDDBTxn, DbTxn from gen.db import BSDDBTxn
from gen.lib.nameorigintype import NameOriginType from gen.lib.nameorigintype import NameOriginType
from gen.db.write import _mkname, SURNAMES from gen.db.write import _mkname, SURNAMES
from gen.db.dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY, from gen.db.dbconst import (PERSON_KEY, FAMILY_KEY, EVENT_KEY,
@ -94,55 +94,48 @@ def gramps_upgrade_16(self):
start_time = time.time() start_time = time.time()
for person_handle in self.person_map.keys(): for person_handle in self.person_map.keys():
person = self.person_map[person_handle] person = self.person_map[person_handle]
with DbTxn(_("convert a person record"), self, batch=True, (handle, gramps_id, gender, primary_name, alternate_names,
no_magic=True) as transaction: death_ref_index, birth_ref_index, event_ref_list, family_list,
(handle, gramps_id, gender, primary_name, alternate_names, parent_family_list, media_list, address_list, attribute_list,
death_ref_index, birth_ref_index, event_ref_list, family_list, urls, lds_seal_list, source_list, note_list, change, tag_list,
parent_family_list, media_list, address_list, attribute_list, private, person_ref_list) = person
urls, lds_seal_list, source_list, note_list, change, tag_list, if primary_name:
private, person_ref_list) = person primary_name = upgrade_name_16(self, primary_name)
if primary_name: if alternate_names:
primary_name = upgrade_name_16(self, primary_name, transaction) alternate_names = upgrade_name_list_16(
if alternate_names: self, alternate_names)
alternate_names = upgrade_name_list_16( if address_list:
self, alternate_names, transaction) address_list = upgrade_address_list_16(
if address_list: self, address_list)
address_list = upgrade_address_list_16( if media_list:
self, address_list, transaction) media_list = upgrade_media_list_16(
if media_list: self, media_list)
media_list = upgrade_media_list_16( if attribute_list:
self, media_list, transaction) attribute_list = upgrade_attribute_list_16(
if attribute_list: self, attribute_list)
attribute_list = upgrade_attribute_list_16( if lds_seal_list:
self, attribute_list, transaction) lds_seal_list = upgrade_lds_seal_list_16(
if lds_seal_list: self, lds_seal_list)
lds_seal_list = upgrade_lds_seal_list_16( if source_list:
self, lds_seal_list, transaction) new_citation_list = convert_source_list_to_citation_list_16(
if source_list: self, source_list)
new_citation_list = convert_source_list_to_citation_list_16( else:
self, source_list, transaction) new_citation_list = []
else: if person_ref_list:
new_citation_list = [] person_ref_list = upgrade_person_ref_list_16(
if person_ref_list: self, person_ref_list)
person_ref_list = upgrade_person_ref_list_16( if primary_name or alternate_names or address_list or \
self, person_ref_list, transaction) media_list or attribute_list or lds_seal_list or source_list or \
if primary_name or alternate_names or address_list or \ person_ref_list:
media_list or attribute_list or lds_seal_list or source_list or \ new_person = (handle, gramps_id, gender, primary_name,
person_ref_list: alternate_names, death_ref_index,
new_person = (handle, gramps_id, gender, primary_name, birth_ref_index, event_ref_list, family_list,
alternate_names, death_ref_index, parent_family_list, media_list, address_list,
birth_ref_index, event_ref_list, family_list, attribute_list, urls, lds_seal_list,
parent_family_list, media_list, address_list, new_citation_list, note_list, change, tag_list,
attribute_list, urls, lds_seal_list, private, person_ref_list)
new_citation_list, note_list, change, tag_list, with BSDDBTxn(self.env, self.person_map) as txn:
private, person_ref_list) txn.put(str(handle), new_person)
with BSDDBTxn(self.env, self.person_map) as txn:
txn.put(str(handle), new_person, txn=transaction)
# with BSDDBTxn(self.env) as txn:
# self.update_reference_map(
# self.get_person_from_handle(handle),
# transaction,
# txn.txn)
self.update() self.update()
LOG.debug("%d persons upgraded with %d citations in %d seconds. " % LOG.debug("%d persons upgraded with %d citations in %d seconds. " %
@ -161,43 +154,22 @@ def gramps_upgrade_16(self):
for media_handle in self.media_map.keys(): for media_handle in self.media_map.keys():
media = self.media_map[media_handle] media = self.media_map[media_handle]
LOG.debug("upgrade media %s" % media[4]) LOG.debug("upgrade media %s" % media[4])
with DbTxn(_("convert a media record"), self, batch=True, (handle, gramps_id, path, mime, desc,
no_magic=True) as transaction: attribute_list, source_list, note_list, change,
# FIXME: This should be a single transaction, so that date, tag_list, private) = media
# either the whole of the media object is updated or none is new_citation_list = convert_source_list_to_citation_list_16(
# but it doesn't seem to work like that because if self, source_list)
# update_refernce_map fails, the put of the new_media new_attribute_list = upgrade_attribute_list_16(
# remains committed. self, attribute_list)
# (1) create each citation
# (2) update the Media to reference the Citations
# (3) remove backlinks for references from Media to Source
# (4) add backlinks for references from Media to Citations
# (5) add backlinks for references from Citation to Source
(handle, gramps_id, path, mime, desc,
attribute_list, source_list, note_list, change,
date, tag_list, private) = media
new_citation_list = convert_source_list_to_citation_list_16(
self, source_list, transaction)
new_attribute_list = upgrade_attribute_list_16(
self, attribute_list, transaction)
new_media = (handle, gramps_id, path, mime, desc,
new_attribute_list, new_citation_list, note_list,
change, date, tag_list, private)
LOG.debug(" upgrade new_media %s" % [new_media])
with BSDDBTxn(self.env, self.media_map) as txn:
txn.put(str(handle), new_media, txn=transaction)
# (3) remove backlinks for references from Media to Source new_media = (handle, gramps_id, path, mime, desc,
# (4) add backlinks for references from Media to Citations new_attribute_list, new_citation_list, note_list,
# (get_object is really get_MediaObject !) change, date, tag_list, private)
LOG.debug(" update ref map media %s" % [handle, LOG.debug(" upgrade new_media %s" % [new_media])
self.get_object_from_handle(handle) ]) with BSDDBTxn(self.env, self.media_map) as txn:
# with BSDDBTxn(self.env) as txn: txn.put(str(handle), new_media)
# self.update_reference_map( LOG.debug(" update ref map media %s" % [handle,
# self.get_object_from_handle(handle), self.get_object_from_handle(handle) ])
# transaction,
# txn.txn)
self.update() self.update()
LOG.debug("Media upgrade %d citations upgraded in %d seconds" % LOG.debug("Media upgrade %d citations upgraded in %d seconds" %
@ -214,31 +186,24 @@ def gramps_upgrade_16(self):
start_time = time.time() start_time = time.time()
for place_handle in self.place_map.keys(): for place_handle in self.place_map.keys():
place = self.place_map[place_handle] place = self.place_map[place_handle]
with DbTxn(_("convert a place record"), self, batch=True, (handle, gramps_id, title, long, lat,
no_magic=True) as transaction: main_loc, alt_loc, urls, media_list, source_list, note_list,
(handle, gramps_id, title, long, lat, change, private) = place
main_loc, alt_loc, urls, media_list, source_list, note_list, if source_list:
change, private) = place new_citation_list = convert_source_list_to_citation_list_16(
if source_list: self, source_list)
new_citation_list = convert_source_list_to_citation_list_16( else:
self, source_list, transaction) new_citation_list = []
else: if media_list:
new_citation_list = [] media_list = upgrade_media_list_16(
if media_list: self, media_list)
media_list = upgrade_media_list_16( if source_list or media_list:
self, media_list, transaction) new_place = (handle, gramps_id, title,
if source_list or media_list: long, lat, main_loc, alt_loc, urls,
new_place = (handle, gramps_id, title, media_list, new_citation_list, note_list,
long, lat, main_loc, alt_loc, urls, change, private)
media_list, new_citation_list, note_list, with BSDDBTxn(self.env, self.place_map) as txn:
change, private) txn.put(str(handle), new_place)
with BSDDBTxn(self.env, self.place_map) as txn:
txn.put(str(handle), new_place, txn=transaction)
# with BSDDBTxn(self.env) as txn:
# self.update_reference_map(
# self.get_place_from_handle(handle),
# transaction,
# txn.txn)
self.update() self.update()
LOG.debug("%d places upgraded with %d citations in %d seconds. " % LOG.debug("%d places upgraded with %d citations in %d seconds. " %
@ -256,42 +221,35 @@ def gramps_upgrade_16(self):
start_time = time.time() start_time = time.time()
for family_handle in self.family_map.keys(): for family_handle in self.family_map.keys():
family = self.family_map[family_handle] family = self.family_map[family_handle]
with DbTxn(_("convert a family record"), self, batch=True, (handle, gramps_id, father_handle, mother_handle,
no_magic=True) as transaction: child_ref_list, the_type, event_ref_list, media_list,
(handle, gramps_id, father_handle, mother_handle, attribute_list, lds_seal_list, source_list, note_list,
child_ref_list, the_type, event_ref_list, media_list, change, tag_list, private) = family
attribute_list, lds_seal_list, source_list, note_list, if source_list:
change, tag_list, private) = family new_citation_list = convert_source_list_to_citation_list_16(
if source_list: self, source_list)
new_citation_list = convert_source_list_to_citation_list_16( else:
self, source_list, transaction) new_citation_list = []
else: if child_ref_list:
new_citation_list = [] child_ref_list = upgrade_child_ref_list_16(
if child_ref_list: self, child_ref_list)
child_ref_list = upgrade_child_ref_list_16( if lds_seal_list:
self, child_ref_list, transaction) lds_seal_list = upgrade_lds_seal_list_16(
if lds_seal_list: self, lds_seal_list)
lds_seal_list = upgrade_lds_seal_list_16( if media_list:
self, lds_seal_list, transaction) media_list = upgrade_media_list_16(
if media_list: self, media_list)
media_list = upgrade_media_list_16( if attribute_list:
self, media_list, transaction) attribute_list = upgrade_attribute_list_16(
if attribute_list: self, attribute_list)
attribute_list = upgrade_attribute_list_16( if source_list or media_list or child_ref_list or \
self, attribute_list, transaction) attribute_list or lds_seal_list:
if source_list or media_list or child_ref_list or \ new_family = (handle, gramps_id, father_handle, mother_handle,
attribute_list or lds_seal_list: child_ref_list, the_type, event_ref_list, media_list,
new_family = (handle, gramps_id, father_handle, mother_handle, attribute_list, lds_seal_list, new_citation_list, note_list,
child_ref_list, the_type, event_ref_list, media_list, change, tag_list, private)
attribute_list, lds_seal_list, new_citation_list, note_list, with BSDDBTxn(self.env, self.family_map) as txn:
change, tag_list, private) txn.put(str(handle), new_family)
with BSDDBTxn(self.env, self.family_map) as txn:
txn.put(str(handle), new_family, txn=transaction)
# with BSDDBTxn(self.env) as txn:
# self.update_reference_map(
# self.get_family_from_handle(handle),
# transaction,
# txn.txn)
self.update() self.update()
LOG.debug("%d familys upgraded with %d citations in %d seconds. " % LOG.debug("%d familys upgraded with %d citations in %d seconds. " %
@ -311,44 +269,31 @@ def gramps_upgrade_16(self):
for event_handle in self.event_map.keys(): for event_handle in self.event_map.keys():
t1 = time.time() t1 = time.time()
event = self.event_map[event_handle] event = self.event_map[event_handle]
with DbTxn(_("convert a media record"), self, batch=True, (handle, gramps_id, the_type, date, description, place,
no_magic=True) as transaction: source_list, note_list, media_list, attribute_list,
(handle, gramps_id, the_type, date, description, place, change, private) = event
source_list, note_list, media_list, attribute_list, if source_list:
change, private) = event new_citation_list = convert_source_list_to_citation_list_16(
if source_list: self, source_list)
new_citation_list = convert_source_list_to_citation_list_16( else:
self, source_list, transaction) new_citation_list = []
else: if attribute_list:
new_citation_list = [] attribute_list = upgrade_attribute_list_16(
if attribute_list: self, attribute_list)
attribute_list = upgrade_attribute_list_16( if media_list:
self, attribute_list, transaction) media_list = upgrade_media_list_16(
if media_list: self, media_list)
media_list = upgrade_media_list_16( if source_list or attribute_list or media_list:
self, media_list, transaction) new_event = (handle, gramps_id, the_type, date, description, place,
if source_list or attribute_list or media_list: new_citation_list, note_list, media_list,
new_event = (handle, gramps_id, the_type, date, description, place, attribute_list,
new_citation_list, note_list, media_list, change, private)
attribute_list, with BSDDBTxn(self.env, self.event_map) as txn:
change, private) txn.put(str(handle), new_event)
# LOG.debug(" upgrade new_event %s" % [new_event]) t2 = time.time()
with BSDDBTxn(self.env, self.event_map) as txn: upgrade_time += t2 - t1
txn.put(str(handle), new_event, txn=transaction) t3 = time.time()
t2 = time.time() backlink_time += t3 - t2
upgrade_time += t2 - t1
# remove backlinks for references from Media to Source
# add backlinks for references from Media to Citations
# if source_list or attribute_list or media_list:
# LOG.debug(" upgrade backlinks %s" %
# [source_list, attribute_list, media_list])
# with BSDDBTxn(self.env) as txn:
# self.update_reference_map(
# self.get_event_from_handle(handle),
# transaction,
# txn.txn)
t3 = time.time()
backlink_time += t3 - t2
self.update() self.update()
LOG.debug("%d events upgraded with %d citations in %d seconds. " LOG.debug("%d events upgraded with %d citations in %d seconds. "
@ -413,6 +358,17 @@ def gramps_upgrade_16(self):
# 7 Media Objects upgraded with 4 citations in 2 secs # 7 Media Objects upgraded with 4 citations in 2 secs
# 852 Places upgraded with 0 citations in 1 secs # 852 Places upgraded with 0 citations in 1 secs
# without incorrect nestetd tranaction structure:
#Number of new objects upgraded:
# 73 People upgraded with 76 citations in 0 secs
# 35 Families upgraded with 36 citations in 0 secs
# 3403 Events upgraded with 4 citations in 0 secs
# 7 Media Objects upgraded with 4 citations in 0 secs
# 852 Places upgraded with 0 citations in 0 secs
#[[(73, 76, 0.12430405616760254), (35, 36, 0.042523860931396484), (3403, 4, 0.52303886413574219), (7, 4, 0.058229923248291016), (852, 0, 0.14816904067993164)]]
@ -430,92 +386,92 @@ def gramps_upgrade_16(self):
txt += key2string[key] txt += key2string[key]
InfoDialog(_('Upgrade Statistics'), txt) InfoDialog(_('Upgrade Statistics'), txt)
def upgrade_media_list_16(self, media_list, transaction): def upgrade_media_list_16(self, media_list):
new_media_list = [] new_media_list = []
for media in media_list: for media in media_list:
(privacy, source_list, note_list, attribute_list, ref, rect) = media (privacy, source_list, note_list, attribute_list, ref, rect) = media
new_citation_list = convert_source_list_to_citation_list_16( new_citation_list = convert_source_list_to_citation_list_16(
self, source_list, transaction) self, source_list)
new_attribute_list = upgrade_attribute_list_16( new_attribute_list = upgrade_attribute_list_16(
self, attribute_list, transaction) self, attribute_list)
new_media = (privacy, new_citation_list, note_list, new_attribute_list, new_media = (privacy, new_citation_list, note_list, new_attribute_list,
ref, rect) ref, rect)
new_media_list.append((new_media)) new_media_list.append((new_media))
return new_media_list return new_media_list
def upgrade_attribute_list_16(self, attribute_list, transaction): def upgrade_attribute_list_16(self, attribute_list):
new_attribute_list = [] new_attribute_list = []
for attribute in attribute_list: for attribute in attribute_list:
(privacy, source_list, note_list, the_type, (privacy, source_list, note_list, the_type,
value) = attribute value) = attribute
new_citation_list = convert_source_list_to_citation_list_16( new_citation_list = convert_source_list_to_citation_list_16(
self, source_list, transaction) self, source_list)
new_attribute = (privacy, new_citation_list, note_list, new_attribute = (privacy, new_citation_list, note_list,
the_type, value) the_type, value)
new_attribute_list.append((new_attribute)) new_attribute_list.append((new_attribute))
return new_attribute_list return new_attribute_list
def upgrade_child_ref_list_16(self, child_ref_list, transaction): def upgrade_child_ref_list_16(self, child_ref_list):
new_child_ref_list = [] new_child_ref_list = []
for child_ref in child_ref_list: for child_ref in child_ref_list:
(privacy, source_list, note_list, ref, frel, mrel) = child_ref (privacy, source_list, note_list, ref, frel, mrel) = child_ref
new_citation_list = convert_source_list_to_citation_list_16( new_citation_list = convert_source_list_to_citation_list_16(
self, source_list, transaction) self, source_list)
new_child_ref = (privacy, new_citation_list, note_list, ref, frel, mrel) new_child_ref = (privacy, new_citation_list, note_list, ref, frel, mrel)
new_child_ref_list.append((new_child_ref)) new_child_ref_list.append((new_child_ref))
return new_child_ref_list return new_child_ref_list
def upgrade_lds_seal_list_16(self, lds_seal_list, transaction): def upgrade_lds_seal_list_16(self, lds_seal_list):
new_lds_seal_list = [] new_lds_seal_list = []
for lds_seal in lds_seal_list: for lds_seal in lds_seal_list:
(source_list, note_list, date, type, place, (source_list, note_list, date, type, place,
famc, temple, status, private) = lds_seal famc, temple, status, private) = lds_seal
new_citation_list = convert_source_list_to_citation_list_16( new_citation_list = convert_source_list_to_citation_list_16(
self, source_list, transaction) self, source_list)
new_lds_seal = (new_citation_list, note_list, date, type, place, new_lds_seal = (new_citation_list, note_list, date, type, place,
famc, temple, status, private) famc, temple, status, private)
new_lds_seal_list.append((new_lds_seal)) new_lds_seal_list.append((new_lds_seal))
return new_lds_seal_list return new_lds_seal_list
def upgrade_address_list_16(self, address_list, transaction): def upgrade_address_list_16(self, address_list):
new_address_list = [] new_address_list = []
for address in address_list: for address in address_list:
(privacy, source_list, note_list, date, location) = address (privacy, source_list, note_list, date, location) = address
new_citation_list = convert_source_list_to_citation_list_16( new_citation_list = convert_source_list_to_citation_list_16(
self, source_list, transaction) self, source_list)
new_address = (privacy, new_citation_list, note_list, date, location) new_address = (privacy, new_citation_list, note_list, date, location)
new_address_list.append((new_address)) new_address_list.append((new_address))
return new_address_list return new_address_list
def upgrade_name_list_16(self, name_list, transaction): def upgrade_name_list_16(self, name_list):
new_name_list = [] new_name_list = []
for name in name_list: for name in name_list:
new_name = upgrade_name_16(self, name, transaction) new_name = upgrade_name_16(self, name)
new_name_list.append((new_name)) new_name_list.append((new_name))
return new_name_list return new_name_list
def upgrade_name_16(self, name, transaction): def upgrade_name_16(self, name):
(privacy, source_list, note, date, first_name, surname_list, suffix, (privacy, source_list, note, date, first_name, surname_list, suffix,
title, name_type, group_as, sort_as, display_as, call, nick, title, name_type, group_as, sort_as, display_as, call, nick,
famnick) = name famnick) = name
new_citation_list = convert_source_list_to_citation_list_16( new_citation_list = convert_source_list_to_citation_list_16(
self, source_list, transaction) self, source_list)
new_name = (privacy, new_citation_list, note, date, first_name, new_name = (privacy, new_citation_list, note, date, first_name,
surname_list, suffix, title, name_type, group_as, sort_as, surname_list, suffix, title, name_type, group_as, sort_as,
display_as, call, nick, famnick) display_as, call, nick, famnick)
return new_name return new_name
def upgrade_person_ref_list_16(self, person_ref_list, transaction): def upgrade_person_ref_list_16(self, person_ref_list):
new_person_ref_list = [] new_person_ref_list = []
for person_ref in person_ref_list: for person_ref in person_ref_list:
(privacy, source_list, note_list, ref, rel) = person_ref (privacy, source_list, note_list, ref, rel) = person_ref
new_citation_list = convert_source_list_to_citation_list_16( new_citation_list = convert_source_list_to_citation_list_16(
self, source_list, transaction) self, source_list)
new_person_ref = (privacy, new_citation_list, note_list, ref, rel) new_person_ref = (privacy, new_citation_list, note_list, ref, rel)
new_person_ref_list.append((new_person_ref)) new_person_ref_list.append((new_person_ref))
return new_person_ref_list return new_person_ref_list
def convert_source_list_to_citation_list_16(self, source_list, transaction): def convert_source_list_to_citation_list_16(self, source_list):
citation_list = [] citation_list = []
for source in source_list: for source in source_list:
(date, private, note_list, confidence, ref, page) = source (date, private, note_list, confidence, ref, page) = source
@ -528,7 +484,7 @@ def convert_source_list_to_citation_list_16(self, source_list, transaction):
date, page, confidence, ref, note_list, new_media_list, date, page, confidence, ref, note_list, new_media_list,
new_data_map, new_change, private) new_data_map, new_change, private)
with BSDDBTxn(self.env, self.citation_map) as txn: with BSDDBTxn(self.env, self.citation_map) as txn:
txn.put(str(new_handle), new_citation, txn=transaction) txn.put(str(new_handle), new_citation)
self.cmap_index += 1 self.cmap_index += 1
# # add backlinks for references from Citation to Source # # add backlinks for references from Citation to Source
# with BSDDBTxn(self.env) as txn: # with BSDDBTxn(self.env) as txn:

7
src/gen/db/write.py
View File

@ -1875,13 +1875,14 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
if version < 15: if version < 15:
upgrade.gramps_upgrade_15(self) upgrade.gramps_upgrade_15(self)
if version < 16: if version < 16:
upgrade.gramps_upgrade_16(self)
self.reset()
self.set_total(6)
self.__connect_secondary() self.__connect_secondary()
# Open undo database # Open undo database
self.__open_undodb() self.__open_undodb()
self.db_is_open = True self.db_is_open = True
upgrade.gramps_upgrade_16(self)
self.reset()
self.set_total(6)
self.reindex_reference_map(self.update) self.reindex_reference_map(self.update)
self.reset() self.reset()
# Close undo database # Close undo database