Cyclone-Team/gramps
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

0008537: Gedcom import crashes

Browse Source 
(1) Fix count of individuals in libgedcom/GedcomStageOne/parse so that
magic transactions are used when more than 1000 individuals.
(2) Remove use of find_backlink_handles in libgedcom so it doesn't
crash.
This commit is contained in:
kulath 2015-07-27 22:12:30 +01:00
parent 5c88ccfe7a
commit f96c6ce2a0
1 changed files with 69 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

168
src/plugins/lib/libgedcom.py
View File

@ -2832,22 +2832,31 @@ class GedcomParser(UpdateCallback):
return place return place
return None return None
def __create_place(self, title, location): def __add_place(self, event, sub_state):
""" """
Create a new place based on the title and primary location. Add a new place to an event if not already present, or update a
place.
@param title: The place title @param event: The event
@type title: string @type event: gen.lib.Event
@param location: The current location @param substate: The sub-state for PLAC or ADDR elements (i.e. parsed by
@type location: gen.lib.Location event_parse_tbl)
@return gen.lib.Place @type sub_state: CurrentState
""" """
place = gen.lib.Place() if sub_state.place:
place.set_title(title) # see whether this place already exists
place.set_main_location(location) place = self.__find_place(sub_state.place.get_title(),
sub_state.place.get_main_location())
if place is None:
place = sub_state.place
self.dbase.add_place(place, self.trans) self.dbase.add_place(place, self.trans)
self.place_names[title].append(place.get_handle()) self.place_names[place.get_title()].append(place.get_handle())
return place event.set_place_handle(place.get_handle())
else:
place.merge(sub_state.place)
self.dbase.commit_place(place, self.trans)
event.set_place_handle(place.get_handle())
sub_state.pf.load_place(place, place.get_title())
def __find_file(self, fullname, altpath): def __find_file(self, fullname, altpath):
tries = [] tries = []
@ -3754,10 +3763,13 @@ class GedcomParser(UpdateCallback):
sub_state.level = state.level+1 sub_state.level = state.level+1
sub_state.event = event sub_state.event = event
sub_state.event_ref = event_ref sub_state.event_ref = event_ref
sub_state.pf = self.place_parser
self.__parse_level(sub_state, self.event_parse_tbl, self.__undefined) self.__parse_level(sub_state, self.event_parse_tbl, self.__undefined)
state.msg += sub_state.msg state.msg += sub_state.msg
self.__add_place(event, sub_state)
self.dbase.commit_event(event, self.trans) self.dbase.commit_event(event, self.trans)
event_ref.ref = event.handle event_ref.ref = event.handle
state.person.add_event_ref(event_ref) state.person.add_event_ref(event_ref)
@ -3958,10 +3970,13 @@ class GedcomParser(UpdateCallback):
sub_state.level = state.level+1 sub_state.level = state.level+1
sub_state.event = event sub_state.event = event
sub_state.event_ref = event_ref sub_state.event_ref = event_ref
sub_state.pf = self.place_parser
self.__parse_level(sub_state, self.event_parse_tbl, self.__undefined) self.__parse_level(sub_state, self.event_parse_tbl, self.__undefined)
state.msg += sub_state.msg state.msg += sub_state.msg
self.__add_place(event, sub_state)
self.dbase.add_event(event, self.trans) self.dbase.add_event(event, self.trans)
event_ref.ref = event.handle event_ref.ref = event.handle
state.person.add_event_ref(event_ref) state.person.add_event_ref(event_ref)
@ -4408,11 +4423,14 @@ class GedcomParser(UpdateCallback):
try: try:
title = line.data title = line.data
place = self.__find_place(title, None) place = self.__find_place(title, None)
if place: if place is None:
state.place = place place = gen.lib.Place()
place.set_title(title)
self.dbase.add_place(place, self.trans)
self.place_names[place.get_title()].append(place.get_handle())
else: else:
state.place = self.__create_place(title, None) pass
state.lds_ord.set_place_handle(state.place.handle) state.lds_ord.set_place_handle(place.handle)
except NameError: except NameError:
return return
@ -4773,10 +4791,13 @@ class GedcomParser(UpdateCallback):
sub_state.level = state.level+1 sub_state.level = state.level+1
sub_state.event = event sub_state.event = event
sub_state.event_ref = event_ref sub_state.event_ref = event_ref
sub_state.pf = self.place_parser
self.__parse_level(sub_state, self.event_parse_tbl, self.__undefined) self.__parse_level(sub_state, self.event_parse_tbl, self.__undefined)
state.msg += sub_state.msg state.msg += sub_state.msg
self.__add_place(event, sub_state)
if event.type == gen.lib.EventType.MARRIAGE: if event.type == gen.lib.EventType.MARRIAGE:
descr = event.get_description() descr = event.get_description()
if descr == "Civil Union": if descr == "Civil Union":
@ -4816,10 +4837,13 @@ class GedcomParser(UpdateCallback):
sub_state.level = state.level+1 sub_state.level = state.level+1
sub_state.event = event sub_state.event = event
sub_state.event_ref = event_ref sub_state.event_ref = event_ref
sub_state.pf = self.place_parser
self.__parse_level(sub_state, self.event_parse_tbl, self.__undefined) self.__parse_level(sub_state, self.event_parse_tbl, self.__undefined)
state.msg += sub_state.msg state.msg += sub_state.msg
self.__add_place(event, sub_state)
self.dbase.commit_event(event, self.trans) self.dbase.commit_event(event, self.trans)
event_ref.ref = event.handle event_ref.ref = event.handle
state.family.add_event_ref(event_ref) state.family.add_event_ref(event_ref)
@ -5241,74 +5265,33 @@ class GedcomParser(UpdateCallback):
@type state: CurrentState @type state: CurrentState
""" """
location = None
if self.is_ftw and state.event.type in FTW_BAD_PLACE: if self.is_ftw and state.event.type in FTW_BAD_PLACE:
state.event.set_description(line.data) state.event.set_description(line.data)
else: else:
title = line.data place = state.place
place_handle = state.event.get_place_handle() if place:
if place_handle:
# We encounter a PLAC, having previously encountered an ADDR # We encounter a PLAC, having previously encountered an ADDR
old_place = self.dbase.get_place_from_handle(place_handle) if place.get_title() and place.get_title() != "":
old_title = old_place.get_title()
location = old_place.get_main_location()
if old_title != "":
# We have previously found a PLAC # We have previously found a PLAC
self.__add_msg(_("A second PLAC ignored"), line, state) self.__add_msg(_("A second PLAC ignored"), line, state)
# ignore this second PLAC, and use the old one # ignore this second PLAC, and use the old one
title = old_title
place = old_place
else: else:
# This is the first PLAC # This is the first PLAC
refs = list(self.dbase.find_backlink_handles(place_handle)) place.set_title(line.data)
# We haven't commited the event yet, so the place will not
# be linked to it. If there are any refs they will be from
# other events (etc)
if len(refs) == 0:
place = self.__find_place(title, location)
if place is None:
place = old_place
place.set_title(title)
self.place_names[old_title].remove(place_handle)
self.place_names[title].append(place_handle)
else:
place.merge(old_place)
self.dbase.remove_place(place_handle, self.trans)
self.place_names[old_title].remove(place_handle)
else:
place = self.__find_place(title, location)
if place is None:
place = self.__create_place(title, location)
else:
pass
else: else:
# The first thing we encounter is PLAC # The first thing we encounter is PLAC
location = None state.place = gen.lib.Place()
place = self.__find_place(title, location) place = state.place
if place is None: place.set_title(line.data)
place = self.__create_place(title, location)
state.event.set_place_handle(place.handle)
sub_state = CurrentState() sub_state = CurrentState()
sub_state.place = place sub_state.place = place
sub_state.level = state.level+1 sub_state.level = state.level+1
sub_state.pf = self.place_parser
self.__parse_level(sub_state, self.event_place_map, self.__parse_level(sub_state, self.event_place_map,
self.__undefined) self.__undefined)
state.msg += sub_state.msg state.msg += sub_state.msg
sub_state.pf.load_place(place, place.get_title())
# If we already had a remembered location, we set it into the main
# location if that is empty, else the alternate location
if location and not location.is_empty():
if place.get_main_location().is_empty():
place.set_main_location(location)
else:
place.add_alternate_locations(location)
self.dbase.commit_place(place, self.trans)
def __event_place_note(self, line, state): def __event_place_note(self, line, state):
""" """
@param line: The current line in GedLine format @param line: The current line in GedLine format
@ -5428,53 +5411,29 @@ class GedcomParser(UpdateCallback):
self.__merge_address(free_form, sub_state.location, line, state) self.__merge_address(free_form, sub_state.location, line, state)
location = sub_state.location location = sub_state.location
place_handle = state.event.get_place_handle()
if place_handle: place = state.place
if place:
# We encounter an ADDR having previously encountered a PLAC # We encounter an ADDR having previously encountered a PLAC
old_place = self.dbase.get_place_from_handle(place_handle) if len(place.get_alternate_locations()) != 0 and \
title = old_place.get_title() not place.get_main_location().is_empty():
if not old_place.get_main_location().is_empty():
# We have perviously found an ADDR, or have populated location # We have perviously found an ADDR, or have populated location
# from PLAC title # from PLAC title
self.__add_msg(_("Location already populated; ADDR ignored"), self.__add_msg(_("Location already populated; ADDR ignored"),
line, state) line, state)
# ignore this second ADDR, and use the old one # ignore this second ADDR, and use the old one
location = old_place.get_main_location()
place = old_place
else: else:
# This is the first ADDR # This is the first ADDR
refs = list(self.dbase.find_backlink_handles(place_handle))
# We haven't commited the event yet, so the place will not be
# linked to it. If there are any refs they will be from other
# events (etc)
if len(refs) == 0:
place = self.__find_place(title, location)
if place is None:
place = old_place
place.set_main_location(location) place.set_main_location(location)
else:
place.merge(old_place)
self.dbase.remove_place(place_handle, self.trans)
self.place_names[title].remove(place_handle)
else:
place = self.__find_place(title, location)
if place is None:
place = self.__create_place(title, location)
else:
pass
else: else:
# The first thing we encounter is ADDR # The first thing we encounter is ADDR
title = "" state.place = gen.lib.Place()
place = self.__find_place(title, location) place = state.place
if place is None: place.set_main_location(location)
place = self.__create_place(title, location)
# merge notes etc into place # merge notes etc into place
place.merge(sub_state.place) place.merge(sub_state.place)
state.event.set_place_handle(place.get_handle())
self.dbase.commit_place(place, self.trans)
def __add_location(self, place, location): def __add_location(self, place, location):
""" """
@param place: A place object we have found or created @param place: A place object we have found or created
@ -5502,12 +5461,10 @@ class GedcomParser(UpdateCallback):
@param state: The current state @param state: The current state
@type state: CurrentState @type state: CurrentState
""" """
place_handle = state.event.get_place_handle() place = state.place
if place_handle: if place:
place = self.dbase.get_place_from_handle(place_handle)
location = place.get_main_location() location = place.get_main_location()
location.set_phone(line.data) location.set_phone(line.data)
self.dbase.commit_place(place, self.trans)
def __event_privacy(self, line, state): def __event_privacy(self, line, state):
""" """
@ -7412,9 +7369,13 @@ class GedcomParser(UpdateCallback):
sub_state.event_ref = event_ref sub_state.event_ref = event_ref
sub_state.event = event sub_state.event = event
sub_state.person = state.person sub_state.person = state.person
sub_state.pf = self.place_parser
self.__parse_level(sub_state, event_map, self.__undefined) self.__parse_level(sub_state, event_map, self.__undefined)
state.msg += sub_state.msg state.msg += sub_state.msg
self.__add_place(event, sub_state)
self.dbase.commit_event(event, self.trans) self.dbase.commit_event(event, self.trans)
event_ref.set_reference_handle(event.handle) event_ref.set_reference_handle(event.handle)
@ -7436,10 +7397,13 @@ class GedcomParser(UpdateCallback):
sub_state.level = state.level+1 sub_state.level = state.level+1
sub_state.event = event sub_state.event = event
sub_state.event_ref = event_ref sub_state.event_ref = event_ref
sub_state.pf = self.place_parser
self.__parse_level(sub_state, event_map, self.__undefined) self.__parse_level(sub_state, event_map, self.__undefined)
state.msg += sub_state.msg state.msg += sub_state.msg
self.__add_place(event, sub_state)
self.dbase.commit_event(event, self.trans) self.dbase.commit_event(event, self.trans)
event_ref.set_reference_handle(event.handle) event_ref.set_reference_handle(event.handle)
return event_ref return event_ref
@ -7624,9 +7588,9 @@ class GedcomStageOne(object):
continue continue
if level == 0 and key[0] == '@': if level == 0 and key[0] == '@':
if value == ("FAM", "FAMILY") : if value in ("FAM", "FAMILY") :
current_family_id = key.strip()[1:-1] current_family_id = key.strip()[1:-1]
elif value == ("INDI", "INDIVIDUAL"): elif value in ("INDI", "INDIVIDUAL"):
self.pcnt += 1 self.pcnt += 1
elif key in ("HUSB", "HUSBAND", "WIFE") and \ elif key in ("HUSB", "HUSBAND", "WIFE") and \
self.__is_xref_value(value): self.__is_xref_value(value):