From cd4afcdb32fb6e2c8802ab2ca53a5eb47f6dabdd Mon Sep 17 00:00:00 2001 From: prculley Date: Tue, 25 Oct 2016 12:07:44 -0500 Subject: [PATCH] Fix bug 9733, CSV import Fails, 9676, failure for certain types of cross references etc. --- gramps/plugins/importer/importcsv.py | 251 +++++++++++++-------------- 1 file changed, 123 insertions(+), 128 deletions(-) diff --git a/gramps/plugins/importer/importcsv.py b/gramps/plugins/importer/importcsv.py index b1438f385..0dee09497 100644 --- a/gramps/plugins/importer/importcsv.py +++ b/gramps/plugins/importer/importcsv.py @@ -44,15 +44,15 @@ LOG = logging.getLogger(".ImportCSV") #------------------------------------------------------------------------- # -# GRAMPS modules +# Gramps modules # #------------------------------------------------------------------------- from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.sgettext ngettext = glocale.translation.ngettext # else "nearby" comments are ignored -from gramps.gen.lib import (ChildRef, Citation, Event, EventRef, EventType, - Family, FamilyRelType, Name, NameType, Note, - NoteType, Person, Place, Source, Surname, Tag, +from gramps.gen.lib import (ChildRef, Citation, Event, EventRef, EventType, + Family, FamilyRelType, Name, NameType, Note, + NoteType, Person, Place, Source, Surname, Tag, PlaceName, PlaceType, PlaceRef) from gramps.gen.db import DbTxn from gramps.gen.datehandler import parser as _dp @@ -89,7 +89,7 @@ def rd(line_number, row, col, key, default = None): """ Return Row data by column name """ if key in col: if col[key] >= len(row): - LOG.warn("missing '%s, on line %d" % (key, line_number)) + LOG.warning("missing '%s, on line %d" % (key, line_number)) return default retval = row[col[key]].strip() if retval == "": @@ -104,10 +104,10 @@ def importData(dbase, filename, user): if dbase.get_feature("skip-import-additions"): # don't add source or tags parser = CSVParser(dbase, user, None) else: - parser = CSVParser(dbase, user, (config.get('preferences.tag-on-import-format') if + parser = CSVParser(dbase, user, (config.get('preferences.tag-on-import-format') if config.get('preferences.tag-on-import') else None)) try: - with open(filename, 'r') as filehandle: + with open(filename, 'rb') as filehandle: line = filehandle.read(3) if line == codecs.BOM_UTF8: filehandle.seek(0) @@ -125,10 +125,10 @@ def importData(dbase, filename, user): #------------------------------------------------------------------------- # -# CSV Parser +# CSV Parser # #------------------------------------------------------------------------- -class CSVParser(object): +class CSVParser: """Class to read data in CSV format from a file object.""" def __init__(self, dbase, user, default_tag_format=None): self.db = dbase @@ -140,7 +140,7 @@ class CSVParser(object): self.indi_count = 0 self.place_count = 0 self.pref = {} # person ref, internal to this sheet - self.fref = {} # family ref, internal to this sheet + self.fref = {} # family ref, internal to this sheet self.placeref = {} self.place_types = {} # Build reverse dictionary, name to type number @@ -154,94 +154,83 @@ class CSVParser(object): self.place_types[custom_type] = 0 self.place_types[custom_type.lower()] = 0 column2label = { - "surname": ("Lastname", "Surname", _("Surname"), "lastname", - "last_name", "surname", _("surname")), - "firstname": ("Firstname", "Given name", _("Given name"), "Given", - _("Given"), "firstname", "first_name", "given_name", - "given name", _("given name"), "given", _("given")), - "callname": ("Callname", "Call name", _("Call name"), "Call", - _("Call"), "callname", "call_name", "call name", "call", - _("call")), - "title": ("Title", _("Person or Place|Title"), "title", _("Person or Place|title")), - "prefix": ("Prefix", _("Prefix"), "prefix", _("prefix")), - "suffix": ("Suffix", _("Suffix"), "suffix", _("suffix")), - "gender": ("Gender", _("Gender"), "gender", _("gender")), - "source": ("Source", _("Source"), "source", _("source")), - "note": ("Note", _("Note"), "note", _("note")), - "birthplace": ("Birthplace", "Birth place", _("Birth place"), - "birthplace", "birth_place", "birth place", _("birth place")), - "birthplace_id": ("Birthplaceid", "Birth place id", _("Birth place id"), - "birthplaceid", "birth_place_id", "birth place id", _("birth place id"), + "surname": ("lastname", "last_name", "surname", _("surname"), + _("Surname")), + "firstname": ("firstname", "first_name", "given_name", "given", + "given name", _("given name"), _("given"), + _("Given"), _("Given name")), + "callname": ("call name", _("Call name"), "callname", "call_name", + "call", _("Call"), _("call")), + "title": ("title", _("title"), _("Person or Place|title")), + "prefix": ("prefix", _("prefix"), _("Prefix")), + "suffix": ("suffix", _("suffix"), _("Suffix")), + "gender": ("gender", _("gender"), _("Gender")), + "source": ("source", _("source"), _("Source")), + "note": ("note", _("note"), _("Note")), + "birthplace": ("birthplace", "birth_place", "birth place", + _("birth place"), _("Birth place")), + "birthplace_id": ("birthplaceid", "birth_place_id", + "birth place id", _("birth place id"), "birthplace_id"), - "birthdate": ("Birthdate", "Birth date", _("Birth date"), - "birthdate", "birth_date", "birth date", _("birth date")), - "birthsource": ("Birthsource", "Birth source", _("Birth source"), - "birthsource", "birth_source", "birth source", - _("birth source")), - "baptismplace": ("Baptismplace", "Baptism place", - _("Baptism place"), "baptismplace", "baptism place", - _("baptism place")), - "baptismplace_id": ("Baptismplaceid", "Baptism place id", - _("Baptism place id"), "baptismplaceid", "baptism place id", + "birthdate": ("birthdate", "birth_date", "birth date", + _("birth date")), + "birthsource": ("birthsource", "birth_source", "birth source", + _("birth source")), + "baptismplace": ("baptismplace", "baptism place", + _("baptism place")), + "baptismplace_id": ("baptismplaceid", "baptism place id", _("baptism place id"), "baptism_place_id", "baptismplace_id"), - "baptismdate": ("Baptismdate", "Baptism date", _("Baptism date"), - "baptismdate", "baptism date", _("baptism date")), - "baptismsource": ("Baptismsource", "Baptism source", - _("Baptism source"), "baptismsource", "baptism source", - _("baptism source")), - "burialplace": ("Burialplace", "Burial place", _("Burial place"), - "burialplace", "burial place", _("burial place")), - "burialplace_id": ("Burialplaceid", "Burial place id", _("Burial place id"), - "burialplaceid", "burial place id", _("burial place id"), - "burial_place_id", "burialplace_id"), - "burialdate": ("Burialdate", "Burial date", _("Burial date"), - "burialdate", "burial date", _("burial date")), - "burialsource": ("Burialsource", "Burial source", - _("Burial source"), "burialsource", "burial source", - _("burial source")), - "deathplace": ("Deathplace", "Death place", _("Death place"), - "deathplace", "death_place", "death place", _("death place")), - "deathplace_id": ("Deathplaceid", "Death place id", _("Death place id"), - "deathplaceid", "death_place_id", "death place id", _("death place id"), - "death_place_id", "deathplace_id"), - "deathdate": ("Deathdate", "Death date", _("Death date"), - "deathdate", "death_date", "death date", _("death date")), - "deathsource": ("Deathsource", "Death source", _("Death source"), - "deathsource", "death_source", "death source", - _("death source")), - "deathcause": ("Deathcause", "Death cause", _("Death cause"), - "deathcause", "death_cause", "death cause", _("death cause")), - "grampsid": ("Grampsid", "ID", "Gramps id", _("Gramps ID"), - "grampsid", "id", "gramps_id", "gramps id", _("Gramps id")), - "person": ("Person", _("Person"), "person", _("person")), + "baptismdate": ("baptismdate", "baptism date", _("baptism date")), + "baptismsource": ("baptismsource", "baptism source", + _("baptism source")), + "burialplace": ("burialplace", "burial place", _("burial place")), + "burialplace_id": ("burialplaceid", "burial place id", + _("burial place id"), "burial_place_id", + "burialplace_id"), + "burialdate": ("burialdate", "burial date", _("burial date")), + "burialsource": ("burialsource", "burial source", + _("burial source")), + "deathplace": ("deathplace", "death_place", "death place", + _("death place")), + "deathplace_id": ("deathplaceid", "death place id", + _("death place id"), "death_place_id", + "deathplace_id"), + "deathdate": ("deathdate", "death_date", "death date", + _("death date")), + "deathsource": ("deathsource", "death_source", "death source", + _("death source")), + "deathcause": ("deathcause", "death_cause", "death cause", + _("death cause")), + "grampsid": (_("Gramps ID"), "grampsid", "id", "gramps_id", + "gramps id"), + "person": ("person", _("person"), _("Person")), # ---------------------------------- - "child": ("Child", _("Child"), "child", _("child")), - "family": ("Family", _("Family"), "family", _("family")), + "child": ("child", _("child"), _("Child")), + "family": ("family", _("family"), _("Family")), # ---------------------------------- - "wife": ("Mother", _("Mother"), "Wife", _("Wife"), "Parent2", - _("Parent2"), "mother", _("mother"), "wife", _("wife"), - "parent2", _("parent2")), - "husband": ("Father", _("Father"), "Husband", _("Husband"), - "Parent1", _("Parent1"), "father", _("father"), "husband", - _("husband"), "parent1", _("parent1")), - "marriage": ("Marriage", _("Marriage"), "marriage", _("marriage")), - "date": ("Date", _("Date"), "date", _("date")), - "place": ("Place", _("Place"), "place", _("place")), - "place_id": ("Placeid", "place id", "Place id", "place_id", "placeid"), - "name": ("Name", _("Name"), "name", _("name")), - "type": ("Type", _("Type"), "type", _("type")), - "latitude": ("Latitude", _("latitude"), "latitude", _("latitude")), - "longitude": ("Longitude", _("Longitude"), "longitude", _("longitude")), - "code": ("Code", _("Code"), "code", _("code")), - "enclosed_by": ("Enclosed by", _("Enclosed by"), "enclosed by", _("enclosed by"), - "enclosed_by", _("enclosed_by"), "Enclosed_by", _("Enclosed_by"), - "enclosedby") + "wife": ("mother", _("mother"), _("Mother"), + "wife", _("wife"), _("Wife"), + "parent2", _("parent2")), + "husband": ("father", _("father"), _("Father"), + "husband", _("husband"), _("Husband"), + "parent1", _("parent1")), + "marriage": ("marriage", _("marriage"), _("Marriage")), + "date": ("date", _("date"), _("Date")), + "place": ("place", _("place"), _("Place")), + "place_id": ("place id", "place_id", "placeid", _("place id")), + "name": ("name", _("name"), _("Name")), + "type": ("type", _("type"), _("Type")), + "latitude": ("latitude", _("latitude")), + "longitude": ("longitude", _("longitude")), + "code": ("code", _("code"), _("Code")), + "enclosed_by": ("enclosed by", _("enclosed by"), + "enclosed_by", _("enclosed_by"), "enclosedby") } lab2col_dict = [] for key in list(column2label.keys()): for val in column2label[key]: - lab2col_dict.append((val, key)) + lab2col_dict.append((val.lower(), key)) self.label2column = dict(lab2col_dict) if default_tag_format: name = time.strftime(default_tag_format) @@ -283,10 +272,12 @@ class CSVParser(object): return self.lookup(type_, id_) else: return db_lookup - elif id_.lower() in self.fref: - return self.fref[id_.lower()] else: - return None + id_ = self.db.fid2user_format(id_) + if id_.lower() in self.fref: + return self.fref[id_.lower()] + else: + return None elif type_ == "person": if id_.startswith("[") and id_.endswith("]"): id_ = self.db.id2user_format(id_[1:-1]) @@ -295,24 +286,28 @@ class CSVParser(object): return self.lookup(type_, id_) else: return db_lookup - elif id_.lower() in self.pref: - return self.pref[id_.lower()] else: - return None + id_ = self.db.id2user_format(id_) + if id_.lower() in self.pref: + return self.pref[id_.lower()] + else: + return None elif type_ == "place": if id_.startswith("[") and id_.endswith("]"): - id_ = self.db.id2user_format(id_[1:-1]) + id_ = self.db.pid2user_format(id_[1:-1]) db_lookup = self.db.get_place_from_gramps_id(id_) if db_lookup is None: return self.lookup(type_, id_) else: return db_lookup - elif id_.lower() in self.placeref: - return self.placeref[id_.lower()] else: - return None + id_ = self.db.pid2user_format(id_) + if id_.lower() in self.placeref: + return self.placeref[id_.lower()] + else: + return None else: - LOG.warn("invalid lookup type in CSV import: '%s'" % type_) + LOG.warning("invalid lookup type in CSV import: '%s'" % type_) return None def storeup(self, type_, id_, object_): @@ -330,7 +325,7 @@ class CSVParser(object): id_ = self.db.pid2user_format(id_) self.placeref[id_.lower()] = object_ else: - LOG.warn("invalid storeup type in CSV import: '%s'" % type_) + LOG.warning("invalid storeup type in CSV import: '%s'" % type_) def parse(self, filehandle): """ @@ -339,11 +334,11 @@ class CSVParser(object): :param filehandle: open file handle positioned at start of the file """ progress_title = _('CSV Import') - with self.user.progress(progress_title, + with self.user.progress(progress_title, _('Reading data...'), 1) as step: data = self.read_csv(filehandle) - with self.user.progress(progress_title, + with self.user.progress(progress_title, _('Importing data...'), len(data)) as step: tym = time.time() self.db.disable_signals() @@ -370,7 +365,7 @@ class CSVParser(object): self.indi_count = 0 self.place_count = 0 self.pref = {} # person ref, internal to this sheet - self.fref = {} # family ref, internal to this sheet + self.fref = {} # family ref, internal to this sheet self.placeref = {} header = None line_number = 0 @@ -382,7 +377,7 @@ class CSVParser(object): continue ###################################### if header is None: - header = [self.cleanup_column_name(r) for r in row] + header = [self.cleanup_column_name(r.lower()) for r in row] col = {} count = 0 for key in header: @@ -401,7 +396,7 @@ class CSVParser(object): elif "place" in header: self._parse_place(line_number, row, col) else: - LOG.warn("ignoring line %d" % line_number) + LOG.warning("ignoring line %d" % line_number) return None def _parse_marriage(self, line_number, row, col): @@ -418,8 +413,8 @@ class CSVParser(object): husband = self.lookup("person", husband) if husband is None and wife is None: # might have children, so go ahead and add - LOG.warn("no parents on line %d; adding family anyway" % - line_number) + LOG.warning("no parents on line %d; adding family anyway" % + line_number) family = self.get_or_create_family(marriage_ref, husband, wife) # adjust gender, if not already provided if husband: @@ -433,7 +428,7 @@ class CSVParser(object): wife.set_gender(Person.FEMALE) self.db.commit_person(wife, self.trans) if marriage_ref: - self.storeup("family", marriage_ref.lower(), family) + self.storeup("family", marriage_ref, family) if marriagesource: # add, if new new, marriagesource = self.get_or_create_source(marriagesource) @@ -489,8 +484,8 @@ class CSVParser(object): "Parse the content of a family line" family_ref = rd(line_number, row, col, "family") if family_ref is None: - LOG.warn("no family reference found for family on line %d" % - line_number) + LOG.warning("no family reference found for family on line %d" % + line_number) return # required child = rd(line_number, row, col, "child") source = rd(line_number, row, col, "source") @@ -499,12 +494,12 @@ class CSVParser(object): child = self.lookup("person", child) family = self.lookup("family", family_ref) if family is None: - LOG.warn("no matching family reference found for family " - "on line %d" % line_number) + LOG.warning("no matching family reference found for family " + "on line %d" % line_number) return if child is None: - LOG.warn("no matching child reference found for family " - "on line %d" % line_number) + LOG.warning("no matching child reference found for family " + "on line %d" % line_number) return # is this child already in this family? If so, don't add LOG.debug("children: %s", [ref.ref for ref in @@ -596,8 +591,8 @@ class CSVParser(object): person = self.lookup("person", person_ref) if person is None: if surname is None: - LOG.warn("empty surname for new person on line %d" % - line_number) + LOG.warning("empty surname for new person on line %d" % + line_number) surname = "" # new person person = self.create_person() @@ -681,8 +676,8 @@ class CSVParser(object): if birthsource is not None: new, birthsource = self.get_or_create_source(birthsource) if birthdate or birthplace or birthsource: - new, birth = self.get_or_create_event(person, - EventType.BIRTH, birthdate, + new, birth = self.get_or_create_event(person, + EventType.BIRTH, birthdate, birthplace, birthsource) birth_ref = person.get_birth_ref() if birth_ref is None: @@ -703,8 +698,8 @@ class CSVParser(object): if baptismsource is not None: new, baptismsource = self.get_or_create_source(baptismsource) if baptismdate or baptismplace or baptismsource: - new, baptism = self.get_or_create_event(person, - EventType.BAPTISM, baptismdate, + new, baptism = self.get_or_create_event(person, + EventType.BAPTISM, baptismdate, baptismplace, baptismsource) baptism_ref = get_primary_event_ref_from_type(self.db, person, "Baptism") @@ -751,8 +746,8 @@ class CSVParser(object): if burialsource is not None: new, burialsource = self.get_or_create_source(burialsource) if burialdate or burialplace or burialsource: - new, burial = self.get_or_create_event(person, - EventType.BURIAL, burialdate, + new, burial = self.get_or_create_event(person, + EventType.BURIAL, burialdate, burialplace, burialsource) burial_ref = get_primary_event_ref_from_type(self.db, person, "Burial") @@ -786,8 +781,8 @@ class CSVParser(object): place = self.create_place() if place_id is not None: if place_id.startswith("[") and place_id.endswith("]"): - place.gramps_id = self.db.id2user_format(place_id[1:-1]) - self.storeup("place", place_id.lower(), place) + place.gramps_id = self.db.pid2user_format(place_id[1:-1]) + self.storeup("place", place_id, place) if place_title is not None: place.title = place_title if place_name is not None: @@ -866,7 +861,7 @@ class CSVParser(object): self.db.commit_person(wife, self.trans) self.fam_count += 1 return family - + def get_or_create_event(self, object_, type_, date=None, place=None, source=None): """ Add or find a type event on object """ @@ -903,7 +898,7 @@ class CSVParser(object): self.find_and_set_citation(event, source) self.db.add_event(event, self.trans) return (1, event) - + def create_person(self): """ Used to create a new person we know doesn't exist """ person = Person() @@ -973,6 +968,6 @@ class CSVParser(object): LOG.debug(" creating citation") citation.set_reference_handle(source.get_handle()) self.db.add_citation(citation, self.trans) - LOG.debug(" created citation, citation %s %s" % + LOG.debug(" created citation, citation %s %s" % (citation, citation.get_gramps_id())) obj.add_citation(citation.get_handle())