From 802bab7bbe5990a8759025d3df42552dd4435e6b Mon Sep 17 00:00:00 2001 From: prculley Date: Sun, 17 Jul 2016 10:13:30 -0500 Subject: [PATCH] CSV backport to gramps42 for bug 9501 and 9499 --- gramps/plugins/export/exportcsv.py | 153 ++++++++------------------- gramps/plugins/importer/importcsv.py | 14 ++- 2 files changed, 58 insertions(+), 109 deletions(-) diff --git a/gramps/plugins/export/exportcsv.py b/gramps/plugins/export/exportcsv.py index f8650c604..7267a61c9 100644 --- a/gramps/plugins/export/exportcsv.py +++ b/gramps/plugins/export/exportcsv.py @@ -58,6 +58,7 @@ from gramps.gen.utils.string import gender as gender_map from gramps.gen.datehandler import get_date from gramps.gen.display.place import displayer as _pd from gramps.gui.glade import Glade +from gramps.gen.constfunc import win #------------------------------------------------------------------------- # @@ -102,72 +103,6 @@ def get_primary_source_title(db, obj): return source.get_title() return "" -#------------------------------------------------------------------------- -# -# Encoding support for CSV, from http://docs.python.org/lib/csv-examples.html -# -#------------------------------------------------------------------------- -class UTF8Recoder(object): - """Iterator that reads an encoded stream and reencodes the input to UTF-8.""" - def __init__(self, f, encoding): - self.reader = codecs.getreader(encoding)(f) - - def __iter__(self): - return self - - def __next__(self): - return self.reader.next().encode("utf-8") - -class UnicodeReader(object): - """ - A CSV reader which will iterate over lines in the CSV file "f", which is - encoded in the given encoding. - - """ - - def __init__(self, f, encoding="utf-8", **kwds): - f = UTF8Recoder(f, encoding) - self.reader = csv.reader(f, **kwds) - - def __next__(self): - row = next(self.reader) - return [str(s, "utf-8") for s in row] - - def __iter__(self): - return self - -class UnicodeWriter(object): - """ - A CSV writer which will write rows to CSV file "f", which is encoded in - the given encoding. - - """ - - def __init__(self, f, encoding="utf-8", **kwds): - # Redirect output to a queue - self.queue = StringIO() - self.writer = csv.writer(self.queue, **kwds) - self.stream = f - self.encoder = codecs.getencoder(encoding) - - def writerow(self, row): - self.writer.writerow(row) - data = self.queue.getvalue() - #data now contains the csv data in unicode - # ... and reencode it into the target encoding - data, length = self.encoder(data) - # write to the target stream - self.stream.write(data) - # empty queue, go to start position, then truncate - self.queue.seek(0) - self.queue.truncate(0) - - def writerows(self, rows): - list(map(self.writerow, rows)) - - def close(self): - self.stream.close() - #------------------------------------------------------------------------- # # CSVWriter Options @@ -270,7 +205,7 @@ class CSVWriter(object): # make place list so that dependencies are first: self.place_list = [] - place_list = [x for x in self.db.iter_place_handles()] + place_list = sorted([x for x in self.db.iter_place_handles()]) while place_list: handle = place_list[0] place = self.db.get_place_from_handle(handle) @@ -317,9 +252,10 @@ class CSVWriter(object): def export_data(self): self.dirname = os.path.dirname (self.filename) try: - self.g = open(self.filename,"w") - self.fp = open(self.filename, "wb") - self.g = UnicodeWriter(self.fp) + self.fp = open(self.filename, "w", + encoding='utf_8_sig' if win() else 'utf_8', + newline='') + self.g = csv.writer(self.fp) except IOError as msg: msg2 = _("Could not create %s") % self.filename self.user.notify_error(msg2,str(msg)) @@ -343,6 +279,44 @@ class CSVWriter(object): LOG.debug("Possible people to export: %s", len(self.plist)) LOG.debug("Possible families to export: %s", len(self.flist)) LOG.debug("Possible places to export: %s", len(self.place_list)) + ########################### + if self.include_places: + if self.translate_headers: + self.write_csv(_("Place"), _("Title"), _("Name"), + _("Type"), _("Latitude"), _("Longitude"), + _("Code"), _("Enclosed_by"), _("Date")) + else: + self.write_csv("Place", "Title", "Name", + "Type", "Latitude", "Longitude", + "Code", "Enclosed_by", "Date") + for key in self.place_list: + place = self.db.get_place_from_handle(key) + if place: + place_id = place.gramps_id + place_title = place.title + place_name = place.name.value + place_type = str(place.place_type) + place_latitude = place.lat + place_longitude = place.long + place_code = place.code + if place.placeref_list: + for placeref in place.placeref_list: + placeref_obj = self.db.get_place_from_handle(placeref.ref) + placeref_date = "" + if not placeref.date.is_empty(): + placeref_date = placeref.date + placeref_id = "" + if placeref_obj: + placeref_id = "[%s]" % placeref_obj.gramps_id + self.write_csv("[%s]" % place_id, place_title, place_name, place_type, + place_latitude, place_longitude, place_code, placeref_id, + placeref_date) + else: + self.write_csv("[%s]" % place_id, place_title, place_name, place_type, + place_latitude, place_longitude, place_code, "", + "") + self.update() + self.writeln() ########################### sort: sortorder = [] dropped_surnames = set() @@ -557,44 +531,7 @@ class CSVWriter(object): self.write_csv(family_id, grampsid_ref) self.update() self.writeln() - ########################### - if self.include_places: - if self.translate_headers: - self.write_csv(_("Place"), _("Title"), _("Name"), - _("Type"), _("Latitude"), _("Longitude"), - _("Code"), _("Enclosed_by"), _("Date")) - else: - self.write_csv("Place", "Title", "Name", - "Type", "Latitude", "Longitude", - "Code", "Enclosed_by", "Date") - for key in self.place_list: - place = self.db.get_place_from_handle(key) - if place: - place_id = place.gramps_id - place_title = place.title - place_name = place.name.value - place_type = str(place.place_type) - place_latitude = place.lat - place_longitude = place.long - place_code = place.code - if place.placeref_list: - for placeref in place.placeref_list: - placeref_obj = self.db.get_place_from_handle(placeref.ref) - placeref_date = "" - if not placeref.date.is_empty(): - placeref_date = placeref.date - placeref_id = "" - if placeref_obj: - placeref_id = "[%s]" % placeref_obj.gramps_id - self.write_csv("[%s]" % place_id, place_title, place_name, place_type, - place_latitude, place_longitude, place_code, placeref_id, - placeref_date) - else: - self.write_csv("[%s]" % place_id, place_title, place_name, place_type, - place_latitude, place_longitude, place_code, "", - "") - self.writeln() - self.g.close() + self.fp.close() return True def format_date(self, date): diff --git a/gramps/plugins/importer/importcsv.py b/gramps/plugins/importer/importcsv.py index 328115a13..b1438f385 100644 --- a/gramps/plugins/importer/importcsv.py +++ b/gramps/plugins/importer/importcsv.py @@ -32,6 +32,7 @@ import time import csv import codecs +from io import TextIOWrapper #------------------------------------------------------------------------ # @@ -59,7 +60,6 @@ from gramps.gen.utils.string import gender as gender_map from gramps.gen.utils.id import create_id from gramps.gen.utils.location import located_in from gramps.gen.lib.eventroletype import EventRoleType -from gramps.gen.constfunc import conv_to_unicode from gramps.gen.config import config from gramps.gen.display.place import displayer as place_displayer from gramps.gen.utils.libformatting import ImportInfo @@ -108,6 +108,15 @@ def importData(dbase, filename, user): config.get('preferences.tag-on-import') else None)) try: with open(filename, 'r') as filehandle: + line = filehandle.read(3) + if line == codecs.BOM_UTF8: + filehandle.seek(0) + filehandle = TextIOWrapper(filehandle, encoding='utf_8_sig', + errors='replace', newline='') + else: # just open with OS encoding + filehandle.seek(0) + filehandle = TextIOWrapper(filehandle, + errors='replace', newline='') parser.parse(filehandle) except EnvironmentError as err: user.notify_error(_("%s could not be opened\n") % filename, str(err)) @@ -775,6 +784,9 @@ class CSVParser(object): if place is None: # new place place = self.create_place() + if place_id is not None: + if place_id.startswith("[") and place_id.endswith("]"): + place.gramps_id = self.db.id2user_format(place_id[1:-1]) self.storeup("place", place_id.lower(), place) if place_title is not None: place.title = place_title