Merge pull request #203 from prculley/csv
CSV backport to gramps42 for bug 9501 and 9499
This commit is contained in:
Doug Blank
GitHub
commit
6ade6ba419
@ -58,6 +58,7 @@ from gramps.gen.utils.string import gender as gender_map
|
||||
from gramps.gen.datehandler import get_date
|
||||
from gramps.gen.display.place import displayer as _pd
|
||||
from gramps.gui.glade import Glade
|
||||
from gramps.gen.constfunc import win
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
@ -102,72 +103,6 @@ def get_primary_source_title(db, obj):
|
||||
return source.get_title()
|
||||
return ""
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
# Encoding support for CSV, from http://docs.python.org/lib/csv-examples.html
|
||||
#
|
||||
#-------------------------------------------------------------------------
|
||||
class UTF8Recoder(object):
|
||||
"""Iterator that reads an encoded stream and reencodes the input to UTF-8."""
|
||||
def __init__(self, f, encoding):
|
||||
self.reader = codecs.getreader(encoding)(f)
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
return self.reader.next().encode("utf-8")
|
||||
|
||||
class UnicodeReader(object):
|
||||
"""
|
||||
A CSV reader which will iterate over lines in the CSV file "f", which is
|
||||
encoded in the given encoding.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, f, encoding="utf-8", **kwds):
|
||||
f = UTF8Recoder(f, encoding)
|
||||
self.reader = csv.reader(f, **kwds)
|
||||
|
||||
def __next__(self):
|
||||
row = next(self.reader)
|
||||
return [str(s, "utf-8") for s in row]
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
class UnicodeWriter(object):
|
||||
"""
|
||||
A CSV writer which will write rows to CSV file "f", which is encoded in
|
||||
the given encoding.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, f, encoding="utf-8", **kwds):
|
||||
# Redirect output to a queue
|
||||
self.queue = StringIO()
|
||||
self.writer = csv.writer(self.queue, **kwds)
|
||||
self.stream = f
|
||||
self.encoder = codecs.getencoder(encoding)
|
||||
|
||||
def writerow(self, row):
|
||||
self.writer.writerow(row)
|
||||
data = self.queue.getvalue()
|
||||
#data now contains the csv data in unicode
|
||||
# ... and reencode it into the target encoding
|
||||
data, length = self.encoder(data)
|
||||
# write to the target stream
|
||||
self.stream.write(data)
|
||||
# empty queue, go to start position, then truncate
|
||||
self.queue.seek(0)
|
||||
self.queue.truncate(0)
|
||||
|
||||
def writerows(self, rows):
|
||||
list(map(self.writerow, rows))
|
||||
|
||||
def close(self):
|
||||
self.stream.close()
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
#
|
||||
# CSVWriter Options
|
||||
@ -270,7 +205,7 @@ class CSVWriter(object):
|
||||
|
||||
# make place list so that dependencies are first:
|
||||
self.place_list = []
|
||||
place_list = [x for x in self.db.iter_place_handles()]
|
||||
place_list = sorted([x for x in self.db.iter_place_handles()])
|
||||
while place_list:
|
||||
handle = place_list[0]
|
||||
place = self.db.get_place_from_handle(handle)
|
||||
@ -317,9 +252,10 @@ class CSVWriter(object):
|
||||
def export_data(self):
|
||||
self.dirname = os.path.dirname (self.filename)
|
||||
try:
|
||||
self.g = open(self.filename,"w")
|
||||
self.fp = open(self.filename, "wb")
|
||||
self.g = UnicodeWriter(self.fp)
|
||||
self.fp = open(self.filename, "w",
|
||||
encoding='utf_8_sig' if win() else 'utf_8',
|
||||
newline='')
|
||||
self.g = csv.writer(self.fp)
|
||||
except IOError as msg:
|
||||
msg2 = _("Could not create %s") % self.filename
|
||||
self.user.notify_error(msg2,str(msg))
|
||||
@ -343,6 +279,44 @@ class CSVWriter(object):
|
||||
LOG.debug("Possible people to export: %s", len(self.plist))
|
||||
LOG.debug("Possible families to export: %s", len(self.flist))
|
||||
LOG.debug("Possible places to export: %s", len(self.place_list))
|
||||
###########################
|
||||
if self.include_places:
|
||||
if self.translate_headers:
|
||||
self.write_csv(_("Place"), _("Title"), _("Name"),
|
||||
_("Type"), _("Latitude"), _("Longitude"),
|
||||
_("Code"), _("Enclosed_by"), _("Date"))
|
||||
else:
|
||||
self.write_csv("Place", "Title", "Name",
|
||||
"Type", "Latitude", "Longitude",
|
||||
"Code", "Enclosed_by", "Date")
|
||||
for key in self.place_list:
|
||||
place = self.db.get_place_from_handle(key)
|
||||
if place:
|
||||
place_id = place.gramps_id
|
||||
place_title = place.title
|
||||
place_name = place.name.value
|
||||
place_type = str(place.place_type)
|
||||
place_latitude = place.lat
|
||||
place_longitude = place.long
|
||||
place_code = place.code
|
||||
if place.placeref_list:
|
||||
for placeref in place.placeref_list:
|
||||
placeref_obj = self.db.get_place_from_handle(placeref.ref)
|
||||
placeref_date = ""
|
||||
if not placeref.date.is_empty():
|
||||
placeref_date = placeref.date
|
||||
placeref_id = ""
|
||||
if placeref_obj:
|
||||
placeref_id = "[%s]" % placeref_obj.gramps_id
|
||||
self.write_csv("[%s]" % place_id, place_title, place_name, place_type,
|
||||
place_latitude, place_longitude, place_code, placeref_id,
|
||||
placeref_date)
|
||||
else:
|
||||
self.write_csv("[%s]" % place_id, place_title, place_name, place_type,
|
||||
place_latitude, place_longitude, place_code, "",
|
||||
"")
|
||||
self.update()
|
||||
self.writeln()
|
||||
########################### sort:
|
||||
sortorder = []
|
||||
dropped_surnames = set()
|
||||
@ -557,44 +531,7 @@ class CSVWriter(object):
|
||||
self.write_csv(family_id, grampsid_ref)
|
||||
self.update()
|
||||
self.writeln()
|
||||
###########################
|
||||
if self.include_places:
|
||||
if self.translate_headers:
|
||||
self.write_csv(_("Place"), _("Title"), _("Name"),
|
||||
_("Type"), _("Latitude"), _("Longitude"),
|
||||
_("Code"), _("Enclosed_by"), _("Date"))
|
||||
else:
|
||||
self.write_csv("Place", "Title", "Name",
|
||||
"Type", "Latitude", "Longitude",
|
||||
"Code", "Enclosed_by", "Date")
|
||||
for key in self.place_list:
|
||||
place = self.db.get_place_from_handle(key)
|
||||
if place:
|
||||
place_id = place.gramps_id
|
||||
place_title = place.title
|
||||
place_name = place.name.value
|
||||
place_type = str(place.place_type)
|
||||
place_latitude = place.lat
|
||||
place_longitude = place.long
|
||||
place_code = place.code
|
||||
if place.placeref_list:
|
||||
for placeref in place.placeref_list:
|
||||
placeref_obj = self.db.get_place_from_handle(placeref.ref)
|
||||
placeref_date = ""
|
||||
if not placeref.date.is_empty():
|
||||
placeref_date = placeref.date
|
||||
placeref_id = ""
|
||||
if placeref_obj:
|
||||
placeref_id = "[%s]" % placeref_obj.gramps_id
|
||||
self.write_csv("[%s]" % place_id, place_title, place_name, place_type,
|
||||
place_latitude, place_longitude, place_code, placeref_id,
|
||||
placeref_date)
|
||||
else:
|
||||
self.write_csv("[%s]" % place_id, place_title, place_name, place_type,
|
||||
place_latitude, place_longitude, place_code, "",
|
||||
"")
|
||||
self.writeln()
|
||||
self.g.close()
|
||||
self.fp.close()
|
||||
return True
|
||||
|
||||
def format_date(self, date):
|
||||
|
||||
@ -32,6 +32,7 @@
|
||||
import time
|
||||
import csv
|
||||
import codecs
|
||||
from io import TextIOWrapper
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
#
|
||||
@ -59,7 +60,6 @@ from gramps.gen.utils.string import gender as gender_map
|
||||
from gramps.gen.utils.id import create_id
|
||||
from gramps.gen.utils.location import located_in
|
||||
from gramps.gen.lib.eventroletype import EventRoleType
|
||||
from gramps.gen.constfunc import conv_to_unicode
|
||||
from gramps.gen.config import config
|
||||
from gramps.gen.display.place import displayer as place_displayer
|
||||
from gramps.gen.utils.libformatting import ImportInfo
|
||||
@ -108,6 +108,15 @@ def importData(dbase, filename, user):
|
||||
config.get('preferences.tag-on-import') else None))
|
||||
try:
|
||||
with open(filename, 'r') as filehandle:
|
||||
line = filehandle.read(3)
|
||||
if line == codecs.BOM_UTF8:
|
||||
filehandle.seek(0)
|
||||
filehandle = TextIOWrapper(filehandle, encoding='utf_8_sig',
|
||||
errors='replace', newline='')
|
||||
else: # just open with OS encoding
|
||||
filehandle.seek(0)
|
||||
filehandle = TextIOWrapper(filehandle,
|
||||
errors='replace', newline='')
|
||||
parser.parse(filehandle)
|
||||
except EnvironmentError as err:
|
||||
user.notify_error(_("%s could not be opened\n") % filename, str(err))
|
||||
@ -775,6 +784,9 @@ class CSVParser(object):
|
||||
if place is None:
|
||||
# new place
|
||||
place = self.create_place()
|
||||
if place_id is not None:
|
||||
if place_id.startswith("[") and place_id.endswith("]"):
|
||||
place.gramps_id = self.db.id2user_format(place_id[1:-1])
|
||||
self.storeup("place", place_id.lower(), place)
|
||||
if place_title is not None:
|
||||
place.title = place_title
|
||||
|
||||
Loading…
Reference in New Issue
Block a user