From 548507008bc1b087312addbb27e49659cab5fc13 Mon Sep 17 00:00:00 2001 From: John Ralls Date: Tue, 19 Mar 2013 18:22:19 +0000 Subject: [PATCH] GrampsLocale: Ensure correct stdout encoding Replace stdout with one that uses a transcoding Streamwriter. This better handles differences between Py2 and Py3 than does trying to encode strings prior to output. In particular Py3's default stdout demands unencoded strings and prints byte-strings when one tries to pre-encode them. svn: r21695 --- gramps/cli/arghandler.py | 81 ++++++++++++------------- gramps/cli/argparser.py | 48 ++++++++------- gramps/cli/grampscli.py | 10 +-- gramps/cli/plug/__init__.py | 34 +++++------ gramps/gen/utils/grampslocale.py | 19 ++++-- gramps/plugins/importer/importprogen.py | 12 ++-- gramps/plugins/tool/check.py | 22 +++---- 7 files changed, 115 insertions(+), 111 deletions(-) diff --git a/gramps/cli/arghandler.py b/gramps/cli/arghandler.py index a7b6773d0..16769de7a 100644 --- a/gramps/cli/arghandler.py +++ b/gramps/cli/arghandler.py @@ -39,8 +39,6 @@ Module responsible for handling the command line arguments for GRAMPS. from __future__ import print_function import os import sys -from gramps.gen.const import GRAMPS_LOCALE as glocale -_ = glocale.get_translation().gettext #------------------------------------------------------------------------- # @@ -57,6 +55,8 @@ from gramps.gen.plug import BasePluginManager from gramps.gen.plug.report import CATEGORY_BOOK, CATEGORY_CODE, BookList from .plug import cl_report, cl_book from .user import User +from gramps.gen.const import GRAMPS_LOCALE as glocale +_ = glocale.get_translation().gettext #------------------------------------------------------------------------- # @@ -186,9 +186,9 @@ class ArgHandler(object): else: # Need to convert to system file encoding before printing # For non latin characters in path/file/user names - print(msg1.encode(sys.stdout.encoding, 'backslashreplace'), file=sys.stderr) + print(msg1, file=sys.stderr) if msg2 is not None: - print(msg2.encode(sys.stdout.encoding, 'backslashreplace'), file=sys.stderr) + print(msg2, file=sys.stderr) #------------------------------------------------------------------------- # Argument parser: sorts out given arguments @@ -294,8 +294,7 @@ class ArgHandler(object): ask = raw_input else: ask = input - ans = ask(_('OK to overwrite? (yes/no) ') \ - .encode(sys.stdout.encoding, 'backslashreplace')) + ans = ask(_('OK to overwrite? (yes/no) ')) except EOFError: print() sys.exit(0) @@ -408,27 +407,24 @@ class ArgHandler(object): """ if self.list: - print(_('List of known family trees in your database path\n').\ - encode(sys.stdout.encoding, 'backslashreplace')) + print(_('List of known family trees in your database path\n')) + for name, dirname in sorted(self.dbman.family_tree_list(), key=lambda pair: pair[0].lower()): - - print((_("%(full_DB_path)s with name \"%(f_t_name)s\"") % \ - {'full_DB_path' : dirname, - 'f_t_name' : name}).encode(sys.stdout.encoding, 'backslashreplace')) + + print(_("%(full_DB_path)s with name \"%(f_t_name)s\"") + % {'full_DB_path' : dirname, 'f_t_name' : name}) sys.exit(0) - + if self.list_more: - print(_('Gramps Family Trees:').encode(sys.stdout.encoding, 'backslashreplace')) + print(_('Gramps Family Trees:')) summary_list = self.dbman.family_tree_summary() for summary in sorted(summary_list, key=lambda sum: sum["Family tree"].lower()): - print(_("Family Tree \"%s\":") % summary["Family tree"].\ - encode(sys.stdout.encoding, 'backslashreplace')) + print(_("Family Tree \"%s\":") % summary["Family tree"]) for item in sorted(summary): if item != "Family tree": - print((" %s: %s" % (item, summary[item])).\ - encode(sys.stdout.encoding, 'backslashreplace')) + print(" %s: %s" % (item, summary[item])) sys.exit(0) self.__open_action() @@ -441,13 +437,9 @@ class ArgHandler(object): self.cl_action(action, op_string) for expt in self.exports: - # Need to convert path/filename to str before printing - # For non latin characters in Windows path/file/user names - fn = expt[0].encode(sys.stdout.encoding, 'backslashreplace') - fmt = str(expt[1]) print(_("Exporting: file %(filename)s, " - "format %(format)s.") % \ - {'filename' : fn, + "format %(format)s.") % \ + {'filename' : fn, 'format' : fmt}, file=sys.stderr) self.cl_export(expt[0], expt[1]) @@ -481,21 +473,21 @@ class ArgHandler(object): self.imp_db_path, title = self.dbman.create_new_db_cli() else: self.imp_db_path = get_empty_tempdir("import_dbdir") \ - .encode(sys.stdout.encoding, 'backslashreplace') + .encode(sys.filesystem.encoding, 'backslashreplace') newdb = DbBsddb() newdb.write_version(self.imp_db_path) try: self.sm.open_activate(self.imp_db_path) msg = _("Created empty family tree successfully") - print(msg, file=sys.stderr) + gloclale.print(msg, file=sys.stderr) except: print(_("Error opening the file."), file=sys.stderr) print(_("Exiting..."), file=sys.stderr) sys.exit(0) for imp in self.imports: - fn = imp[0].encode(sys.stdout.encoding, 'backslashreplace') + fn = imp[0] fmt = str(imp[1]) msg = _("Importing: file %(filename)s, format %(format)s.") % \ {'filename' : fn, 'format' : fmt} @@ -590,7 +582,8 @@ class ArgHandler(object): options_str_dict = _split_options(options_str) except: options_str_dict = {} - print(_("Ignoring invalid options string."), file=sys.stderr) + print(_("Ignoring invalid options string."), + file=sys.stderr) name = options_str_dict.pop('name', None) _cl_list = pmgr.get_reg_reports(gui=False) @@ -618,16 +611,17 @@ class ArgHandler(object): msg = _("Report name not given. " "Please use one of %(donottranslate)s=reportname") % \ {'donottranslate' : '[-p|--options] name'} - - print(_("%s\n Available names are:") % msg, file=sys.stderr) + + glcoale.print(_("%s\n Available names are:") % msg, file=sys.stderr) for pdata in sorted(_cl_list, key= lambda pdata: pdata.id.lower()): # Print cli report name ([item[0]), GUI report name (item[4]) if len(pdata.id) <= 25: - print(" %s%s- %s" % ( pdata.id, " " * (26 - len(pdata.id)), - pdata.name.encode(sys.stdout.encoding, 'backslashreplace')), file=sys.stderr) + glocle.print(" %s%s- %s" + % ( pdata.id, " " * (26 - len(pdata.id)), + pdata.name), file=sys.stderr) else: - print(" %s\t- %s" % (pdata.id, - pdata.name.encode(sys.stdout.encoding, 'backslashreplace')), file=sys.stderr) + print(" %s\t- %s" + % (pdata.id, pdata.name), file=sys.stderr) elif action == "tool": from gramps.gui.plug import tool @@ -636,7 +630,8 @@ class ArgHandler(object): chunk in options_str.split(',') ] ) except: options_str_dict = {} - print(_("Ignoring invalid options string."), file=sys.stderr) + print(_("Ignoring invalid options string."), + file=sys.stderr) name = options_str_dict.pop('name', None) _cli_tool_list = pmgr.get_reg_tools(gui=False) @@ -658,24 +653,26 @@ class ArgHandler(object): msg = _("Tool name not given. " "Please use one of %(donottranslate)s=toolname.") % \ {'donottranslate' : '[-p|--options] name'} - - print(_("%s\n Available names are:") % msg, file=sys.stderr) + + glcoale.print(_("%s\n Available names are:") % msg, file=sys.stderr) for pdata in sorted(_cli_tool_list, key=lambda pdata: pdata.id.lower()): # Print cli report name ([item[0]), GUI report name (item[4]) if len(pdata.id) <= 25: - print(" %s%s- %s" % ( pdata.id, " " * (26 - len(pdata.id)), - pdata.name.encode(sys.stdout.encoding, 'backslashreplace')), file=sys.stderr) + print(" %s%s- %s" + % ( pdata.id, " " * (26 - len(pdata.id)), + pdata.name), file=sys.stderr) else: - print(" %s\t- %s" % (pdata.id, - pdata.name.encode(sys.stdout.encoding, 'backslashreplace')), file=sys.stderr) + print(" %s\t- %s" + % (pdata.id, pdata.name), file=sys.stderr) elif action == "book": try: options_str_dict = _split_options(options_str) except: options_str_dict = {} - print(_("Ignoring invalid options string."), file=sys.stderr) + print(_("Ignoring invalid options string."), + file=sys.stderr) name = options_str_dict.pop('name', None) book_list = BookList('books.xml', self.dbstate.db) diff --git a/gramps/cli/argparser.py b/gramps/cli/argparser.py index e3a2cce0c..327cc819e 100644 --- a/gramps/cli/argparser.py +++ b/gramps/cli/argparser.py @@ -39,8 +39,6 @@ Module responsible for handling the command line arguments for GRAMPS. from __future__ import print_function import sys import getopt -from gramps.gen.const import GRAMPS_LOCALE as glocale -_ = glocale.get_translation().gettext import logging #------------------------------------------------------------------------- @@ -52,6 +50,8 @@ from gramps.gen.const import LONGOPTS, SHORTOPTS from gramps.gen.config import config from gramps.gen.utils.configmanager import safe_eval from gramps.gen.utils.file import get_unicode_path_from_env_var +from gramps.gen.const import GRAMPS_LOCALE as glocale +_ = glocale.get_translation().gettext # Note: Make sure to edit const.py.in POPT_TABLE too! _HELP = _(""" @@ -242,7 +242,8 @@ class ArgParser(object): # if there were an argument without option, # use it as a file to open and return self.open_gui = leftargs[0] - print ("Trying to open: %s ..." % leftargs[0], file=sys.stderr) + print(_("Trying to open: %s ...") % leftargs[0], + file=sys.stderr) #see if force open is on for opt_ix in range(len(options)): option, value = options[opt_ix] @@ -275,7 +276,8 @@ class ArgParser(object): elif option in ( '-a', '--action' ): action = value if action not in ('report', 'tool', 'book'): - print ("Unknown action: %s. Ignoring." % action, file=sys.stderr) + print(_("Unknown action: %s. Ignoring.") % action, + file=sys.stderr) continue options_str = "" if opt_ix < len(options)-1 \ @@ -283,7 +285,7 @@ class ArgParser(object): options_str = options[opt_ix+1][1] self.actions.append((action, options_str)) elif option in ('-d', '--debug'): - print ('setup debugging', value, file=sys.stderr) + print(_('setup debugging'), value, file=sys.stderr) logger = logging.getLogger(value) logger.setLevel(logging.DEBUG) cleandbg += [opt_ix] @@ -292,14 +294,14 @@ class ArgParser(object): elif option in ('-L'): self.list_more = True elif option in ('-s','--show'): - print ("Gramps config settings from %s:" % \ - config.filename.encode(sys.stdout.encoding, 'backslashreplace')) + print(_("Gramps config settings from %s:") + % config.filename) for section in config.data: for setting in config.data[section]: - print ("%s.%s=%s" % ( - section, setting, - repr(config.data[section][setting]))) - print ('') + print ("%s.%s=%s" + % (section, setting, + repr(config.data[section][setting]))) + print () sys.exit(0) elif option in ('-c', '--config'): setting_name = value @@ -310,24 +312,24 @@ class ArgParser(object): set_value = True if config.has_default(setting_name): setting_value = config.get(setting_name) - print ("Current Gramps config setting: " \ - "%s:%s" % (setting_name, repr(setting_value)), file=sys.stderr) + print(_("Current Gramps config setting: %s:%s") + % (setting_name, repr(setting_value)), + file=sys.stderr) if set_value: if new_value == "DEFAULT": new_value = config.get_default(setting_name) else: new_value = safe_eval(new_value) config.set(setting_name, new_value) - print (" New Gramps config " \ - "setting: %s:%s" % ( - setting_name, - repr(config.get(setting_name)) - ), file=sys.stderr) + print(_(" New Gramps config setting: %s:%s") + % (setting_name, + repr(config.get(setting_name))), + file=sys.stderr) else: need_to_quit = True else: - print ("Gramps: no such config setting:" \ - " '%s'" % setting_name, file=sys.stderr) + print(_("Gramps: no such config setting: %s") + % setting_name, file=sys.stderr) need_to_quit = True cleandbg += [opt_ix] elif option in ('-h', '-?', '--help'): @@ -403,14 +405,14 @@ class ArgParser(object): """ if self.help: # Convert Help messages to file system encoding before printing - print (_HELP.encode(sys.stdout.encoding, 'backslashreplace')) + print (_HELP) sys.exit(0) - + def print_usage(self): """ If the user gives the --usage print the output to terminal. """ if self.usage: # Convert Help messages to file system encoding before printing - print (_USAGE.encode(sys.stdout.encoding, 'backslashreplace')) + print(_USAGE) sys.exit(0) diff --git a/gramps/cli/grampscli.py b/gramps/cli/grampscli.py index ef57acc66..4eceb6574 100644 --- a/gramps/cli/grampscli.py +++ b/gramps/cli/grampscli.py @@ -197,7 +197,7 @@ class CLIManager(object): self.db_loader = None self.file_loaded = False self._pmgr = BasePluginManager.get_instance() - + def open_activate(self, path): """ Open and make a family tree active @@ -297,10 +297,10 @@ def startcli(errors, argparser): #already errors encountered. Show first one on terminal and exit # Convert error message to file system encoding before print errmsg = _('Error encountered: %s') % errors[0][0] - errmsg = errmsg.encode(sys.stdout.encoding, 'backslashreplace') + errmsg = errmsg print(errmsg) errmsg = _(' Details: %s') % errors[0][1] - errmsg = errmsg.encode(sys.stdout.encoding, 'backslashreplace') + errmsg = errmsg print(errmsg) sys.exit(1) @@ -308,10 +308,10 @@ def startcli(errors, argparser): # Convert error message to file system encoding before print errmsg = _('Error encountered in argument parsing: %s') \ % argparser.errors[0][0] - errmsg = errmsg.encode(sys.stdout.encoding, 'backslashreplace') + errmsg = errmsg print(errmsg) errmsg = _(' Details: %s') % argparser.errors[0][1] - errmsg = errmsg.encode(sys.stdout.encoding, 'backslashreplace') + errmsg = errmsg print(errmsg) sys.exit(1) diff --git a/gramps/cli/plug/__init__.py b/gramps/cli/plug/__init__.py index f9a844385..2e8098ac4 100644 --- a/gramps/cli/plug/__init__.py +++ b/gramps/cli/plug/__init__.py @@ -34,9 +34,6 @@ # #------------------------------------------------------------------------- from __future__ import print_function - -from gramps.gen.const import GRAMPS_LOCALE as glocale -_ = glocale.get_translation().gettext import traceback import os import sys @@ -67,6 +64,8 @@ from gramps.gen.dbstate import DbState from gramps.gen.constfunc import STRTYPE, conv_to_unicode_direct from ..grampscli import CLIManager from ..user import User +from gramps.gen.const import GRAMPS_LOCALE as glocale +_ = glocale.get_translation().gettext #------------------------------------------------------------------------ # @@ -473,12 +472,12 @@ class CommandLineReport(object): self.format = None if _chosen_format and _format_str: print((_("Ignoring '%(notranslate1)s=%(notranslate2)s' " - "and using '%(notranslate1)s=%(notranslate3)s'.") % - {'notranslate1' : "off", - 'notranslate2' : self.options_dict['off'], - 'notranslate3' : _chosen_format})) + "and using '%(notranslate1)s=%(notranslate3)s'.") % + {'notranslate1' : "off", + 'notranslate2' : self.options_dict['off'], + 'notranslate3' : _chosen_format})) print((_("Use '%(notranslate)s' to see valid values.") % - {'notranslate' : "show=off"})) + {'notranslate' : "show=off"})) self.do_doc_options() @@ -571,14 +570,14 @@ class CommandLineReport(object): # Make the output nicer to read, assume a tab has 8 spaces tabs = '\t\t' if len(key) < 10 else '\t' optmsg = " %s%s%s (%s)" % (key, tabs, opt[1], opt[0]) - print(optmsg.encode(sys.stdout.encoding, 'backslashreplace')) + print(optmsg) else: optmsg = " %s%s%s" % (key, tabs, _('(no help available)')) - print(optmsg.encode(sys.stdout.encoding, 'backslashreplace')) - print((_(" Use '%(donottranslate)s' to see description " + print(optmsg) + print(_(" Use '%(donottranslate)s' to see description " "and acceptable values") % - {'donottranslate' : "show=option"})) + {'donottranslate' : "show=option"}) elif self.show in self.options_help: opt = self.options_help[self.show] tabs = '\t\t' if len(self.show) < 10 else '\t' @@ -588,16 +587,17 @@ class CommandLineReport(object): if isinstance(vals, (list, tuple)): for val in vals: optmsg = " %s" % val - print(optmsg.encode(sys.stdout.encoding, 'backslashreplace')) + print(optmsg) else: optmsg = " %s" % opt[2] - print(optmsg.encode(sys.stdout.encoding, 'backslashreplace')) + print(optmsg) else: #there was a show option given, but the option is invalid - print((_("option '%(optionname)s' not valid. " - "Use '%(donottranslate)s' to see all valid options.") % - {'optionname' : self.show, 'donottranslate' : "show=all"})) + print(_("option '%(optionname)s' not valid. " + "Use '%(donottranslate)s' to see all valid options.") + % {'optionname' : self.show, + 'donottranslate' : "show=all"}) #------------------------------------------------------------------------ # diff --git a/gramps/gen/utils/grampslocale.py b/gramps/gen/utils/grampslocale.py index dd6eff757..b1055525c 100644 --- a/gramps/gen/utils/grampslocale.py +++ b/gramps/gen/utils/grampslocale.py @@ -26,9 +26,11 @@ # python modules # #------------------------------------------------------------------------ +from __future__ import print_function import gettext import sys import os +import codecs import locale import logging LOG = logging.getLogger("grampslocale") @@ -210,11 +212,18 @@ class GrampsLocale(object): except locale.Error: pass #Next, we need to know what is the encoding from the native environment: - self.encoding = locale.getlocale()[1] - if not self.encoding: - self.encoding = locale.getpreferredencoding() - if not self.encoding: - self.encoding = 'utf-8' + self.encoding = sys.stdout.encoding or sys.getdefaultencoding() + +#Ensure that output is encoded correctly to stdout and stderr. This is +#much less cumbersome and error-prone than encoding individual outputs +#and better handles the differences between Python 2 and Python 3: + if sys.version_info[0] < 3: + sys.stdout = codecs.getwriter(self.encoding)(sys.stdout, 'backslashreplace') + sys.stderr = codecs.getwriter(self.encoding)(sys.stderr, 'backslashreplace') + else: + sys.stdout = codecs.getwriter(self.encoding)(sys.stdout.detach(), 'backslashreplace') + sys.stderr = codecs.getwriter(self.encoding)(sys.stderr.detach(), 'backslashreplace') + #GtkBuilder depends on reading Glade files as UTF-8 and crashes if it #doesn't, so set $LANG to have a UTF-8 locale. NB: This does *not* diff --git a/gramps/plugins/importer/importprogen.py b/gramps/plugins/importer/importprogen.py index 16cea2034..095996d3c 100644 --- a/gramps/plugins/importer/importprogen.py +++ b/gramps/plugins/importer/importprogen.py @@ -24,7 +24,7 @@ "Import from Pro-Gen" -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals #------------------------------------------------------------------------- # # standard python modules @@ -221,7 +221,7 @@ def _get_mem_text(mems, i): # Strip leading/trailing whitespace text = text.strip() - #print text.encode('utf-8') + #print(text) return text month_values = { @@ -306,7 +306,7 @@ class PG30_Def_Table: #f02=Persoon gewijzigd ,32,10,10, 1,68,"","INDI CHAN DATE" line_pat = re.compile(r'(\w+) = (.*)', re.VERBOSE) for l in lines: - #print l + #print(l) m = line_pat.match(l) if m: # TODO. Catch duplicates? @@ -325,12 +325,12 @@ class PG30_Def_Table: self.recflds = [] # list of fields that use up space in a record j = 0 for i, f in enumerate(self.flds): - #print "# field %s" % f + #print("# field %s" % f) nam = f.name self.nam2fld[nam] = f if f.size != 0: self.nam2idx[nam] = j - #print "# %s <= %d" % (f.fieldname, j) + #print("# %s <= %d" % (f.fieldname, j)) self.recflds.append(f) j = j + 1 @@ -398,7 +398,7 @@ class PG30_Def_Table: # Convert to unicode fld = fld.decode('cp850') flds.append(fld) - #print ', '.join([f.encode('utf-8') for f in flds]) + #print(', '.join(flds)) return flds def get_field_names(self): diff --git a/gramps/plugins/tool/check.py b/gramps/plugins/tool/check.py index d29611776..cd5176156 100644 --- a/gramps/plugins/tool/check.py +++ b/gramps/plugins/tool/check.py @@ -31,8 +31,7 @@ # python modules # #------------------------------------------------------------------------- -from __future__ import print_function, with_statement - +from __future__ import print_function import os import sys if sys.version_info[0] < 3: @@ -40,10 +39,6 @@ if sys.version_info[0] < 3: else: from io import StringIO import time - -from gramps.gen.const import GRAMPS_LOCALE as glocale -_ = glocale.get_translation().gettext -ngettext = glocale.get_translation().ngettext from collections import defaultdict #------------------------------------------------------------------------ @@ -81,6 +76,9 @@ from gramps.gui.dialog import OkDialog, MissingMediaDialog from gramps.gen.display.name import displayer as _nd from gramps.gui.glade import Glade from gramps.gen.constfunc import UNITYPE, cuni +from gramps.gen.const import GRAMPS_LOCALE as glocale +_ = glocale.get_translation().gettext +ngettext = glocale.get_translation().ngettext # table for handling control chars in notes. # All except 09, 0A, 0D are replaced with space. @@ -690,8 +688,7 @@ class CheckIntegrity(object): photo_desc = obj.get_description() if photo_name is not None and photo_name != "" and not find_file(photo_name): if cl: - # Convert to stdout encoding before prining - fn = os.path.basename(photo_name).encode(sys.stdout.encoding, 'backslashreplace') + fn = os.path.basename(photo_name) logging.warning(" FAIL: media file %s was not found." % fn) self.bad_photo.append(ObjectId) @@ -1944,7 +1941,7 @@ class CheckIntegrity(object): _('The database has passed internal checks'), parent=uistate.window) else: - print("No errors were found: the database has passed internal checks.") + print(_("No errors were found: the database has passed internal checks.") return 0 self.text = StringIO() @@ -2192,14 +2189,13 @@ class CheckIntegrity(object): # #------------------------------------------------------------------------- class Report(ManagedWindow): - + def __init__(self, uistate, text, cl=0): if cl: - print (text.encode(sys.stdout.encoding, 'backslashreplace')) - return + print (text) ManagedWindow.__init__(self, uistate, [], self) - + topDialog = Glade() topDialog.get_object("close").connect('clicked', self.close) window = topDialog.toplevel