diff --git a/gramps/cli/clidbman.py b/gramps/cli/clidbman.py index 5a8ed7bc6..1bf9786c7 100644 --- a/gramps/cli/clidbman.py +++ b/gramps/cli/clidbman.py @@ -64,7 +64,7 @@ _ = glocale.translation.gettext from gramps.gen.db import DbBsddb from gramps.gen.plug import BasePluginManager from gramps.gen.config import config -from gramps.gen.constfunc import win, conv_to_unicode +from gramps.gen.constfunc import win, conv_to_unicode, UNITYPE #------------------------------------------------------------------------- # # constants @@ -167,6 +167,8 @@ class CLIDbManager(object): flags = db.DB_CREATE | db.DB_PRIVATE |\ db.DB_INIT_MPOOL |\ db.DB_INIT_LOG | db.DB_INIT_TXN + if (isinstance(dirpath, UNITYPE) and win() and sys.version_info[0] < 3): + dirpath = dirpath.encode(sys.getfilesystemencoding()) try: env.open(dirpath, flags) except Exception as msg: @@ -232,8 +234,6 @@ class CLIDbManager(object): """ # make the default directory if it does not exist dbdir = os.path.expanduser(config.get('behavior.database-path')) - if sys.version_info[0] < 3: - dbdir = dbdir.encode(glocale.getfilesystemencoding()) db_ok = make_dbdir(dbdir) self.current_names = [] @@ -407,6 +407,7 @@ class CLIDbManager(object): Returns old_name, new_name if success, None, None if no success """ try: + filepath = conv_to_unicode(filepath, 'utf8') name_file = open(filepath, "r") old_text=name_file.read() name_file.close() diff --git a/gramps/gen/const.py b/gramps/gen/const.py index ab2372022..795f13359 100644 --- a/gramps/gen/const.py +++ b/gramps/gen/const.py @@ -44,7 +44,7 @@ import uuid # #------------------------------------------------------------------------- from .git_revision import get_git_revision - +from .constfunc import get_env_var #------------------------------------------------------------------------- # # Gramps Version @@ -88,28 +88,18 @@ APP_VCARD = ["text/x-vcard", "text/x-vcalendar"] # #------------------------------------------------------------------------- if 'GRAMPSHOME' in os.environ: - USER_HOME = os.environ['GRAMPSHOME'] + USER_HOME = get_env_var('GRAMPSHOME') HOME_DIR = os.path.join(USER_HOME, 'gramps') elif 'USERPROFILE' in os.environ: - USER_HOME = os.environ['USERPROFILE'] + USER_HOME = get_env_var('USERPROFILE') if 'APPDATA' in os.environ: HOME_DIR = os.path.join(os.environ['APPDATA'], 'gramps') else: HOME_DIR = os.path.join(USER_HOME, 'gramps') else: - USER_HOME = os.environ['HOME'] + USER_HOME = get_env_var('HOME') HOME_DIR = os.path.join(USER_HOME, '.gramps') -# Conversion of USER_HOME to unicode was needed to have better -# support for non ASCII path names in Windows for the Gramps database. - -if sys.version_info[0] < 3: - if not isinstance(USER_HOME, unicode): - USER_HOME = unicode(USER_HOME, sys.getfilesystemencoding()) - if not isinstance(HOME_DIR, unicode): - HOME_DIR = unicode(HOME_DIR, sys.getfilesystemencoding()) -else: - pass VERSION_DIR = os.path.join( HOME_DIR, "gramps%s%s" % (VERSION_TUPLE[0], VERSION_TUPLE[1])) @@ -125,7 +115,7 @@ THUMB_NORMAL = os.path.join(THUMB_DIR, "normal") THUMB_LARGE = os.path.join(THUMB_DIR, "large") USER_PLUGINS = os.path.join(VERSION_DIR, "plugins") # dirs checked/made for each Gramps session -USER_DIRLIST = (HOME_DIR, VERSION_DIR, ENV_DIR, TEMP_DIR, THUMB_DIR, +USER_DIRLIST = (USER_HOME, HOME_DIR, VERSION_DIR, ENV_DIR, TEMP_DIR, THUMB_DIR, THUMB_NORMAL, THUMB_LARGE, USER_PLUGINS) #------------------------------------------------------------------------- diff --git a/gramps/gen/constfunc.py b/gramps/gen/constfunc.py index 202eaa5d1..4c2270304 100644 --- a/gramps/gen/constfunc.py +++ b/gramps/gen/constfunc.py @@ -34,6 +34,8 @@ perform a translation on import, eg Gtk. #------------------------------------------------------------------------ import platform import sys +import ctypes +import os #------------------------------------------------------------------------- # @@ -53,7 +55,11 @@ WINDOWS = ["Windows", "win32"] #python 2 and 3 support, use correct conversion to unicode if sys.version_info[0] < 3: - conv_to_unicode = unicode + def conv_to_unicode(x, y): + if isinstance(x, unicode): + return x + return unicode(x, y) + conv_to_unicode_direct = unicode STRTYPE = basestring UNITYPE = unicode @@ -159,3 +165,28 @@ def mod_key(): return "" return "" + +# Python2 on Windows munges environemnt variables to match the system +# code page. This breaks all manner of things and the workaround +# though a bit ugly, is encapsulated here. Use this to retrieve +# environment variables if there's any chance they might contain +# Unicode, and especially for paths. + +# Shamelessly lifted from http://stackoverflow.com/questions/2608200/problems-with-umlauts-in-python-appdata-environvent-variable, answer 1. + + +def get_env_var(name, default=None): + if not name or not name in os.environ: + return default + + if sys.version_info[0] < 3 and win(): + name = unicode(name) # make sure string argument is unicode + n = ctypes.windll.kernel32.GetEnvironmentVariableW(name, None, 0) + if n==0: + return default + buf = ctypes.create_unicode_buffer(u'\0'*n) + ctypes.windll.kernel32.GetEnvironmentVariableW(name, buf, n) + return buf.value + + return os.environ[name] + diff --git a/gramps/gen/db/read.py b/gramps/gen/db/read.py index b7f8ff4f3..72d6f8174 100644 --- a/gramps/gen/db/read.py +++ b/gramps/gen/db/read.py @@ -48,10 +48,8 @@ if config.get('preferences.use-bsddb3') or sys.version_info[0] >= 3: from bsddb3 import db else: from bsddb import db -from ..const import GRAMPS_LOCALE as glocale -_ = glocale.translation.gettext -import re +import re import logging #------------------------------------------------------------------------- @@ -79,8 +77,9 @@ from ..utils.cast import conv_dbstr_to_unicode from . import (BsddbBaseCursor, DbReadBase) from ..utils.id import create_id from ..errors import DbError -from ..constfunc import UNITYPE, STRTYPE, cuni, handle2internal +from ..constfunc import UNITYPE, STRTYPE, cuni, handle2internal, get_env_var from ..const import GRAMPS_LOCALE as glocale +_ = glocale.translation.gettext LOG = logging.getLogger(DBLOGNAME) LOG = logging.getLogger(".citation") @@ -248,7 +247,7 @@ class DbBsddbRead(DbReadBase, Callback): __signals__ = {} # If this is True logging will be turned on. try: - _LOG_ALL = int(os.environ.get('GRAMPS_SIGNAL', "0")) == 1 + _LOG_ALL = int(get_env_var('GRAMPS_SIGNAL', "0")) == 1 except: _LOG_ALL = False diff --git a/gramps/gen/db/undoredo.py b/gramps/gen/db/undoredo.py index c6e07eae0..07f7778bd 100644 --- a/gramps/gen/db/undoredo.py +++ b/gramps/gen/db/undoredo.py @@ -54,7 +54,7 @@ _ = glocale.translation.gettext # Gramps modules # #------------------------------------------------------------------------- -from ..constfunc import conv_to_unicode, handle2internal +from ..constfunc import conv_to_unicode, handle2internal, win, UNITYPE from .dbconst import * from . import BSDDBTxn from ..errors import DbError @@ -410,7 +410,11 @@ class DbUndoBSDDB(DbUndo): """ Open the undo/redo database """ - self.undodb.open(self.path, db.DB_RECNO, db.DB_CREATE) + path = (self.path.encode(sys.getfilesystemencoding()) + if (isinstance(self.path, UNITYPE) and win() + and sys.version_info[0] < 3) + else self.path) + self.undodb.open(path, db.DB_RECNO, db.DB_CREATE) def close(self): """ diff --git a/gramps/gen/db/write.py b/gramps/gen/db/write.py index a70807834..a285928af 100644 --- a/gramps/gen/db/write.py +++ b/gramps/gen/db/write.py @@ -79,7 +79,8 @@ from ..utils.callback import Callback from ..utils.cast import conv_dbstr_to_unicode from ..updatecallback import UpdateCallback from ..errors import DbError -from ..constfunc import win, conv_to_unicode, cuni, UNITYPE, handle2internal +from ..constfunc import (win, conv_to_unicode, cuni, UNITYPE, handle2internal, + get_env_var) from ..const import GRAMPS_LOCALE as glocale _ = glocale.translation.gettext @@ -197,6 +198,16 @@ def find_referenced_handle(key, data): val = val.encode('utf-8') return val +def _encode(path): + """ + Conditionally return the unicode string encoded to sys.filesystem.encoding + """ + if not (isinstance(path, UNITYPE) and win() and sys.version_info[0] < 3): + _LOG.debug("Didn't Encode %s", repr(path)) + return path + _LOG.debug("Encoding %s", repr(path)) + return path.encode(sys.getfilesystemencoding()) + #------------------------------------------------------------------------- # # BsddbWriteCursor @@ -291,7 +302,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): dbmap = db.DB(self.env) dbmap.set_flags(flags) - fname = os.path.join(file_name, table_name + DBEXT) + fname = _encode(os.path.join(file_name, table_name + DBEXT)) if self.readonly: dbmap.open(fname, table_name, dbtype, DBFLAGS_R) @@ -302,7 +313,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): def __open_shelf(self, file_name, table_name, dbtype=db.DB_HASH): dbmap = dbshelve.DBShelf(self.env) - fname = os.path.join(file_name, table_name + DBEXT) + fname = _encode(os.path.join(file_name, table_name + DBEXT)) if self.readonly: dbmap.open(fname, table_name, dbtype, DBFLAGS_R) @@ -426,7 +437,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): """Older version of Berkeley DB can't read data created by a newer version.""" bdb_version = db.version() - versionpath = os.path.join(self.path, BDBVERSFN) + versionpath = os.path.join(self.path, cuni(BDBVERSFN)) # Compare the current version of the database (bsddb_version) with the # version of the database code (env_version). If it is a downgrade, # raise an exception because we can't do anything. If they are the same, @@ -625,7 +636,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): env_name = name try: - self.env.open(env_name, env_flags) + self.env.open(_encode(env_name), env_flags) except Exception as msg: _LOG.warning("Error opening db environment: " + str(msg)) try: @@ -2203,7 +2214,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback): # Environment name is now based on the filename env_name = name - self.env.open(env_name, env_flags) + self.env.open(_encode(env_name), env_flags) self.env.txn_checkpoint() self.metadata = self.__open_shelf(full_name, META) @@ -2266,7 +2277,7 @@ def write_lock_file(name): try: user = os.getlogin() except: - user = os.environ.get('USER') + user = get_env_var('USER') if host: text = "%s@%s" % (user, host) else: diff --git a/gramps/gen/utils/grampslocale.py b/gramps/gen/utils/grampslocale.py index 59f3ac46d..06403c8e7 100644 --- a/gramps/gen/utils/grampslocale.py +++ b/gramps/gen/utils/grampslocale.py @@ -172,7 +172,7 @@ class GrampsLocale(object): and (localedir is None or localedir == cls.__first_instance.localedir) and (domain is None or domain == cls.__first_instance.localedomain) and (languages is None or len(languages) == 0 or - languages == cls.__first_instance.languages)): + languages == cls.__first_instance.language)): return cls.__first_instance return super(GrampsLocale, cls).__new__(cls) @@ -521,7 +521,7 @@ class GrampsLocale(object): if hasattr(self, 'initialized') and self.initialized: return _first = self._GrampsLocale__first_instance - + self.localedir = None # Everything breaks without localedir, so get that set up # first. Warnings are logged in _init_first_instance or # _init_secondary_locale if this comes up empty. @@ -530,7 +530,7 @@ class GrampsLocale(object): elif _first and hasattr(_first, 'localedir'): self.localedir = _first.localedir else: - self.localedir = None + LOG.warn("No Localedir or localdir %s invalid", localedir) self.lang = lang self.localedomain = domain or 'gramps' diff --git a/gramps/gen/utils/resourcepath.py b/gramps/gen/utils/resourcepath.py index ba457dafe..a02d79a54 100644 --- a/gramps/gen/utils/resourcepath.py +++ b/gramps/gen/utils/resourcepath.py @@ -29,6 +29,8 @@ _hdlr = logging.StreamHandler() _hdlr.setFormatter(logging.Formatter(fmt="%(name)s.%(levelname)s: %(message)s")) LOG.addHandler(_hdlr) +from ..constfunc import get_env_var + class ResourcePath(object): """ ResourcePath is a singleton, meaning that only one of them is ever @@ -58,10 +60,9 @@ class ResourcePath(object): else: test_path = os.path.join("data", "authors.xml") resource_path = None - if ('GRAMPS_RESOURCES' in os.environ and - os.path.exists(os.path.join(os.environ['GRAMPS_RESOURCES'], - test_path))): - resource_path = os.environ['GRAMPS_RESOURCES'] + tmp_path = get_env_var('GRAMPS_RESOURCES') + if (tmp_path and os.path.exists(os.path.join(tmp_path, test_path))): + resource_path = tmp_path elif installed: try: with io.open(resource_file, encoding='utf-8', diff --git a/gramps/grampsapp.py b/gramps/grampsapp.py index 0d313bb58..1bdc69940 100644 --- a/gramps/grampsapp.py +++ b/gramps/grampsapp.py @@ -29,7 +29,7 @@ # Python modules # #------------------------------------------------------------------------- -from __future__ import print_function +from __future__ import print_function, unicode_literals import sys ## hack to avoid mentioning 'utf8' encoding everywhere unicode or str is is used @@ -53,7 +53,7 @@ from subprocess import Popen, PIPE #------------------------------------------------------------------------- from .gen.const import APP_GRAMPS, USER_DIRLIST, HOME_DIR from .version import VERSION_TUPLE -from .gen.constfunc import win +from .gen.constfunc import win, get_env_var #------------------------------------------------------------------------- # @@ -293,11 +293,11 @@ def show_settings(): else: kernel = None - lang_str = os.environ.get('LANG','not set') - language_str = os.environ.get('LANGUAGE','not set') - grampsi18n_str = os.environ.get('GRAMPSI18N','not set') - grampshome_str = os.environ.get('GRAMPSHOME','not set') - grampsdir_str = os.environ.get('GRAMPSDIR','not set') + lang_str = get_env_var('LANG','not set') + language_str = get_env_var('LANGUAGE','not set') + grampsi18n_str = get_env_var('GRAMPSI18N','not set') + grampshome_str = get_env_var('GRAMPSHOME','not set') + grampsdir_str = get_env_var('GRAMPSDIR','not set') try: dotversion_str = Popen(['dot', '-V'], stderr=PIPE).communicate(input=None)[1] @@ -320,7 +320,7 @@ def show_settings(): except: gsversion_str = 'Ghostscript not in system PATH' - os_path = os.environ.get('PATH','not set') + os_path = get_env_var('PATH','not set') os_path = os_path.split(os.pathsep) print ("Gramps Settings:") @@ -410,12 +410,12 @@ def run(): if 'LANG' in os.environ: LOG.debug('Using LANG: %s' % - os.environ.get('LANG')) + get_env_var('LANG')) else: LOG.debug('environment: LANG is not defined') if 'LANGUAGE' in os.environ: LOG.debug('Using LANGUAGE: %s' % - os.environ.get('LANGUAGE')) + get_env_var('LANGUAGE')) else: LOG.debug('environment: LANGUAGE is not defined') diff --git a/gramps/gui/aboutdialog.py b/gramps/gui/aboutdialog.py index 5a4be4e56..7dbc94048 100644 --- a/gramps/gui/aboutdialog.py +++ b/gramps/gui/aboutdialog.py @@ -27,8 +27,9 @@ # Python modules # #------------------------------------------------------------------------- -from gramps.gen.const import GRAMPS_LOCALE as glocale -_ = glocale.translation.gettext + +from __future__ import unicode_literals + import os import sys @@ -57,6 +58,9 @@ from gi.repository import GdkPixbuf from gramps.gen.const import (AUTHORS, AUTHORS_FILE, COMMENTS, COPYRIGHT_MSG, DOCUMENTERS, LICENSE_FILE, PROGRAM_NAME, SPLASH, TRANSLATORS, URL_HOMEPAGE, VERSION) +from gramps.gen.const import GRAMPS_LOCALE as glocale +_ = glocale.translation.gettext +from gramps.gen.constfunc import get_env_var from .display import display_url from gramps.gen.config import config @@ -139,7 +143,7 @@ class GrampsAboutDialog(Gtk.AboutDialog): % (ellipses(str(VERSION)), ellipses(str(sys.version).replace('\n','')), ellipses(str(bsddb.__version__) + " " + str(bsddb.db.version())), - ellipses(os.environ.get('LANG','')), + ellipses(get_env_var('LANG','')), ellipses(operatingsystem), ellipses(distribution))) diff --git a/gramps/gui/dbman.py b/gramps/gui/dbman.py index e53624946..b916b29e0 100644 --- a/gramps/gui/dbman.py +++ b/gramps/gui/dbman.py @@ -50,7 +50,7 @@ else: import logging LOG = logging.getLogger(".DbManager") -from gramps.gen.constfunc import win, UNITYPE +from gramps.gen.constfunc import win, UNITYPE, conv_to_unicode if win(): _RCS_FOUND = os.system("rcs -V >nul 2>nul") == 0 if _RCS_FOUND and "TZ" not in os.environ: @@ -247,8 +247,8 @@ class DbManager(CLIDbManager): self.rcs.set_sensitive(False) if store.get_value(node, STOCK_COL) == Gtk.STOCK_DIALOG_ERROR: - path = get_unicode_path_from_env_var(store.get_value(node, PATH_COL)) - backup = os.path.join(path, "person.gbkp") + path = conv_to_unicode(store.get_value(node, PATH_COL), 'utf8') + backup = os.path.join(path, u"person.gbkp") self.repair.set_sensitive(os.path.isfile(backup)) else: self.repair.set_sensitive(False) @@ -365,7 +365,7 @@ class DbManager(CLIDbManager): self.top.destroy() del self.selection del self.name_renderer - path = get_unicode_path_from_env_var(store.get_value(node, PATH_COL)) + path = conv_to_unicode(store.get_value(node, PATH_COL), 'utf8') return (path, store.get_value(node, NAME_COL)) else: self.top.destroy() @@ -400,7 +400,7 @@ class DbManager(CLIDbManager): try: self.break_lock(self.lock_file) store, node = self.selection.get_selected() - dbpath = get_unicode_path_from_env_var(store.get_value(node, PATH_COL)) + dbpath = conv_to_unicode(store.get_value(node, PATH_COL), 'utf8') (tval, last) = time_val(dbpath) store.set_value(node, OPEN_COL, 0) store.set_value(node, STOCK_COL, "") @@ -582,19 +582,20 @@ class DbManager(CLIDbManager): store, node = self.selection.get_selected() path = store.get_path(node) node = self.model.get_iter(path) - filename = self.model.get_value(node, FILE_COL) + filename = conv_to_unicode(self.model.get_value(node, FILE_COL), 'utf8') try: name_file = open(filename, "r") file_name_to_delete=name_file.read() name_file.close() remove_filename(file_name_to_delete) - for (top, dirs, files) in os.walk(self.data_to_delete[1]): + directory = conv_to_unicode(self.data_to_delete[1], 'utf8') + for (top, dirs, files) in os.walk(directory): for filename in files: os.unlink(os.path.join(top, filename)) - os.rmdir(self.data_to_delete[1]) + os.rmdir(directory) except (IOError, OSError) as msg: DbManager.ERROR(_("Could not delete Family Tree"), - str(msg)) + str(msg)) # rebuild the display self.__populate() diff --git a/gramps/gui/logger/_errorreportassistant.py b/gramps/gui/logger/_errorreportassistant.py index 93bc615d0..12e07945a 100644 --- a/gramps/gui/logger/_errorreportassistant.py +++ b/gramps/gui/logger/_errorreportassistant.py @@ -26,8 +26,8 @@ # Python modules # #------------------------------------------------------------------------- -from gramps.gen.const import GRAMPS_LOCALE as glocale -_ = glocale.translation.gettext +from __future__ import unicode_literals + from gi.repository import Gdk from gi.repository import Gtk from gi.repository import GdkPixbuf @@ -46,7 +46,9 @@ else: # Gramps modules # #------------------------------------------------------------------------- -from gramps.gen.const import ICON, SPLASH +from gramps.gen.const import ICON, SPLASH, GRAMPS_LOCALE as glocale +_ = glocale.translation.gettext +from gramps.gen.constfunc import get_env_var from gramps.version import VERSION from ..display import display_help, display_url @@ -166,7 +168,7 @@ class ErrorReportAssistant(Gtk.Assistant): % (str(sys.version).replace('\n',''), str(bsddb.__version__) + " " + str(bsddb.db.version()), str(VERSION), - os.environ.get('LANG',''), + get_env_var('LANG',''), operatingsystem, distribution, '%d.%d.%d' % (Gtk.get_major_version(), diff --git a/gramps/gui/viewmanager.py b/gramps/gui/viewmanager.py index ee68880f2..dd5bf2395 100644 --- a/gramps/gui/viewmanager.py +++ b/gramps/gui/viewmanager.py @@ -85,7 +85,7 @@ from .displaystate import DisplayState, RecentDocsMenu from gramps.gen.const import (HOME_DIR, ICON, URL_BUGTRACKER, URL_HOMEPAGE, URL_MAILINGLIST, URL_MANUAL_PAGE, URL_WIKISTRING, WIKI_EXTRAPLUGINS) -from gramps.gen.constfunc import is_quartz +from gramps.gen.constfunc import is_quartz, conv_to_unicode from gramps.gen.config import config from gramps.gen.errors import WindowActiveError from .dialog import ErrorDialog, WarningDialog, QuestionDialog2, InfoDialog @@ -1097,6 +1097,7 @@ class ViewManager(CLIManager): value = dialog.run() if value: (filename, title) = value + filename = conv_to_unicode(filename, 'utf8') self.db_loader.read_file(filename) self._post_load_newdb(filename, 'x-directory/normal', title) diff --git a/gramps/plugins/lib/maps/osmgps.py b/gramps/plugins/lib/maps/osmgps.py index 303de991e..55ea3542b 100644 --- a/gramps/plugins/lib/maps/osmgps.py +++ b/gramps/plugins/lib/maps/osmgps.py @@ -64,6 +64,7 @@ from gramps.gen.const import GRAMPS_LOCALE as glocale _ = glocale.translation.sgettext from gramps.gen.config import config from gramps.gui.dialog import ErrorDialog +from gramps.gen.constfunc import get_env_var #------------------------------------------------------------------------- # @@ -133,7 +134,7 @@ class OsmGps(): constants.map_title[map_type]) config.set("geography.map_service", map_type) self.current_map = map_type - http_proxy = os.environ.get('http_proxy') + http_proxy = get_env_var('http_proxy') if 0: self.osm = DummyMapNoGpsPoint() else: diff --git a/gramps/test/test/test_util_test.py b/gramps/test/test/test_util_test.py index 1ea79be6c..074d14714 100644 --- a/gramps/test/test/test_util_test.py +++ b/gramps/test/test/test_util_test.py @@ -29,6 +29,8 @@ import os import tempfile import unittest as U +from gramps.gen.constfunc import get_env_var + usage_note=""" ************************************************************** Testing (and runing) Gramps requires that PYTHONPATH include @@ -134,7 +136,7 @@ class Test3(U.TestCase): here = tu.absdir() bases = (here, tempfile.gettempdir()) asubs = [os.path.join(b,"test_sub") for b in bases] - home= os.environ["HOME"] + home = get_env_var("HOME") if home: home_junk = os.path.join(home,"test_junk") def _rmsubs(self):