2620: GEPS 031: Python 3 support - 3.2

* fix introduced errors: write of reference in db, check in dbloader on string type
 * working import of xml file in python 3


svn: r20659

gramps/gen/constfunc.py

@@ -49,16 +49,13 @@ from .const import WINDOWS, MACOS, LINUX
 #-------------------------------------------------------------------------
 
 #python 2 and 3 support, use correct conversion to unicode
-
 if sys.version_info[0] < 3:
     conv_to_unicode = unicode
-    conv_to_unicode_3 = unicode
     conv_to_unicode_direct = unicode
     STRTYPE = basestring
     UNITYPE = unicode
 else:
     conv_to_unicode = lambda x,y: str(x)
-    conv_to_unicode_3 = lambda x,y,z: str(x)
     conv_to_unicode_direct = str
     STRTYPE = str
     UNITYPE = str

gramps/gen/db/read.py

@@ -1190,22 +1190,23 @@ class DbBsddbRead(DbReadBase, Callback):
         pattern_match = re.match(r"(.*)%[0 ](\d+)[diu]$", id_pattern)
         if pattern_match:
             str_prefix = pattern_match.group(1)
-            nr_width = pattern_match.group(2)
+            ##nr_width = pattern_match.group(2)
             def closure_func(gramps_id):
                 if gramps_id and gramps_id.startswith(str_prefix):
                     id_number = gramps_id[len(str_prefix):]
                     if id_number.isdigit():
                         id_value = int(id_number, 10)
-                        if len(cuni(id_value)) > nr_width:
+                        ## this code never ran, as an int compared to str with > is False!
-                            # The ID to be imported is too large to fit in the
+##                        if len(cuni(id_value)) > nr_width:
-                            # users format. For now just create a new ID,
+##                            # The ID to be imported is too large to fit in the
-                            # because that is also what happens with IDs that
+##                            # users format. For now just create a new ID,
-                            # are identical to IDs already in the database. If
+##                            # because that is also what happens with IDs that
-                            # the problem of colliding import and already
+##                            # are identical to IDs already in the database. If
-                            # present IDs is solved the code here also needs
+##                            # the problem of colliding import and already
-                            # some solution.
+##                            # present IDs is solved the code here also needs
-                            gramps_id = id_pattern % 1
+##                            # some solution.
-                        else:
+##                            gramps_id = id_pattern % 1
+##                        else:
                         gramps_id = id_pattern % id_value
                 return gramps_id
         else:

gramps/gen/db/write.py

@@ -410,6 +410,9 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         except:
             # Just assume that the Berkeley DB version is OK.
             pass
+        if not env_version:
+            #empty file, assume it is ok to open
+            env_version = (0, 0, 0)
         if (env_version[0] > bdb_version[0]) or \
             (env_version[0] == bdb_version[0] and
              env_version[1] > bdb_version[1]):
@@ -534,12 +537,12 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
             with BSDDBTxn(self.env, self.metadata) as txn:
                 if gstats is None:
                     # New database. Set up the current version.
-                    #self.metadata.put('version', _DBVERSION, txn=the_txn)
+                    #self.metadata.put(b'version', _DBVERSION, txn=the_txn)
-                    txn.put('version', _DBVERSION)
+                    txn.put(b'version', _DBVERSION)
                 elif b'version' not in self.metadata:
                     # Not new database, but the version is missing.
                     # Use 0, but it is likely to fail anyway.
-                    txn.put('version', 0)
+                    txn.put(b'version', 0)
             
         self.genderStats = GenderStats(gstats)
 
@@ -946,6 +949,9 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         Remove the reference specified by the key, preserving the change in 
         the passed transaction.
         """
+        if isinstance(key, tuple):
+            #create a string key
+            key = str(key)
         if isinstance(key, UNITYPE):
             key = key.encode('utf-8')
         if not self.readonly:
@@ -960,6 +966,9 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         Add the reference specified by the key and the data, preserving the 
         change in the passed transaction.
         """
+        if isinstance(key, tuple):
+            #create a string key
+            key = str(key)
         if isinstance(key, UNITYPE):
             key = key.encode('utf-8')
         if self.readonly or not key:
@@ -1193,8 +1202,9 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         if self.update_env_version:
             versionpath = os.path.join(self.path, BDBVERSFN)
             try:
-                with open(versionpath, "wb") as version_file:
+                with open(versionpath, "w") as version_file:
-                    version = db.version()
+                    version = str(db.version())
+                    if sys.version_info[0] < 3:
                         if isinstance(version, UNITYPE):
                             version = version.encode('utf-8')
                     version_file.write(version)
@@ -2050,7 +2060,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.metadata  = self.__open_shelf(full_name, META)
         
         with BSDDBTxn(self.env, self.metadata) as txn:
-            txn.put('version', _DBVERSION)
+            txn.put(b'version', _DBVERSION)
         
         self.metadata.close()
         self.env.close()

gramps/gen/plug/_manager.py

@@ -312,7 +312,11 @@ class BasePluginManager(object):
         Reloads modules that might not be in the path.
         """
         try:
+            if sys.version_info[0] < 3:
                 reload(module)
+            else:
+                import imp
+                imp.reload(module)
         except:
             if pdata.mod_name in sys.modules:
                 del sys.modules[pdata.mod_name]

gramps/gen/utils/file.py

@@ -43,7 +43,7 @@ LOG = logging.getLogger(".gen.utils.file")
 # Gramps modules
 #
 #-------------------------------------------------------------------------
-from ..constfunc import win, mac, cuni, conv_to_unicode, conv_to_unicode_3
+from ..constfunc import win, mac, cuni, conv_to_unicode, UNITYPE
 from ..const import TEMP_DIR, USER_HOME
 
 #-------------------------------------------------------------------------
@@ -129,23 +129,35 @@ def get_unicode_path_from_file_chooser(path):
     :returns:    The Unicode version of path.
     """
     # make only unicode of path of type 'str'
-    if not (isinstance(path,  str)):
+    if isinstance(path, UNITYPE):
         return path
     
+    if not (isinstance(path,  str)):
+        return path
+    ## ONLY PYTHON 2 code shoulr reach this !
     if win():
         # in windows filechooser returns officially utf-8, not filesystemencoding
         try:
             return cuni(path)
         except:
             LOG.warn("Problem encountered converting string: %s." % path)
-            return conv_to_unicode_3(path, sys.getfilesystemencoding(), 
+            if sys.version_info[0] < 3:
+                return unicode(path, sys.getfilesystemencoding(), 
                                     errors='replace')
+            else:
+                #no idea, need to know what path is to know what to do
+                raise NotImplementedError("Path of type", type(path))
     else:
         try:
             return cuni(path, sys.getfilesystemencoding())
         except:
             LOG.warn("Problem encountered converting string: %s." % path)
-            return conv_to_unicode_3(path, sys.getfilesystemencoding(), errors='replace')    
+            if sys.version_info[0] < 3:
+                return unicode(path, sys.getfilesystemencoding(), errors='replace')
+            else:
+                #no idea, need to know what path is to know what to do
+                raise NotImplementedError("Path of type", type(path))
+            
 
 def get_unicode_path_from_env_var(path):
     """
@@ -167,15 +179,21 @@ def get_unicode_path_from_env_var(path):
             return new_path
         except:
             LOG.warn("Problem encountered converting string: %s." % path)
-            return conv_to_unicode_3(path, sys.getfilesystemencoding(), 
+            if sys.version_info[0] < 3:
-                                     errors='replace')
+                return unicode(path, sys.getfilesystemencoding(), errors='replace')
+            else:
+                #no idea, need to know what path is to know what to do
+                raise NotImplementedError("Path of type", type(path))
     else:
         try:
             return str(path)
         except:
             LOG.warn("Problem encountered converting string: %s." % path)
-            return conv_to_unicode_3(path, sys.getfilesystemencoding(), 
+            if sys.version_info[0] < 3:
-                                     errors='replace')    
+                return unicode(path, sys.getfilesystemencoding(), errors='replace')
+            else:
+                #no idea, need to know what path is to know what to do
+                raise NotImplementedError("Path of type", type(path))
 
 def get_new_filename(ext, folder='~/'):
     ix = 1
@@ -300,6 +318,9 @@ def fix_encoding(value, errors='strict'):
                     codeset = locale.getpreferredencoding()
             except:
                 codeset = "UTF-8"
-            return conv_to_unicode_3(value, codeset, errors)
+            if sys.version_info[0] < 3:
+                return unicode(value, codeset, errors)
+            else:
+                return value.decode(encoding=codeset, errors=errors)
     else:
         return value

gramps/grampsapp.py

@@ -33,12 +33,9 @@ from __future__ import print_function
 
 import sys
 ## hack to avoid mentioning 'utf8' encoding everywhere unicode or str is is used
-try:
+if sys.version_info[0] < 3:
     reload(sys)
     sys.setdefaultencoding('utf8')
-except:
-    #python3 has no reload
-    pass
 ##
 import os
 import signal

gramps/gui/dbloader.py

@@ -61,6 +61,7 @@ from gramps.gen.db.exceptions import (DbUpgradeRequiredError,
                                       BsddbDowngradeError, 
                                       DbVersionError, 
                                       DbEnvironmentError)
+from gramps.gen.constfunc import STRTYPE
 from gramps.gen.utils.file import get_unicode_path_from_file_chooser
 from .pluginmanager import GuiPluginManager
 from .dialog import (DBErrorDialog, ErrorDialog, QuestionDialog2, 
@@ -132,7 +133,7 @@ class DbLoader(CLIDbLoader):
             
         pmgr = GuiPluginManager.get_instance()
         
-        import_dialog = Gtk.FileChooserDialog(_('Gramps: Import database'), 
+        import_dialog = Gtk.FileChooserDialog(_('Gramps: Import Family Tree'), 
                                        self.uistate.window, 
                                        Gtk.FileChooserAction.OPEN, 
                                        (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, 
@@ -205,8 +206,7 @@ class DbLoader(CLIDbLoader):
         In this process, a warning dialog can pop up.
         
         """
-
+        if not isinstance(filename, STRTYPE):
-        if not isinstance(filename, str):
             return True
 
         filename = os.path.normpath(os.path.abspath(filename))

gramps/gui/tipofday.py

@@ -29,10 +29,11 @@ Handles the Tip of the Day dialog
 # standard python modules
 #
 #-------------------------------------------------------------------------
-from xml.parsers.expat import ParserCreate,ExpatError
+from xml.parsers.expat import ParserCreate, ExpatError
 from random import Random
 from gramps.gen.ggettext import gettext as _
 import os
+import sys
 
 #-------------------------------------------------------------------------
 #
@@ -139,7 +140,10 @@ class TipParser(object):
 
         self.mylist = []
         self.skip = False
+        if sys.version_info[0] < 3:
             xml_file = open(TIP_DATA)
+        else:
+            xml_file = open(TIP_DATA, 'rb')
         self.tlist = []
         parser = ParserCreate()
         parser.StartElementHandler = self.startElement

gramps/plugins/import/importgedcom.py

@@ -28,6 +28,7 @@
 # python modules
 #
 #------------------------------------------------------------------------
+import sys
 from gramps.gen.ggettext import gettext as _
 
 #------------------------------------------------------------------------
@@ -47,8 +48,12 @@ from gramps.gen.errors import DbError, GedcomError
 from gramps.gui.glade import Glade
 from gramps.plugins.lib.libmixin import DbMixin
 from gramps.plugins.lib import libgedcom
-module = __import__("gramps.plugins.lib.libgedcom")
+module = __import__("gramps.plugins.lib.libgedcom")   # why o why ??
-reload (module)
+if sys.version_info[0] < 3:
+    reload (module)
+else:
+    import imp
+    imp.reload(module)
 
 from gramps.gen.config import config
 from gramps.gen.constfunc import STRTYPE

gramps/plugins/import/importgeneweb.py

@@ -32,6 +32,7 @@ from __future__ import print_function
 
 import re
 import time
+import sys
 from gramps.gen.ggettext import gettext as _
 from gramps.gen.ggettext import ngettext
 

gramps/plugins/import/importgpkg.py

@@ -47,8 +47,9 @@ log = logging.getLogger(".ReadPkg")
 #
 #-------------------------------------------------------------------------
 from gramps.gen.const import XMLFILE
-from . import importxml
 from gramps.gen.utils.file import media_path
+## we need absolute import as this is dynamically loaded:
+from gramps.plugins.importers.importxml import importData
 
 #-------------------------------------------------------------------------
 #
@@ -91,7 +92,7 @@ def impData(database, name, user):
 
     imp_db_name = os.path.join(tmpdir_path, XMLFILE)  
 
-    importer = importxml.importData
+    importer = importData
     info = importer(database, imp_db_name, user)
     newmediapath = database.get_mediapath()
     #import of gpkg should not change media path as all media has new paths!

gramps/plugins/import/importprogen.py

@@ -53,7 +53,7 @@ from gramps.gui.utils import ProgressMeter
 from gramps.gen.lib import (Attribute, AttributeType, ChildRef, Date, Event, 
                             EventRef, EventType, Family, FamilyRelType, Name, 
                             NameType, Note, NoteType, Person, Place, Source, 
-                            SourceRef, Surname)
+                            Surname)
 from gramps.gen.db import DbTxn
 
 class ProgenError(Exception):
@@ -153,17 +153,17 @@ def _read_recs(table, bname):
 
 
 def _get_defname(fname):
-    '''
+    """
     Get the name of the PG30 DEF file by looking at the user DEF file. And return
     the name of the DEF file. fname is expected to be somewhere in the PG30 tree.
 
     Contents of <fname> is something like:
-        => \0
+        => \\0
-        => C:\PG30\NL\PG30-1.DEF
+        => C:\\PG30\\NL\\PG30-1.DEF
 
     We will strip the C: and convert the rest to a native pathname. Next, this pathname
     is compared with <fname>.
-    '''
+    """
     lines = open(fname).readlines()
     if not lines[0].startswith(r'\0') or len(lines) < 2:
         raise ProgenError(_("Not a Pro-Gen file"))
@@ -596,6 +596,7 @@ class ProgenParser(object):
             self.db.add_source(source, self.trans)
             self.db.commit_source(source, self.trans)
             self.skeys[source_name] = source.get_handle()
+        raise Error('Sourceref no longer exists, this piece must be converted to citation!')
         sref = SourceRef()
         sref.set_reference_handle(source.get_handle())
         return sref

gramps/plugins/import/importxml.py

@@ -437,7 +437,10 @@ class ImportOpenFileContextManager:
             if use_gzip:
                 xml_file = gzip.open(filename, "rb")
             else:
+                if sys.version_info[0] < 3:
                     xml_file = open(filename, "r")
+                else:
+                    xml_file = open(filename, "rb")
         except IOError as msg:
             self.user.notify_error(_("%s could not be opened") % filename, str(msg))
             xml_file = None

gramps/plugins/lib/libholiday.py

@@ -302,7 +302,10 @@ class _Xml2Obj:
         parser.StartElementHandler = self.start_element
         parser.EndElementHandler = self.end_element
         # Parse the XML File
+        if sys.version_info[0] < 3:
             parser.Parse(open(filename, 'r').read(), 1)
+        else:
+            parser.Parse(open(filename, 'rb').read(), 1)
         return self.root
 
 #------------------------------------------------------------------------