0006713: Databases written with pickle protocol 3 (Python3) should not be opened with pickle protocol 2 (Python2). Also give warning when about to upgrade a Python2 database to Python3.

svn: r22243

gramps/cli/grampscli.py

@@ -55,7 +55,13 @@ from gramps.gen.errors import DbError
 from gramps.gen.dbstate import DbState
 from gramps.gen.db import DbBsddb
 from gramps.gen.db.exceptions import (DbUpgradeRequiredError, 
-                                      DbVersionError)
+                                      BsddbDowngradeError, 
+                                      DbVersionError, 
+                                      DbEnvironmentError,
+                                      BsddbUpgradeRequiredError,
+                                      BsddbDowngradeRequiredError,
+                                      PythonUpgradeRequiredError,
+                                      PythonDowngradeError)
 from gramps.gen.plug import BasePluginManager
 from gramps.gen.utils.config import get_researcher
 from gramps.gen.recentfiles import recent_files
@@ -158,29 +164,35 @@ class CLIDbLoader(object):
         try:
             self.dbstate.db.load(filename, self._pulse_progress, mode)
             self.dbstate.db.set_save_path(filename)
-        except gen.db.exceptions.DbEnvironmentError as msg:
+        except DbEnvironmentError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))
-        except gen.db.exceptions.BsddbUpgradeRequiredError as msg:
+        except BsddbUpgradeRequiredError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))
-        except gen.db.exceptions.BsddbDowngradeRequiredError as msg:
+        except BsddbDowngradeRequiredError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))
-        except gen.db.exceptions.BsddbDowngradeError as msg:
+        except BsddbDowngradeError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))
-        except gen.db.exceptions.DbUpgradeRequiredError as msg:
+        except DbUpgradeRequiredError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))
-        except gen.db.exceptions.DbVersionError as msg:
+        except PythonDowngradeError as msg:
+            self.dbstate.no_database()
+            self._errordialog( _("Cannot open database"), str(msg))
+        except PythonUpgradeRequiredError as msg:
+            self.dbstate.no_database()
+            self._errordialog( _("Cannot open database"), str(msg))
+        except DbVersionError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))
         except OSError as msg:
             self.dbstate.no_database()
             self._errordialog(
                 _("Could not open file: %s") % filename, str(msg))
-        except Errors.DbError as msg:
+        except DbError as msg:
             self.dbstate.no_database()
             self._dberrordialog(msg)
         except Exception:

gramps/gen/db/exceptions.py

@@ -218,3 +218,59 @@ class DbUpgradeRequiredError(Exception):
                  'of your Family Tree.') % \
                  {'oldschema': self.oldschema,
                   'newschema': self.newschema}
+                 
+class PythonDowngradeError(Exception):
+    """
+    Error used to report that the Python version used to create the family tree
+    (i.e. Python3) is of a version that is newer than the current version
+    (i.e.Python2), so the Family Tree cannot be opened
+    """
+    def __init__(self, db_python_version, current_python_version):
+        Exception.__init__(self)
+        self.db_python_version = str(db_python_version)
+        self.current_python_version = str(current_python_version)
+
+    def __str__(self):
+        return _('The Family Tree you are trying to load was created with '
+                 'Python version %(db_python_version)s. This version of Gramps '
+                 'uses Python version %(current_python_version)s.  So you are '
+                 'trying to load '
+                 'data created in a newer format into an older program, and '
+                 'this is bound to fail.\n\n'
+                 'You should start your <b>newer</b> version of Gramps and '
+                 '<a href="http://www.gramps-project.org/wiki/index.php?title=How_to_make_a_backup">'
+                 'make a backup</a> of your Family Tree. You can then import '
+                 'this backup into this version of Gramps.') % \
+                 {'db_python_version': self.db_python_version,
+                 'current_python_version': self.current_python_version}
+
+class PythonUpgradeRequiredError(Exception):
+    """
+    Error used to report that the Python version used to create the family tree
+    (i.e. Python2) is earlier than the current Python version (i.e. Python3), so
+    the Family Tree needs to be upgraded..
+    """
+    def __init__(self, db_python_version, current_python_version):
+        Exception.__init__(self)
+        self.db_python_version = str(db_python_version)
+        self.current_python_version = str(current_python_version)
+
+    def __str__(self):
+        return _('The Family Tree you are trying to load is in the Python '
+                 'version %(db_python_version)s format. This version of Gramps '
+                 'uses Python version %(current_python_version)s. Therefore '
+                 'you cannot load this Family Tree without upgrading the '
+                 'Python version of the Family Tree.\n\n'
+                 'If you upgrade then you won\'t be able to use the previous '
+                 'version of Gramps, even if you subsequently '
+                 '<a href="http://www.gramps-project.org/wiki/index.php?title=Gramps_4.0_Wiki_Manual_-_Manage_Family_Trees#Backing_up_a_Family_Tree">backup</a> '
+                 'or <a href="http://www.gramps-project.org/wiki/index.php?title=Gramps_4.0_Wiki_Manual_-_Manage_Family_Trees#Export_into_Gramps_formats">export</a> '
+                 'your upgraded Family Tree.\n\n'
+                 'Upgrading is a difficult task which could irretrievably '
+                 'corrupt your Family Tree if it is interrupted or fails.\n\n'
+                 'If you have not already made a backup of your Family Tree, '
+                 'then you should start your <b>old</b> version of Gramps and '
+                 '<a href="http://www.gramps-project.org/wiki/index.php?title=How_to_make_a_backup">make a backup</a> '
+                 'of your Family Tree.') % \
+                 {'db_python_version': self.db_python_version,
+                 'current_python_version': self.current_python_version}

gramps/gen/db/write.py

@@ -43,7 +43,7 @@ import time
 import bisect
 from functools import wraps
 import logging
-from sys import maxsize, getfilesystemencoding
+from sys import maxsize, getfilesystemencoding, version_info
 
 from ..config import config
 if config.get('preferences.use-bsddb3') or sys.version_info[0] >= 3:
@@ -270,6 +270,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.has_changed = False
         self.brief_name = None
         self.update_env_version = False
+        self.update_python_version = False
 
     def catch_db_error(func):
         """
@@ -438,6 +439,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         else:
             # bsddb version is unknown
             env_version = "Unknown"
+#        _LOG.debug("db version %s, program version %s" % (bsddb_version, bdb_version))
 
         if env_version == "Unknown" or \
             (env_version[0] < bdb_version[0]) or \
@@ -488,6 +490,47 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
             # This can't happen
             raise "Comparison between Bsddb version failed"
 
+    def __check_python_version(self, name, force_python_upgrade=False):
+        """
+        The 'pickle' format (may) change with each Python version, see
+        http://docs.python.org/3.2/library/pickle.html#pickle. Code commits
+        21777 and 21778 ensure that when going from python2 to python3, the old
+        format can be read. However, once the data has been written in the
+        python3 format, it will not be possible to go back to pyton2. This check
+        test whether we are changing python versions. If going from 2 to 3 it
+        warns the user, and allows it if he confirms. When going from 3 to 3, an
+        error is raised. Because code for python2 did not write the Python
+        version file, if the file is absent, python2 is assumed.
+        """
+        current_python_version = version_info[0]
+        versionpath = os.path.join(self.path, "pythonversion.txt")
+        if os.path.isfile(versionpath):
+            with open(versionpath, "r") as version_file:
+                db_python_version = int(version_file.read().strip())
+        else:
+            db_python_version = 2
+            
+        if db_python_version == 3 and current_python_version == 2:
+            clear_lock_file(name)
+            raise exceptions.PythonDowngradeError(db_python_version,
+                                                  current_python_version)
+        elif db_python_version == 2 and current_python_version > 2:
+            if not force_python_upgrade:
+                _LOG.debug("Python upgrade required from %s to %s" %
+                           (db_python_version, current_python_version))
+                clear_lock_file(name)
+                raise exceptions.PythonUpgradeRequiredError(db_python_version,
+                                                        current_python_version)
+            # Try to do an upgrade
+            if not self.readonly:
+                _LOG.warning("Python upgrade requested from %s to %s" %
+                             (db_python_version, current_python_version))
+                self.update_python_version = True
+            # Make a backup of the database files anyway 
+            self.__make_zip_backup(name)
+        elif db_python_version == 2 and current_python_version == 2:
+            pass
+    
     @catch_db_error
     def version_supported(self):
         dbversion = self.metadata.get(b'version', default=0)
@@ -521,7 +564,8 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
 
     @catch_db_error
     def load(self, name, callback, mode=DBMODE_W, force_schema_upgrade=False,
-             force_bsddb_upgrade=False, force_bsddb_downgrade=False):
+             force_bsddb_upgrade=False, force_bsddb_downgrade=False,
+             force_python_upgrade=False):
 
         if self.__check_readonly(name):
             mode = DBMODE_R
@@ -541,8 +585,14 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         self.path = self.full_name
         self.brief_name = os.path.basename(name)
 
-        self.__check_bdb_version(name, force_bsddb_upgrade,
+        # If we re-enter load with force_python_upgrade True, then we have
-                                 force_bsddb_downgrade)
+        # already checked the bsddb version, and then checked python version,
+        # and are agreeing on the upgrade
+        if not force_python_upgrade:
+            self.__check_bdb_version(name, force_bsddb_upgrade,
+                                     force_bsddb_downgrade)
+        
+        self.__check_python_version(name, force_python_upgrade)
 
         # Set up database environment
         self.env = db.DBEnv()
@@ -656,6 +706,16 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
                 version_file.write(version)
             _LOG.debug("Updated BDBVERSFN file to %s" % str(db.version()))
 
+        if self.update_python_version:
+            versionpath = os.path.join(name, "pythonversion.txt")
+            version = str(version_info[0])
+            if sys.version_info[0] < 3:
+                if isinstance(version, UNITYPE):
+                    version = version.encode('utf-8')
+            _LOG.debug("Updated python version file to %s" % version)
+            with open(versionpath, "w") as version_file:
+                version_file.write(version)
+            
         # Here we take care of any changes in the tables related to new code.
         # If secondary indices change, then they should removed
         # or rebuilt by upgrade as well. In any case, the
@@ -1207,7 +1267,7 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
                 txn.put(b'surname_list', self.surname_list)
 
         self.metadata.close()
-
+    
     def __close_early(self):
         """
         Bail out if the incompatible version is discovered:
@@ -2162,6 +2222,15 @@ class DbBsddb(DbBsddbRead, DbWriteBase, UpdateCallback):
         with open(versionpath, "w") as version_file:
             version_file.write(version)
 
+        versionpath = os.path.join(name, "pythonversion.txt")
+        version = str(version_info[0])
+        if sys.version_info[0] < 3:
+            if isinstance(version, UNITYPE):
+                version = version.encode('utf-8')
+        _LOG.debug("Write python version file to %s" % version)
+        with open(versionpath, "w") as version_file:
+            version_file.write(version)
+
         self.metadata.close()
         self.env.close()
   

gramps/gui/dbloader.py

@@ -63,7 +63,9 @@ from gramps.gen.db.exceptions import (DbUpgradeRequiredError,
                                       DbVersionError, 
                                       DbEnvironmentError,
                                       BsddbUpgradeRequiredError,
-                                      BsddbDowngradeRequiredError)
+                                      BsddbDowngradeRequiredError,
+                                      PythonUpgradeRequiredError,
+                                      PythonDowngradeError)
 from gramps.gen.constfunc import STRTYPE
 from gramps.gen.utils.file import get_unicode_path_from_file_chooser
 from .pluginmanager import GuiPluginManager
@@ -309,13 +311,15 @@ class DbLoader(CLIDbLoader):
         force_schema_upgrade = False
         force_bsddb_upgrade = False
         force_bsddb_downgrade = False
+        force_python_upgrade = False
         try:
             while True:
                 try:
                     db.load(filename, self._pulse_progress, 
                             mode, force_schema_upgrade,
                             force_bsddb_upgrade,
-                            force_bsddb_downgrade)
+                            force_bsddb_downgrade,
+                            force_python_upgrade)
                     db.set_save_path(filename)
                     self.dbstate.change_database(db)
                     break
@@ -329,6 +333,7 @@ class DbLoader(CLIDbLoader):
                         force_schema_upgrade = True
                         force_bsddb_upgrade = False
                         force_bsddb_downgrade = False
+                        force_python_upgrade = False
                     else:
                         self.dbstate.no_database()
                         break
@@ -342,6 +347,7 @@ class DbLoader(CLIDbLoader):
                         force_schema_upgrade = False
                         force_bsddb_upgrade = True
                         force_bsddb_downgrade = False
+                        force_python_upgrade = False
                     else:
                         self.dbstate.no_database()
                         break
@@ -355,6 +361,21 @@ class DbLoader(CLIDbLoader):
                         force_schema_upgrade = False
                         force_bsddb_upgrade = False
                         force_bsddb_downgrade = True
+                        force_python_upgrade = False
+                    else:
+                        self.dbstate.no_database()
+                        break
+                except PythonUpgradeRequiredError as msg:
+                    if QuestionDialog2(_("Are you sure you want to upgrade "
+                                         "this Family Tree?"), 
+                                       str(msg), 
+                                       _("I have made a backup,\n"
+                                         "please upgrade my Family Tree"), 
+                                       _("Cancel"), self.uistate.window).run():
+                        force_schema_upgrade = False
+                        force_bsddb_upgrade = False
+                        force_bsddb_downgrade = False
+                        force_python_upgrade = True
                     else:
                         self.dbstate.no_database()
                         break
@@ -368,6 +389,9 @@ class DbLoader(CLIDbLoader):
         except DbEnvironmentError as msg:
             self.dbstate.no_database()
             self._errordialog( _("Cannot open database"), str(msg))
+        except PythonDowngradeError as msg:
+            self.dbstate.no_database()
+            self._warn( _("Cannot open database"), str(msg))
         except OSError as msg:
             self.dbstate.no_database()
             self._errordialog(

gramps/gui/dialog.py

@@ -112,7 +112,7 @@ class QuestionDialog(object):
         if response == Gtk.ResponseType.ACCEPT:
             task()
 
-from display import display_url
+from gramps.gui.display import display_url
 def on_activate_link(label, uri):
     # see aboutdialog.py _show_url()
     display_url(uri)